From 68920fd239f6df8e3a6392e02cf2947e38047234 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Tue, 19 Nov 2024 23:24:52 +0100 Subject: [PATCH 001/114] Move file versioning.py into subdirectory "versioning". --- scons/changelog.py | 2 +- scons/sconstruct.py | 2 +- scons/{ => versioning}/versioning.py | 16 ++++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) rename scons/{ => versioning}/versioning.py (93%) diff --git a/scons/changelog.py b/scons/changelog.py index 8c7a5ab1b..d27a48daa 100644 --- a/scons/changelog.py +++ b/scons/changelog.py @@ -11,7 +11,7 @@ from os.path import isfile from typing import List, Optional -from versioning import Version, get_current_version +from versioning.versioning import Version, get_current_version PREFIX_HEADER = '# ' diff --git a/scons/sconstruct.py b/scons/sconstruct.py index b8b6b8ecf..36d7e6f7a 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -19,7 +19,7 @@ from modules import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE from packaging import build_python_wheel, install_python_wheels from testing import tests_cpp, tests_python -from versioning import apply_development_version, increment_development_version, increment_major_version, \ +from versioning.versioning import apply_development_version, increment_development_version, increment_major_version, \ increment_minor_version, increment_patch_version, print_current_version, reset_development_version from SCons.Script import COMMAND_LINE_TARGETS diff --git a/scons/versioning.py b/scons/versioning/versioning.py similarity index 93% rename from scons/versioning.py rename to scons/versioning/versioning.py index d8f0aee31..61872770c 100644 --- a/scons/versioning.py +++ b/scons/versioning/versioning.py @@ -1,7 +1,7 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Provides utility functions for updating the project's version. +Provides actions for updating the project's version. """ import sys @@ -115,14 +115,14 @@ def get_current_version() -> Version: return __parse_version(__read_version_file(VERSION_FILE)) -def print_current_version(**_): +def print_current_version(): """ Prints the project's current version. """ return print(str(get_current_version())) -def increment_development_version(**_): +def increment_development_version(): """ Increments the development version. """ @@ -131,7 +131,7 @@ def increment_development_version(**_): __update_development_version(dev) -def reset_development_version(**_): +def reset_development_version(): """ Resets the development version. """ @@ -139,7 +139,7 @@ def reset_development_version(**_): __update_development_version(0) -def apply_development_version(**_): +def apply_development_version(): """ Appends the development version to the current semantic version. """ @@ -148,7 +148,7 @@ def apply_development_version(**_): __update_version(version) -def increment_patch_version(**_): +def increment_patch_version(): """ Increments the patch version. """ @@ -157,7 +157,7 @@ def increment_patch_version(**_): __update_version(version) -def increment_minor_version(**_): +def increment_minor_version(): """ Increments the minor version. """ @@ -167,7 +167,7 @@ def increment_minor_version(**_): __update_version(version) -def increment_major_version(**_): +def increment_major_version(): """ Increments the major version. """ From 2c65e21567376b26c38a62ac140a4cacff49a0d8 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 20 Nov 2024 16:07:27 +0100 Subject: [PATCH 002/114] Add functions "parse" and "parse_version_number" to class Version. --- scons/versioning/versioning.py | 68 +++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/scons/versioning/versioning.py b/scons/versioning/versioning.py index 61872770c..a135cd132 100644 --- a/scons/versioning/versioning.py +++ b/scons/versioning/versioning.py @@ -31,6 +31,42 @@ class Version: patch: int dev: Optional[int] = None + @staticmethod + def parse_version_number(version_number: str) -> int: + """ + Parses and returns a single version number from a given string. + + :param version_number: The string to be parsed + :return: The version number that has been parsed + """ + try: + number = int(version_number) + + if number < 0: + raise ValueError() + + return number + except ValueError as error: + raise ValueError('Version numbers must be non-negative integers, but got: ' + version_number) from error + + @staticmethod + def parse(version: str) -> 'Version': + """ + Parses and returns a version from a given string. + + :param version: The string to be parsed + :return: The version that has been parsed + """ + parts = version.split('.') + + if len(parts) != 3: + raise ValueError('Version must be given in format MAJOR.MINOR.PATCH, but got: ' + version) + + major = Version.parse_version_number(parts[0]) + minor = Version.parse_version_number(parts[1]) + patch = Version.parse_version_number(parts[2]) + return Version(major=major, minor=minor, patch=patch) + def __str__(self) -> str: version = str(self.major) + '.' + str(self.minor) + '.' + str(self.patch) @@ -56,23 +92,10 @@ def __write_version_file(version_file, version: str): file.write(version) -def __parse_version_number(version_number: str) -> int: - try: - number = int(version_number) - - if number < 0: - raise ValueError() - - return number - except ValueError: - print('Version numbers must only consist of non-negative integers, but got: ' + version_number) - sys.exit(-1) - - def __get_current_development_version() -> int: current_version = __read_version_file(DEV_VERSION_FILE) print('Current development version is "' + current_version + '"') - return __parse_version_number(current_version) + return Version.parse_version_number(current_version) def __update_development_version(dev: int): @@ -81,23 +104,10 @@ def __update_development_version(dev: int): __write_version_file(DEV_VERSION_FILE, updated_version) -def __parse_version(version: str) -> Version: - parts = version.split('.') - - if len(parts) != 3: - print('Version must be given in format MAJOR.MINOR.PATCH or MAJOR.MINOR.PATCH.devN, but got: ' + version) - sys.exit(-1) - - major = __parse_version_number(parts[0]) - minor = __parse_version_number(parts[1]) - patch = __parse_version_number(parts[2]) - return Version(major=major, minor=minor, patch=patch) - - def __get_current_version() -> Version: current_version = __read_version_file(VERSION_FILE) print('Current version is "' + current_version + '"') - return __parse_version(current_version) + return Version.parse(current_version) def __update_version(version: Version): @@ -112,7 +122,7 @@ def get_current_version() -> Version: :return: The project's current version """ - return __parse_version(__read_version_file(VERSION_FILE)) + return Version.parse(__read_version_file(VERSION_FILE)) def print_current_version(): From 8ab814004fa4e42ff78ee247c897fd112316e6a0 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Tue, 19 Nov 2024 23:33:41 +0100 Subject: [PATCH 003/114] Add utility functions "read_file" and "write_file". --- scons/documentation.py | 5 +++-- scons/github_actions.py | 9 ++++----- scons/util/__init__.py | 0 scons/util/io.py | 25 +++++++++++++++++++++++++ scons/versioning/versioning.py | 8 ++++---- 5 files changed, 36 insertions(+), 11 deletions(-) create mode 100644 scons/util/__init__.py create mode 100644 scons/util/io.py diff --git a/scons/documentation.py b/scons/documentation.py index d2bca9f95..792cf80a7 100644 --- a/scons/documentation.py +++ b/scons/documentation.py @@ -9,6 +9,7 @@ from environment import set_env from modules import CPP_MODULE, DOC_MODULE, PYTHON_MODULE from run import run_program +from util.io import read_file, write_file def __doxygen(project_name: str, input_dir: str, output_dir: str): @@ -77,7 +78,7 @@ def __sphinx_build(source_dir: str, output_dir: str): def __read_tocfile_template(directory: str) -> List[str]: - with open(path.join(directory, 'index.md.template'), mode='r', encoding='utf-8') as file: + with read_file(path.join(directory, 'index.md.template')) as file: return file.readlines() @@ -91,7 +92,7 @@ def __write_tocfile(directory: str, tocfile_entries: List[str]): else: tocfile.append(line) - with open(path.join(directory, 'index.md'), mode='w', encoding='utf-8') as file: + with write_file(path.join(directory, 'index.md')) as file: file.writelines(tocfile) diff --git a/scons/github_actions.py b/scons/github_actions.py index d2bd3a981..0cea1ab06 100644 --- a/scons/github_actions.py +++ b/scons/github_actions.py @@ -13,11 +13,10 @@ from dependencies import install_build_dependencies from environment import get_env +from util.io import read_file, write_file ENV_GITHUB_TOKEN = 'GITHUB_TOKEN' -WORKFLOW_ENCODING = 'utf-8' - @dataclass class ActionVersion: @@ -176,18 +175,18 @@ def __read_workflow(workflow_file: str) -> Workflow: install_build_dependencies('pyyaml') # pylint: disable=import-outside-toplevel import yaml - with open(workflow_file, mode='r', encoding=WORKFLOW_ENCODING) as file: + with read_file(workflow_file) as file: yaml_dict = yaml.load(file.read(), Loader=yaml.CLoader) return Workflow(workflow_file=workflow_file, yaml_dict=yaml_dict) def __read_workflow_lines(workflow_file: str) -> List[str]: - with open(workflow_file, mode='r', encoding=WORKFLOW_ENCODING) as file: + with read_file(workflow_file) as file: return file.readlines() def __write_workflow_lines(workflow_file: str, lines: List[str]): - with open(workflow_file, mode='w', encoding=WORKFLOW_ENCODING) as file: + with write_file(workflow_file) as file: file.writelines(lines) diff --git a/scons/util/__init__.py b/scons/util/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scons/util/io.py b/scons/util/io.py new file mode 100644 index 000000000..d39fe400d --- /dev/null +++ b/scons/util/io.py @@ -0,0 +1,25 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for reading and writing files. +""" + +ENCODING_UTF8 = 'utf-8' + + +def read_file(file: str): + """ + Opens a file to read from. + + :param file: The file to be opened + """ + return open(file, mode='r', encoding=ENCODING_UTF8) + + +def write_file(file: str): + """ + Opens a file to be written to. + + :param file: The file to be opened + """ + return open(file, mode='w', encoding=ENCODING_UTF8) diff --git a/scons/versioning/versioning.py b/scons/versioning/versioning.py index a135cd132..f4f722f2e 100644 --- a/scons/versioning/versioning.py +++ b/scons/versioning/versioning.py @@ -8,12 +8,12 @@ from dataclasses import dataclass from typing import Optional +from util.io import read_file, write_file + VERSION_FILE = '.version' DEV_VERSION_FILE = '.version-dev' -VERSION_FILE_ENCODING = 'utf-8' - @dataclass class Version: @@ -77,7 +77,7 @@ def __str__(self) -> str: def __read_version_file(version_file) -> str: - with open(version_file, mode='r', encoding=VERSION_FILE_ENCODING) as file: + with read_file(version_file) as file: lines = file.readlines() if len(lines) != 1: @@ -88,7 +88,7 @@ def __read_version_file(version_file) -> str: def __write_version_file(version_file, version: str): - with open(version_file, mode='w', encoding=VERSION_FILE_ENCODING) as file: + with write_file(version_file) as file: file.write(version) From 524c834bf1b074942a16ced03b62122a9fc5ba39 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Tue, 19 Nov 2024 23:36:19 +0100 Subject: [PATCH 004/114] Move file changelog.py into subdirectory "versioning". --- scons/sconstruct.py | 4 ++-- scons/{ => versioning}/changelog.py | 27 +++++++++++++-------------- 2 files changed, 15 insertions(+), 16 deletions(-) rename scons/{ => versioning}/changelog.py (95%) diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 36d7e6f7a..42eaa27f6 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -8,8 +8,6 @@ from functools import reduce from os import path -from changelog import print_latest_changelog, update_changelog_bugfix, update_changelog_feature, \ - update_changelog_main, validate_changelog_bugfix, validate_changelog_feature, validate_changelog_main from code_style import check_cpp_code_style, check_md_code_style, check_python_code_style, check_yaml_code_style, \ enforce_cpp_code_style, enforce_md_code_style, enforce_python_code_style, enforce_yaml_code_style from compilation import compile_cpp, compile_cython, install_cpp, install_cython, setup_cpp, setup_cython @@ -19,6 +17,8 @@ from modules import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE from packaging import build_python_wheel, install_python_wheels from testing import tests_cpp, tests_python +from versioning.changelog import print_latest_changelog, update_changelog_bugfix, update_changelog_feature, \ + update_changelog_main, validate_changelog_bugfix, validate_changelog_feature, validate_changelog_main from versioning.versioning import apply_development_version, increment_development_version, increment_major_version, \ increment_minor_version, increment_patch_version, print_current_version, reset_development_version diff --git a/scons/changelog.py b/scons/versioning/changelog.py similarity index 95% rename from scons/changelog.py rename to scons/versioning/changelog.py index d27a48daa..976f5e6c3 100644 --- a/scons/changelog.py +++ b/scons/versioning/changelog.py @@ -1,16 +1,17 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Provides utility functions for validating and updating the project's changelog. +Provides actions for validating and updating the project's changelog. """ import sys from dataclasses import dataclass, field from datetime import date from enum import Enum, auto -from os.path import isfile +from os import path from typing import List, Optional +from util.io import read_file, write_file from versioning.versioning import Version, get_current_version PREFIX_HEADER = '# ' @@ -33,8 +34,6 @@ CHANGELOG_FILE = 'CHANGELOG.md' -CHANGELOG_ENCODING = 'utf-8' - class LineType(Enum): """ @@ -159,15 +158,15 @@ def __str__(self) -> str: def __read_lines(changelog_file: str, skip_if_missing: bool = False) -> List[str]: - if skip_if_missing and not isfile(changelog_file): + if skip_if_missing and not path.isfile(changelog_file): return [] - with open(changelog_file, mode='r', encoding=CHANGELOG_ENCODING) as file: + with read_file(changelog_file) as file: return file.readlines() def __write_lines(changelog_file: str, lines: List[str]): - with open(changelog_file, mode='w', encoding=CHANGELOG_ENCODING) as file: + with write_file(changelog_file) as file: file.writelines(lines) @@ -321,49 +320,49 @@ def __get_latest_changelog() -> str: return changelog.rstrip('\n') -def validate_changelog_bugfix(**_): +def validate_changelog_bugfix(): """ Validates the changelog file that lists bugfixes. """ __validate_changelog(CHANGELOG_FILE_BUGFIX) -def validate_changelog_feature(**_): +def validate_changelog_feature(): """ Validates the changelog file that lists new features. """ __validate_changelog(CHANGELOG_FILE_FEATURE) -def validate_changelog_main(**_): +def validate_changelog_main(): """ Validates the changelog file that lists major updates. """ __validate_changelog(CHANGELOG_FILE_MAIN) -def update_changelog_main(**_): +def update_changelog_main(): """ Updates the projects changelog when releasing bugfixes. """ __update_changelog(ReleaseType.MAJOR, CHANGELOG_FILE_MAIN, CHANGELOG_FILE_FEATURE, CHANGELOG_FILE_BUGFIX) -def update_changelog_feature(**_): +def update_changelog_feature(): """ Updates the project's changelog when releasing new features. """ __update_changelog(ReleaseType.MINOR, CHANGELOG_FILE_FEATURE, CHANGELOG_FILE_BUGFIX) -def update_changelog_bugfix(**_): +def update_changelog_bugfix(): """ Updates the project's changelog when releasing major updates. """ __update_changelog(ReleaseType.PATCH, CHANGELOG_FILE_BUGFIX) -def print_latest_changelog(**_): +def print_latest_changelog(): """ Prints the changelog of the latest release. """ From 53c2631058e8c422fe7f6835096ae7a4d2f2258a Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 20 Nov 2024 00:08:04 +0100 Subject: [PATCH 005/114] Add utility function "import_source_file". --- scons/util/reflection.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 scons/util/reflection.py diff --git a/scons/util/reflection.py b/scons/util/reflection.py new file mode 100644 index 000000000..2129dea22 --- /dev/null +++ b/scons/util/reflection.py @@ -0,0 +1,26 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for importing and executing Python code at runtime. +""" +import sys + +from importlib.util import module_from_spec, spec_from_file_location +from types import ModuleType as Module + + +def import_source_file(source_file: str) -> Module: + """ + Imports a given source file. + + :param source_file: The path to the source file + :return: The module that has been imported + """ + try: + spec = spec_from_file_location(source_file, source_file) + module = module_from_spec(spec) + sys.modules[source_file] = module + spec.loader.exec_module(module) + return module + except FileNotFoundError as error: + raise ImportError('Source file "' + source_file + '" not found') from error From 3a486cf9e5a6cacc9f6d6b94ead236619a41b3c2 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 20 Nov 2024 00:08:22 +0100 Subject: [PATCH 006/114] Add class DirectorySearch. --- scons/util/files.py | 77 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 scons/util/files.py diff --git a/scons/util/files.py b/scons/util/files.py new file mode 100644 index 000000000..c7f5caf67 --- /dev/null +++ b/scons/util/files.py @@ -0,0 +1,77 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes for listing files and directories. +""" +from functools import partial, reduce +from glob import glob +from os import path +from typing import Callable, List + + +class DirectorySearch: + """ + Allows to search for subdirectories. + """ + + Filter = Callable[[str, str], bool] + + def __init__(self): + self.recursive = False + self.excludes = [] + + def set_recursive(self, recursive: bool) -> 'DirectorySearch': + """ + Sets whether the search should be recursive or not. + + :param recursive: True, if the search should be recursive, False otherwise + :return: The `DirectorySearch` itself + """ + self.recursive = recursive + return self + + def exclude(self, *excludes: Filter) -> 'DirectorySearch': + """ + Sets one or several filters that should be used for excluding subdirectories. + + :param excludes: The filters to be set + :return: The `DirectorySearch` itself + """ + self.excludes.extend(excludes) + return self + + def exclude_by_name(self, *names: str) -> 'DirectorySearch': + """ + Sets one or several filters that should be used for excluding subdirectories by their names. + + :param names: The names of the subdirectories to be excluded + :return: The `DirectorySearch` itself + """ + + def filter_directory(excluded_name: str, _: str, directory_name: str): + return directory_name == excluded_name + + return self.exclude(*[partial(filter_directory, name) for name in names]) + + def list(self, *directories: str) -> List[str]: + """ + Lists all subdirectories that can be found in given directories. + + :param directories: The directories to search for subdirectories + :return: A list that contains all subdirectories that have been found + """ + result = [] + + def filter_file(file: str) -> bool: + return path.isdir(file) and not reduce( + lambda aggr, exclude: aggr or exclude(path.dirname(file), path.basename(file)), self.excludes, False) + + for directory in directories: + subdirectories = [file for file in glob(path.join(directory, '*')) if filter_file(file)] + + if self.recursive: + subdirectories.extend(self.list(*subdirectories)) + + result.extend(subdirectories) + + return result From 58d27d53b56136a611c70fc95952d72e6b91dc91 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 20 Nov 2024 00:09:57 +0100 Subject: [PATCH 007/114] Add class PhonyTarget. --- scons/util/targets.py | 46 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 scons/util/targets.py diff --git a/scons/util/targets.py b/scons/util/targets.py new file mode 100644 index 000000000..23c3791d1 --- /dev/null +++ b/scons/util/targets.py @@ -0,0 +1,46 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides base classes for defining individual targets of the build process. +""" +from abc import ABC, abstractmethod +from typing import Callable + +from SCons.Script.SConscript import SConsEnvironment as Environment + + +class Target(ABC): + """ + An abstract base class for all targets of the build system. + """ + + def __init__(self, name: str): + """ + :param name: The name of the target + """ + self.name = name + + @abstractmethod + def register(self, environment: Environment): + """ + Must be implemented by subclasses in order to register the target. + + :param environment: The environment, the target should be registered at + """ + + +class PhonyTarget(Target): + """ + A phony target, which executes a certain action and does not produce any output files. + """ + + def __init__(self, name: str, action: Callable[[], None]): + """ + :param name: The name of the target + :param action: The function to be executed by the target + """ + super().__init__(name) + self.action = action + + def register(self, environment: Environment): + return environment.AlwaysBuild(environment.Alias(self.name, None, lambda **_: self.action())) From 36b7115eb5a1bfd6562b471789f7cdb40b68d03c Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 20 Nov 2024 00:16:51 +0100 Subject: [PATCH 008/114] Dynamically register targets for updating the project's version and changelog. --- scons/sconstruct.py | 61 +++++++++--------------------------- scons/versioning/__init__.py | 32 +++++++++++++++++++ 2 files changed, 47 insertions(+), 46 deletions(-) create mode 100644 scons/versioning/__init__.py diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 42eaa27f6..a4f653199 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -17,10 +17,9 @@ from modules import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE from packaging import build_python_wheel, install_python_wheels from testing import tests_cpp, tests_python -from versioning.changelog import print_latest_changelog, update_changelog_bugfix, update_changelog_feature, \ - update_changelog_main, validate_changelog_bugfix, validate_changelog_feature, validate_changelog_main -from versioning.versioning import apply_development_version, increment_development_version, increment_major_version, \ - increment_minor_version, increment_patch_version, print_current_version, reset_development_version +from util.files import DirectorySearch +from util.reflection import import_source_file +from util.targets import Target from SCons.Script import COMMAND_LINE_TARGETS from SCons.Script.SConscript import SConsEnvironment @@ -36,20 +35,6 @@ def __print_if_clean(environment, message: str): # Define target names... -TARGET_NAME_INCREMENT_DEVELOPMENT_VERSION = 'increment_development_version' -TARGET_NAME_RESET_DEVELOPMENT_VERSION = 'reset_development_version' -TARGET_NAME_APPLY_DEVELOPMENT_VERSION = 'apply_development_version' -TARGET_NAME_INCREMENT_PATCH_VERSION = 'increment_patch_version' -TARGET_NAME_INCREMENT_MINOR_VERSION = 'increment_minor_version' -TARGET_NAME_INCREMENT_MAJOR_VERSION = 'increment_major_version' -TARGET_NAME_VALIDATE_CHANGELOG_BUGFIX = 'validate_changelog_bugfix' -TARGET_NAME_VALIDATE_CHANGELOG_FEATURE = 'validate_changelog_feature' -TARGET_NAME_VALIDATE_CHANGELOG_MAIN = 'validate_changelog_main' -TARGET_NAME_UPDATE_CHANGELOG_BUGFIX = 'update_changelog_bugfix' -TARGET_NAME_UPDATE_CHANGELOG_FEATURE = 'update_changelog_feature' -TARGET_NAME_UPDATE_CHANGELOG_MAIN = 'update_changelog_main' -TARGET_NAME_PRINT_VERSION = 'print_version' -TARGET_NAME_PRINT_LATEST_CHANGELOG = 'print_latest_changelog' TARGET_NAME_TEST_FORMAT = 'test_format' TARGET_NAME_TEST_FORMAT_PYTHON = TARGET_NAME_TEST_FORMAT + '_python' TARGET_NAME_TEST_FORMAT_CPP = TARGET_NAME_TEST_FORMAT + '_cpp' @@ -81,11 +66,6 @@ def __print_if_clean(environment, message: str): TARGET_NAME_DOC = 'doc' VALID_TARGETS = { - TARGET_NAME_INCREMENT_DEVELOPMENT_VERSION, TARGET_NAME_RESET_DEVELOPMENT_VERSION, - TARGET_NAME_APPLY_DEVELOPMENT_VERSION, TARGET_NAME_INCREMENT_PATCH_VERSION, TARGET_NAME_INCREMENT_MINOR_VERSION, - TARGET_NAME_INCREMENT_MAJOR_VERSION, TARGET_NAME_VALIDATE_CHANGELOG_BUGFIX, TARGET_NAME_VALIDATE_CHANGELOG_FEATURE, - TARGET_NAME_VALIDATE_CHANGELOG_MAIN, TARGET_NAME_UPDATE_CHANGELOG_BUGFIX, TARGET_NAME_UPDATE_CHANGELOG_FEATURE, - TARGET_NAME_UPDATE_CHANGELOG_MAIN, TARGET_NAME_PRINT_VERSION, TARGET_NAME_PRINT_LATEST_CHANGELOG, TARGET_NAME_TEST_FORMAT, TARGET_NAME_TEST_FORMAT_PYTHON, TARGET_NAME_TEST_FORMAT_CPP, TARGET_NAME_TEST_FORMAT_MD, TARGET_NAME_TEST_FORMAT_YAML, TARGET_NAME_FORMAT, TARGET_NAME_FORMAT_PYTHON, TARGET_NAME_FORMAT_CPP, TARGET_NAME_FORMAT_MD, TARGET_NAME_FORMAT_YAML, TARGET_NAME_DEPENDENCIES_CHECK, TARGET_NAME_GITHUB_ACTIONS_CHECK, @@ -97,6 +77,18 @@ def __print_if_clean(environment, message: str): DEFAULT_TARGET = TARGET_NAME_INSTALL_WHEELS +# Register build targets... +env = SConsEnvironment() + +for subdirectory in DirectorySearch().set_recursive(True).list(BUILD_MODULE.root_dir): + init_file = path.join(subdirectory, '__init__.py') + + if path.isfile(init_file): + for build_target in getattr(import_source_file(init_file), 'TARGETS', []): + if isinstance(build_target, Target): + build_target.register(env) + VALID_TARGETS.add(build_target.name) + # Raise an error if any invalid targets are given... invalid_targets = [target for target in COMMAND_LINE_TARGETS if target not in VALID_TARGETS] @@ -106,31 +98,8 @@ def __print_if_clean(environment, message: str): sys.exit(-1) # Create temporary file ".sconsign.dblite" in the build directory... -env = SConsEnvironment() env.SConsignFile(name=path.relpath(path.join(BUILD_MODULE.build_dir, '.sconsign'), BUILD_MODULE.root_dir)) -# Defines targets for updating the project's version... -__create_phony_target(env, TARGET_NAME_INCREMENT_DEVELOPMENT_VERSION, action=increment_development_version) -__create_phony_target(env, TARGET_NAME_RESET_DEVELOPMENT_VERSION, action=reset_development_version) -__create_phony_target(env, TARGET_NAME_APPLY_DEVELOPMENT_VERSION, action=apply_development_version) -__create_phony_target(env, TARGET_NAME_INCREMENT_PATCH_VERSION, action=increment_patch_version) -__create_phony_target(env, TARGET_NAME_INCREMENT_MINOR_VERSION, action=increment_minor_version) -__create_phony_target(env, TARGET_NAME_INCREMENT_MAJOR_VERSION, action=increment_major_version) - -# Define targets for validating changelogs... -__create_phony_target(env, TARGET_NAME_VALIDATE_CHANGELOG_BUGFIX, action=validate_changelog_bugfix) -__create_phony_target(env, TARGET_NAME_VALIDATE_CHANGELOG_FEATURE, action=validate_changelog_feature) -__create_phony_target(env, TARGET_NAME_VALIDATE_CHANGELOG_MAIN, action=validate_changelog_main) - -# Define targets for updating the project's changelog... -__create_phony_target(env, TARGET_NAME_UPDATE_CHANGELOG_BUGFIX, action=update_changelog_bugfix) -__create_phony_target(env, TARGET_NAME_UPDATE_CHANGELOG_FEATURE, action=update_changelog_feature) -__create_phony_target(env, TARGET_NAME_UPDATE_CHANGELOG_MAIN, action=update_changelog_main) - -# Define targets for printing information about the project... -__create_phony_target(env, TARGET_NAME_PRINT_VERSION, action=print_current_version) -__create_phony_target(env, TARGET_NAME_PRINT_LATEST_CHANGELOG, action=print_latest_changelog) - # Define targets for checking code style definitions... target_test_format_python = __create_phony_target(env, TARGET_NAME_TEST_FORMAT_PYTHON, action=check_python_code_style) target_test_format_cpp = __create_phony_target(env, TARGET_NAME_TEST_FORMAT_CPP, action=check_cpp_code_style) diff --git a/scons/versioning/__init__.py b/scons/versioning/__init__.py new file mode 100644 index 000000000..d433bc3ad --- /dev/null +++ b/scons/versioning/__init__.py @@ -0,0 +1,32 @@ +""" +Defines build targets for updating the project's version and changelog. +""" +from util.targets import PhonyTarget +from versioning.changelog import print_latest_changelog, update_changelog_bugfix, update_changelog_feature, \ + update_changelog_main, validate_changelog_bugfix, validate_changelog_feature, validate_changelog_main +from versioning.versioning import apply_development_version, increment_development_version, increment_major_version, \ + increment_minor_version, increment_patch_version, print_current_version, reset_development_version + +TARGETS = [ + # Targets for updating the project's version + PhonyTarget('increment_development_version', action=increment_development_version), + PhonyTarget('reset_development_version', action=reset_development_version), + PhonyTarget('apply_development_version', action=apply_development_version), + PhonyTarget('increment_patch_version', action=increment_patch_version), + PhonyTarget('increment_minor_version', action=increment_minor_version), + PhonyTarget('increment_major_version', action=increment_major_version), + + # Targets for validating changelogs + PhonyTarget('validate_changelog_bugfix', action=validate_changelog_bugfix), + PhonyTarget('validate_changelog_feature', action=validate_changelog_feature), + PhonyTarget('validate_changelog_main', action=validate_changelog_main), + + # Targets for updating the project's changelog + PhonyTarget('update_changelog_bugfix', action=update_changelog_bugfix), + PhonyTarget('update_changelog_feature', action=update_changelog_feature), + PhonyTarget('update_changelog_main', action=update_changelog_main), + + # Targets for printing information about the project + PhonyTarget(name='print_version', action=print_current_version), + PhonyTarget(name='print_latest_changelog', action=print_latest_changelog) +] From 916ad9e74e031b37edffe05f7c411fb77e37a3e2 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 20 Nov 2024 00:24:04 +0100 Subject: [PATCH 009/114] Move file command_line.py into subdirectory "util". --- scons/dependencies.py | 2 +- scons/run.py | 2 +- scons/{command_line.py => util/cmd.py} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename scons/{command_line.py => util/cmd.py} (100%) diff --git a/scons/dependencies.py b/scons/dependencies.py index 36b1f6b35..ae926fa92 100644 --- a/scons/dependencies.py +++ b/scons/dependencies.py @@ -9,8 +9,8 @@ from os import path from typing import List, Optional -from command_line import run_command from modules import ALL_MODULES, BUILD_MODULE, CPP_MODULE, PYTHON_MODULE, Module +from util.cmd import run_command @dataclass diff --git a/scons/run.py b/scons/run.py index 25cb1c3bc..3878097a5 100644 --- a/scons/run.py +++ b/scons/run.py @@ -5,9 +5,9 @@ """ from typing import List, Optional -from command_line import run_command from dependencies import install_dependencies from modules import BUILD_MODULE +from util.cmd import run_command def run_program(program: str, diff --git a/scons/command_line.py b/scons/util/cmd.py similarity index 100% rename from scons/command_line.py rename to scons/util/cmd.py From 5517ef049e45865f2ce4c5457b6e0cd9a8124bb9 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 20 Nov 2024 00:37:35 +0100 Subject: [PATCH 010/114] Move file environment.py into subdirectory "util". --- scons/compilation.py | 2 +- scons/documentation.py | 2 +- scons/github_actions.py | 2 +- scons/modules.py | 2 +- scons/testing.py | 2 +- scons/{environment.py => util/env.py} | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) rename scons/{environment.py => util/env.py} (96%) diff --git a/scons/compilation.py b/scons/compilation.py index 1651e3de9..e6add19c0 100644 --- a/scons/compilation.py +++ b/scons/compilation.py @@ -6,9 +6,9 @@ from os import environ from typing import List, Optional -from environment import get_env from modules import CPP_MODULE, PYTHON_MODULE from run import run_program +from util.env import get_env class BuildOptions: diff --git a/scons/documentation.py b/scons/documentation.py index 792cf80a7..0f0db244f 100644 --- a/scons/documentation.py +++ b/scons/documentation.py @@ -6,9 +6,9 @@ from os import environ, makedirs, path, remove from typing import List -from environment import set_env from modules import CPP_MODULE, DOC_MODULE, PYTHON_MODULE from run import run_program +from util.env import set_env from util.io import read_file, write_file diff --git a/scons/github_actions.py b/scons/github_actions.py index 0cea1ab06..5940c4078 100644 --- a/scons/github_actions.py +++ b/scons/github_actions.py @@ -12,7 +12,7 @@ from typing import List, Optional, Set from dependencies import install_build_dependencies -from environment import get_env +from util.env import get_env from util.io import read_file, write_file ENV_GITHUB_TOKEN = 'GITHUB_TOKEN' diff --git a/scons/modules.py b/scons/modules.py index 7746ac88a..f8f4fcd2a 100644 --- a/scons/modules.py +++ b/scons/modules.py @@ -8,7 +8,7 @@ from os import environ, path, walk from typing import Callable, List, Optional -from environment import get_env_array +from util.env import get_env_array def find_files_recursively(directory: str, diff --git a/scons/testing.py b/scons/testing.py index c50e76670..0fdcb1fcf 100644 --- a/scons/testing.py +++ b/scons/testing.py @@ -5,9 +5,9 @@ """ from os import environ, path -from environment import get_env_bool from modules import CPP_MODULE, PYTHON_MODULE from run import run_program, run_python_program +from util.env import get_env_bool def __meson_test(build_dir: str): diff --git a/scons/environment.py b/scons/util/env.py similarity index 96% rename from scons/environment.py rename to scons/util/env.py index 9d4814179..baf2d278a 100644 --- a/scons/environment.py +++ b/scons/util/env.py @@ -57,4 +57,4 @@ def set_env(env, name: str, value: str): :param value: The value to be set """ env[name] = value - print('Set environment variable \'' + name + '\' to value \'' + value + '\'') + print('Set environment variable "' + name + '" to value "' + value + '"') From a98d80262825c16a94cbaaee5c183bc8029a8336 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 20 Nov 2024 12:13:38 +0100 Subject: [PATCH 011/114] Add utility function "format_iterable". --- scons/sconstruct.py | 5 ++--- scons/util/cmd.py | 5 +++-- scons/util/format.py | 25 +++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 scons/util/format.py diff --git a/scons/sconstruct.py b/scons/sconstruct.py index a4f653199..0da020c6b 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -5,7 +5,6 @@ """ import sys -from functools import reduce from os import path from code_style import check_cpp_code_style, check_md_code_style, check_python_code_style, check_yaml_code_style, \ @@ -18,6 +17,7 @@ from packaging import build_python_wheel, install_python_wheels from testing import tests_cpp, tests_python from util.files import DirectorySearch +from util.format import format_iterable from util.reflection import import_source_file from util.targets import Target @@ -93,8 +93,7 @@ def __print_if_clean(environment, message: str): invalid_targets = [target for target in COMMAND_LINE_TARGETS if target not in VALID_TARGETS] if invalid_targets: - print('The following targets are unknown: ' - + reduce(lambda aggr, target: aggr + (', ' if len(aggr) > 0 else '') + target, invalid_targets, '')) + print('The following targets are unknown: ' + format_iterable(invalid_targets)) sys.exit(-1) # Create temporary file ".sconsign.dblite" in the build directory... diff --git a/scons/util/cmd.py b/scons/util/cmd.py index 257eb3df7..58e9c452c 100644 --- a/scons/util/cmd.py +++ b/scons/util/cmd.py @@ -6,12 +6,13 @@ import subprocess import sys -from functools import reduce from os import path +from util.format import format_iterable + def __format_command(cmd: str, *args, format_args: bool = True) -> str: - return cmd + (reduce(lambda aggr, argument: aggr + ' ' + argument, args, '') if format_args else '') + return cmd + (format_iterable(args, separator=' ') if format_args else '') def __is_virtual_environment() -> bool: diff --git a/scons/util/format.py b/scons/util/format.py new file mode 100644 index 000000000..73816e5cc --- /dev/null +++ b/scons/util/format.py @@ -0,0 +1,25 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for creating textual representations. +""" +from functools import reduce +from typing import Any, Callable, Iterable + + +def format_iterable(objects: Iterable[Any], + separator: str = ', ', + delimiter: str = '', + mapping: Callable[[Any], Any] = lambda x: x) -> str: + """ + Creates and returns a textual representation of objects in an iterable. + + :param objects: The iterable of objects to be formatted + :param separator: The string that should be used as a separator + :param delimiter: The string that should be added at the beginning and end of each object + :param mapping: An optional function that maps each object in the iterable to another one + :return: The textual representation that has been created + """ + return reduce( + lambda aggr, obj: aggr + (separator + if len(aggr) > 0 else '') + delimiter + str(mapping(obj)) + delimiter, objects, '') From 16f5760b0436f0f197363aa11ba97bd8bf8a1096 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 20 Nov 2024 12:25:24 +0100 Subject: [PATCH 012/114] Add utility function "in_virtual_environment". --- scons/util/cmd.py | 7 ++----- scons/util/venv.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 scons/util/venv.py diff --git a/scons/util/cmd.py b/scons/util/cmd.py index 58e9c452c..71baea6db 100644 --- a/scons/util/cmd.py +++ b/scons/util/cmd.py @@ -9,18 +9,15 @@ from os import path from util.format import format_iterable +from util.venv import in_virtual_environment def __format_command(cmd: str, *args, format_args: bool = True) -> str: return cmd + (format_iterable(args, separator=' ') if format_args else '') -def __is_virtual_environment() -> bool: - return sys.prefix != sys.base_prefix - - def __get_qualified_command(cmd: str) -> str: - if __is_virtual_environment(): + if in_virtual_environment(): # On Windows, we use the relative path to the command's executable within the virtual environment, if such an # executable exists. This circumvents situations where the PATH environment variable has not been updated after # activating the virtual environment. This can prevent the executables from being found or can lead to the wrong diff --git a/scons/util/venv.py b/scons/util/venv.py new file mode 100644 index 000000000..90c64dfe9 --- /dev/null +++ b/scons/util/venv.py @@ -0,0 +1,15 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for dealing with virtual Python environments. +""" +import sys + + +def in_virtual_environment() -> bool: + """ + Returns whether the current process is executed in a virtual environment or not. + + :return: True, if the current process is executed in a virtual environment, False otherwise + """ + return sys.prefix != sys.base_prefix From 68eb2db22c2e0face280f908143627aa6cc0181c Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 20 Nov 2024 13:42:43 +0100 Subject: [PATCH 013/114] Add class Command. --- scons/dependencies.py | 55 ++++++--- scons/run.py | 4 +- scons/util/cmd.py | 257 +++++++++++++++++++++++++++++++++--------- 3 files changed, 242 insertions(+), 74 deletions(-) diff --git a/scons/dependencies.py b/scons/dependencies.py index ae926fa92..086c163c5 100644 --- a/scons/dependencies.py +++ b/scons/dependencies.py @@ -4,13 +4,14 @@ Provides utility functions for dealing with dependencies. """ +from abc import ABC from dataclasses import dataclass from functools import reduce from os import path from typing import List, Optional from modules import ALL_MODULES, BUILD_MODULE, CPP_MODULE, PYTHON_MODULE, Module -from util.cmd import run_command +from util.cmd import Command @dataclass @@ -29,31 +30,52 @@ def __str__(self): return self.dependency + (self.version if self.version else '') -def __run_pip_command(*args, **kwargs): - return run_command('python', '-m', 'pip', *args, **kwargs) +class PipCommand(Command, ABC): + """ + An abstract base class for all classes that allow to run pip on the command line. + """ + + def __init__(self, *arguments: str): + super().__init__('python', '-m', 'pip', *arguments) -def __run_pip_install_command(requirement: Requirement, *args, **kwargs): - return __run_pip_command('install', str(requirement), '--upgrade', '--upgrade-strategy', 'eager', '--prefer-binary', - '--disable-pip-version-check', *args, **kwargs) +class PipInstallCommand(PipCommand): + """ + Allows to install a package via the command `pip install`. + """ + + def __init__(self, requirement: Requirement, *arguments: str): + """ + :requirement: Specifies the name and version of the package that should be installed + :arguments: Optional arguments to be passed to the command + """ + super().__init__('install', str(requirement), '--upgrade', '--upgrade-strategy', 'eager', '--prefer-binary', + '--disable-pip-version-check', *arguments) + + +class PipListCommand(PipCommand): + """ + Allows to list all installed Python packages via the command `pip list`. + """ + + def __init__(self, *arguments: str): + """ + :arguments: Optional arguments to be passed to the command + """ + super().__init__('list', *arguments) def __pip_install(requirement: Requirement, dry_run: bool = False): try: args = ['--dry-run'] if dry_run else [] - out = __run_pip_install_command(requirement, - *args, - print_cmd=False, - capture_output=True, - exit_on_error=not dry_run) - stdout = str(out.stdout).strip() - stdout_lines = stdout.split('\n') + stdout = PipInstallCommand(requirement, *args).print_command(False).exit_on_error(not dry_run).capture_output() + stdout_lines = stdout.strip().split('\n') if reduce( lambda aggr, line: aggr | line.startswith('Would install') and __normalize_dependency(line).find( requirement.dependency) >= 0, stdout_lines, False): if dry_run: - __run_pip_install_command(requirement, print_args=True) + PipInstallCommand(requirement).print_arguments(True).run() else: print(stdout) except RuntimeError: @@ -132,9 +154,8 @@ def check_dependency_versions(**_): __install_module_dependencies(module) print('Checking for outdated dependencies...') - out = __run_pip_command('list', '--outdated', print_cmd=False, capture_output=True) - stdout = str(out.stdout).strip() - stdout_lines = stdout.split('\n') + stdout = PipListCommand('--outdated').print_command(False).capture_output() + stdout_lines = stdout.strip().split('\n') i = 0 for line in stdout_lines: diff --git a/scons/run.py b/scons/run.py index 3878097a5..ad05a09b7 100644 --- a/scons/run.py +++ b/scons/run.py @@ -7,7 +7,7 @@ from dependencies import install_dependencies from modules import BUILD_MODULE -from util.cmd import run_command +from util.cmd import Command def run_program(program: str, @@ -37,7 +37,7 @@ def run_program(program: str, dependencies.extend(additional_dependencies) install_dependencies(requirements_file, *dependencies) - run_command(program, *args, print_args=print_args, env=env) + Command(program, *args).print_arguments(print_args).use_environment(env).run() def run_python_program(program: str, diff --git a/scons/util/cmd.py b/scons/util/cmd.py index 71baea6db..a9ad151bd 100644 --- a/scons/util/cmd.py +++ b/scons/util/cmd.py @@ -7,66 +7,213 @@ import sys from os import path +from subprocess import CompletedProcess from util.format import format_iterable from util.venv import in_virtual_environment -def __format_command(cmd: str, *args, format_args: bool = True) -> str: - return cmd + (format_iterable(args, separator=' ') if format_args else '') - - -def __get_qualified_command(cmd: str) -> str: - if in_virtual_environment(): - # On Windows, we use the relative path to the command's executable within the virtual environment, if such an - # executable exists. This circumvents situations where the PATH environment variable has not been updated after - # activating the virtual environment. This can prevent the executables from being found or can lead to the wrong - # executable, from outside the virtual environment, being executed. - executable = path.join(sys.prefix, 'Scripts', cmd + '.exe') - - if path.isfile(executable): - return executable - - return cmd - - -def run_command(cmd: str, - *args, - print_cmd: bool = True, - print_args: bool = False, - capture_output: bool = False, - exit_on_error: bool = True, - env=None): +class Command: """ - Runs a command line program. - - :param cmd: The name of the program to be run - :param args: Optional arguments that should be passed to the program - :param print_cmd: True, if the name of the program should be included in log statements, False otherwise - :param print_args: True, if the arguments should be included in log statements, False otherwise - :param capture_output: True, if the output of the program should be captured and returned, False otherwise - :param exit_on_error: True, if the build system should be terminated when an error occurs, False otherwise - :param env: The environment variables to be passed to the program + Allows to run command line programs. """ - cmd = __get_qualified_command(cmd) - - if print_cmd: - print('Running external command "' + __format_command(cmd, *args, format_args=print_args) + '"...') - - out = subprocess.run([cmd] + list(args), check=False, text=capture_output, capture_output=capture_output, env=env) - exit_code = out.returncode - - if exit_code != 0: - message = ('External command "' + __format_command(cmd, *args) + '" terminated with non-zero exit code ' - + str(exit_code)) - - if exit_on_error: - if capture_output: - print(str(out.stderr).strip()) - - print(message) - sys.exit(exit_code) - else: - raise RuntimeError(message) - return out + class PrintOptions: + """ + Allows to customize how command line programs are presented in log statements. + """ + + def __init__(self): + self.print_arguments = False + + def format(self, command: 'Command') -> str: + """ + Creates and returns a textual representation of a given command line program. + + :param command: The command line program + :return: The textual representation that has been created + """ + result = command.command + + if self.print_arguments: + result += ' ' + format_iterable(command.arguments, separator=' ') + + return result + + class RunOptions: + """ + Allows to customize options for running command line programs. + """ + + def __init__(self): + self.print_command = True + self.exit_on_error = True + self.environment = None + + def run(self, command: 'Command', capture_output: bool) -> CompletedProcess: + """ + Runs a given command line program. + + :param command: The command line program to be run + :param capture_output: True, if the output of the program should be captured, False otherwise + :return: The output of the program + """ + if self.print_command: + print('Running external command "' + command.print_options.format(command) + '"...') + + output = subprocess.run([command.command] + command.arguments, + check=False, + text=capture_output, + capture_output=capture_output, + env=self.environment) + exit_code = output.returncode + + if exit_code != 0: + message = ('External command "' + str(command) + '" terminated with non-zero exit code ' + + str(exit_code)) + + if self.exit_on_error: + if capture_output: + print(str(output.stderr).strip()) + + print(message) + sys.exit(exit_code) + else: + raise RuntimeError(message) + + return output + + def __init__(self, + command: str, + *arguments: str, + print_options: PrintOptions = PrintOptions(), + run_options: RunOptions = RunOptions()): + """ + :param command: The name of the command line program + :param arguments: Optional arguments to be passed to the command line program + :param run_options: The options that should eb used for running the command line program + :param print_options: The options that should be used for creating textual representations of the command line + program + """ + self.command = command + + if in_virtual_environment(): + # On Windows, we use the relative path to the command's executable within the virtual environment, if such + # an executable exists. This circumvents situations where the PATH environment variable has not been updated + # after activating the virtual environment. This can prevent the executables from being found or can lead to + # the wrong executable, from outside the virtual environment, being executed. + executable = path.join(sys.prefix, 'Scripts', command + '.exe') + + if path.isfile(executable): + self.command = executable + + self.arguments = list(arguments) + self.print_options = print_options + self.run_options = run_options + + def add_arguments(self, *arguments: str) -> 'Command': + """ + Adds one or several arguments to be passed to the command line program. + + :param arguments: The arguments to be added + :return: The `Command` itself + """ + self.arguments.extend(arguments) + return self + + def add_conditional_arguments(self, condition: bool, *arguments: str) -> 'Command': + """ + Adds one or several arguments to be passed to the command line program, if a certain condition is True. + + :param condition: The condition + :param arguments: The arguments to be added + :return: The `Command` itself + """ + if condition: + self.arguments.extend(arguments) + return self + + def print_arguments(self, print_arguments: bool) -> 'Command': + """ + Sets whether the arguments of the command line program should be included in log statements or not. + + :param print_arguments: True, if the arguments should be included, False otherwise + :return: The `Command` itself + """ + self.print_options.print_arguments = print_arguments + return self + + def print_command(self, print_command: bool) -> 'Command': + """ + Sets whether the command line program should be printed on the console when being run or not. + + :param print_command: True, if the command line program should be printed, False otherwise + :return: The `Command` itself + """ + self.run_options.print_command = print_command + return self + + def exit_on_error(self, exit_on_error: bool) -> 'Command': + """ + Sets whether the build system should be terminated if the program exits with a non-zero exit code or not. + + :param exit_on_error: True, if the build system should be terminated, False, if a `RuntimeError` should be + raised instead + :return: The `Command` itself + """ + self.run_options.exit_on_error = exit_on_error + return self + + def use_environment(self, environment) -> 'Command': + """ + Sets the environment to be used for running the command line program. + + :param environment: The environment to be set or None, if the default environment should be used + :return: The `Command` itself + """ + self.run_options.environment = environment + return self + + def _should_be_skipped(self) -> bool: + """ + May be overridden by subclasses in order to determine whether the command should be skipped or not. + """ + return False + + def _before(self): + """ + May be overridden by subclasses in order to perform some operations before the command is run. + """ + + def run(self): + """ + Runs the command line program. + """ + if not self._should_be_skipped(): + self._before() + self.run_options.run(self, capture_output=False) + self._after() + + def capture_output(self) -> str: + """ + Runs the command line program and returns its output. + + :return: The output of the program + """ + if not self._should_be_skipped(): + self._before() + output = self.run_options.run(self, capture_output=True) + self._after() + return output.stdout + + return '' + + def _after(self): + """ + May be overridden by subclasses in order to perform some operations after the command has been run. + """ + + def __str__(self) -> str: + print_options = Command.PrintOptions() + print_options.print_arguments = True + return print_options.format(self) From 6b4dfb3813cc628583daa5b9451b8cd11fc33b70 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 20 Nov 2024 15:55:37 +0100 Subject: [PATCH 014/114] Add class Pip. --- scons/dependencies.py | 159 ++++++++------------------------ scons/util/pip.py | 210 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 249 insertions(+), 120 deletions(-) create mode 100644 scons/util/pip.py diff --git a/scons/dependencies.py b/scons/dependencies.py index 086c163c5..f0a1ccf09 100644 --- a/scons/dependencies.py +++ b/scons/dependencies.py @@ -4,128 +4,61 @@ Provides utility functions for dealing with dependencies. """ -from abc import ABC from dataclasses import dataclass -from functools import reduce from os import path -from typing import List, Optional +from typing import List from modules import ALL_MODULES, BUILD_MODULE, CPP_MODULE, PYTHON_MODULE, Module -from util.cmd import Command +from util.pip import Package, Pip, RequirementsFile @dataclass -class Requirement: +class Dependency: """ - Specifies the supported version(s) of a specific dependency. + Provides information about an installed dependency. Attributes: - dependency: The name of the dependency - version: The supported version(s) of the dependency or None, if there are no restrictions + package: The Python package + installed_version: The version of the dependency that is currently installed + latest_version: The latest version of the dependency """ - dependency: str - version: Optional[str] = None + package: Package + installed_version: str + latest_version: str - def __str__(self): - return self.dependency + (self.version if self.version else '') - - -class PipCommand(Command, ABC): - """ - An abstract base class for all classes that allow to run pip on the command line. - """ - - def __init__(self, *arguments: str): - super().__init__('python', '-m', 'pip', *arguments) - - -class PipInstallCommand(PipCommand): - """ - Allows to install a package via the command `pip install`. - """ - - def __init__(self, requirement: Requirement, *arguments: str): - """ - :requirement: Specifies the name and version of the package that should be installed - :arguments: Optional arguments to be passed to the command - """ - super().__init__('install', str(requirement), '--upgrade', '--upgrade-strategy', 'eager', '--prefer-binary', - '--disable-pip-version-check', *arguments) - - -class PipListCommand(PipCommand): - """ - Allows to list all installed Python packages via the command `pip list`. - """ - - def __init__(self, *arguments: str): - """ - :arguments: Optional arguments to be passed to the command - """ - super().__init__('list', *arguments) - - -def __pip_install(requirement: Requirement, dry_run: bool = False): - try: - args = ['--dry-run'] if dry_run else [] - stdout = PipInstallCommand(requirement, *args).print_command(False).exit_on_error(not dry_run).capture_output() - stdout_lines = stdout.strip().split('\n') - - if reduce( - lambda aggr, line: aggr | line.startswith('Would install') and __normalize_dependency(line).find( - requirement.dependency) >= 0, stdout_lines, False): - if dry_run: - PipInstallCommand(requirement).print_arguments(True).run() - else: - print(stdout) - except RuntimeError: - __pip_install(requirement) - - -def __normalize_dependency(dependency: str): - return dependency.replace('_', '-').lower() +def __find_outdated_dependencies() -> List[Dependency]: + stdout = Pip.Command('list', '--outdated').print_command(False).capture_output() + stdout_lines = stdout.strip().split('\n') + i = 0 -def __find_requirements(requirements_file: str, *dependencies: str, raise_error: bool = True) -> List[Requirement]: - with open(requirements_file, mode='r', encoding='utf-8') as file: - lines = [line.split(' ') for line in file.readlines()] - requirements = [ - Requirement(dependency=__normalize_dependency(parts[0].strip()), - version=' '.join(parts[1:]).strip() if len(parts) > 1 else None) for parts in lines - ] - requirements = {requirement.dependency: requirement for requirement in requirements} + for line in stdout_lines: + i += 1 - if dependencies: - found_requirements = [] + if line.startswith('----'): + break - for dependency in dependencies: - if __normalize_dependency(dependency) in requirements: - found_requirements.append(requirements[dependency]) - elif raise_error: - raise RuntimeError('Dependency "' + dependency + '" not found in requirements file "' - + requirements_file + '"') + outdated_dependencies = [] - return found_requirements + for line in stdout_lines[i:]: + parts = line.split() + outdated_dependencies.append(Dependency(Package(parts[0]), installed_version=parts[1], latest_version=parts[2])) - return list(requirements.values()) + return outdated_dependencies def __install_module_dependencies(module: Module, *dependencies: str): requirements_file = module.requirements_file if path.isfile(requirements_file): - install_dependencies(requirements_file, *dependencies) + Pip(RequirementsFile(requirements_file)).install_packages(*(dependencies) -def install_dependencies(requirements_file: str, *dependencies: str): - """ - Installs one or several dependencies if they are listed in a given requirements.txt file. - - :param requirements_file: The path of the requirements.txt file that specifies the dependency versions - :param dependencies: The names of the dependencies that should be installed - """ - for requirement in __find_requirements(requirements_file, *dependencies): - __pip_install(requirement, dry_run=True) +def __print_table(header: List[str], rows: List[List[str]]): + install_build_dependencies('tabulate') + # pylint: disable=import-outside-toplevel + from tabulate import tabulate + print(tabulate(rows, headers=header)) def install_build_dependencies(*dependencies: str): @@ -154,38 +87,24 @@ def check_dependency_versions(**_): __install_module_dependencies(module) print('Checking for outdated dependencies...') - stdout = PipListCommand('--outdated').print_command(False).capture_output() - stdout_lines = stdout.strip().split('\n') - i = 0 - - for line in stdout_lines: - i += 1 - - if line.startswith('----'): - break - - outdated_dependencies = [] - - for line in stdout_lines[i:]: - dependency = __normalize_dependency(line.split()[0]) + outdated_dependencies = __find_outdated_dependencies() + rows = [] + for dependency in outdated_dependencies: for module in ALL_MODULES: requirements_file = module.requirements_file if path.isfile(requirements_file): - requirements = __find_requirements(requirements_file, dependency, raise_error=False) + requirements = RequirementsFile(requirements_file).lookup(dependency.package, accept_missing=True) - if requirements and requirements[0].version: - outdated_dependencies.append(line) + if requirements and requirements.pop().version: + rows.append([str(dependency.package), dependency.installed_version, dependency.latest_version]) break - if outdated_dependencies: + if rows: + rows.sort(key=lambda row: row[0]) + header = ['Dependency', 'Installed version', 'Latest version'] print('The following dependencies are outdated:\n') - - for header_line in stdout_lines[:i]: - print(header_line) - - for outdated_dependency in outdated_dependencies: - print(outdated_dependency) + __print_table(header=header, rows=rows) else: print('All dependencies are up-to-date!') diff --git a/scons/util/pip.py b/scons/util/pip.py new file mode 100644 index 000000000..8d93ea908 --- /dev/null +++ b/scons/util/pip.py @@ -0,0 +1,210 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for installing Python packages via pip. +""" +from abc import ABC +from dataclasses import dataclass +from functools import cached_property +from typing import Dict, Optional, Set + +from util.cmd import Command as Cmd +from util.io import read_file + + +@dataclass +class Package: + """ + A Python package. + + Attributes: + name: The name of the package + """ + name: str + + @property + def normalized_name(self) -> str: + """ + The normalized name of the package in lower-case and with invalid characters being replaced. + """ + return self.name.replace('_', '-').lower() + + def __str__(self) -> str: + return self.normalized_name + + def __eq__(self, other: 'Package') -> bool: + return self.normalized_name == other.normalized_name + + def __hash__(self) -> int: + return hash(self.normalized_name) + + +@dataclass +class Requirement: + """ + A single requirement included in a requirements file, consisting of a Python package and an optional version. + + Attributes: + package: The package + version: The version of the package or None, if no version is specified + """ + package: Package + version: Optional[str] = None + + @staticmethod + def parse(requirement: str) -> 'Requirement': + """ + Parses and returns a single requirement included a requirements file. + + :param requirement: The requirement to be parsed + :return: The requirement that has been parsed + """ + parts = requirement.split() + package = Package(name=parts[0].strip()) + version = ' '.join(parts[1:]).strip() if len(parts) > 1 else None + return Requirement(package, version) + + def __str__(self) -> str: + return str(self.package) + (self.version if self.version else '') + + def __eq__(self, other: 'Requirement') -> bool: + return self.package == other.package + + def __hash__(self) -> int: + return hash(self.package) + + +@dataclass +class RequirementsFile: + """ + Represents a specific requirements.txt file. + + Attributes: + requirements_file: The path to the requirements file + """ + requirements_file: str + + @cached_property + def requirements_by_package(self) -> Dict[Package, Requirement]: + """ + A dictionary that contains all requirements in the requirements file by their package. + """ + with read_file(self.requirements_file) as file: + requirements = [Requirement.parse(line) for line in file.readlines() if line.strip('\n').strip()] + return {requirement.package: requirement for requirement in requirements} + + @property + def requirements(self) -> Set[Requirement]: + """ + A set that contains all requirements in the requirements file + """ + return set(self.requirements_by_package.values()) + + def lookup(self, *packages: Package, accept_missing: bool = False) -> Set[Requirement]: + """ + Looks up the requirements for given packages in the requirements file. + + :param packages: The packages that should be looked up + :param accept_missing: False, if an error should be raised if a package is not listed in the requirements file, + True, if it should simply be ignored + :return: A set that contains the requirements for the given packages + """ + requirements = set() + + for package in packages: + requirement = self.requirements_by_package.get(package) + + if requirement: + requirements.add(requirement) + elif not accept_missing: + raise RuntimeError('Package "' + str(package) + '" not found in requirements file "' + + self.requirements_file + '"') + + return requirements + + +class Pip: + """ + Allows to install Python packages via pip. + """ + + class Command(Cmd, ABC): + """ + An abstract base class for all classes that allow to run pip on the command line. + """ + + def __init__(self, pip_command: str, *arguments: str): + """ + :param pip_command: The pip command to be run, e.g., "install" + :param arguments: Optional arguments to be passed to pip + """ + super().__init__('python', '-m', 'pip', pip_command, *arguments, '--disable-pip-version-check') + + class InstallCommand(Command): + """ + Allows to install requirements via the command `pip install`. + """ + + def __init__(self, requirement: Requirement, dry_run: bool = False): + """ + :param requirement: The requirement be installed + :param dry_run: True, if the --dry-run flag should be set, False otherwise + """ + super().__init__('install', str(requirement), '--upgrade', '--upgrade-strategy', 'eager', '--prefer-binary') + self.add_conditional_arguments(dry_run, '--dry-run') + + @staticmethod + def __would_install_requirement(requirement: Requirement, stdout: str) -> bool: + prefix = 'Would install' + + for line in stdout.split('\n'): + if line.strip().startswith(prefix): + package = Package(line[len(prefix):].strip()) + + if package.normalized_name.find(requirement.package.normalized_name) >= 0: + return True + + return False + + @staticmethod + def __install_requirement(requirement: Requirement, dry_run: bool = False): + """ + Installs a requirement. + + :param requirement: The requirement to be installed + """ + try: + stdout = Pip.InstallCommand(requirement, dry_run=dry_run) \ + .print_command(False) \ + .exit_on_error(not dry_run) \ + .capture_output() + + if Pip.__would_install_requirement(requirement, stdout): + if dry_run: + Pip.InstallCommand(requirement) \ + .print_arguments(True) \ + .run() + else: + print(stdout) + except RuntimeError: + Pip.__install_requirement(requirement) + + def __init__(self, requirements_file: RequirementsFile): + """ + :param requirements_file: The requirements file that should be used for looking up package versions + """ + self.requirements_file = requirements_file + + def install_packages(self, *package_names: str, accept_missing: bool = False): + """ + Installs one or several dependencies. + + :param package_names: The names of the packages that should be installed + :param accept_missing: False, if an error should be raised if a package is not listed in the requirements file, + True, if it should simply be ignored + """ + packages = [Package(package_name) for package_name in package_names] + requirements = self.requirements_file.lookup(*packages, accept_missing=accept_missing) + + for requirement in requirements: + self.__install_requirement(requirement, dry_run=True) From 92fc0b17b42dda0f431e4670eaeb4d2301097204 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 20 Nov 2024 18:09:06 +0100 Subject: [PATCH 015/114] Add classes Program and PythonModule. --- scons/code_style.py | 163 +++++++++++++++++++++++++++-------------- scons/compilation.py | 117 +++++++++++++++++++++-------- scons/documentation.py | 70 +++++++----------- scons/packaging.py | 28 +++---- scons/run.py | 77 ------------------- scons/testing.py | 42 ++++------- scons/util/run.py | 93 +++++++++++++++++++++++ 7 files changed, 339 insertions(+), 251 deletions(-) delete mode 100644 scons/run.py create mode 100644 scons/util/run.py diff --git a/scons/code_style.py b/scons/code_style.py index 769336f43..eadda4050 100644 --- a/scons/code_style.py +++ b/scons/code_style.py @@ -7,74 +7,127 @@ from os import path from modules import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE -from run import run_program +from util.pip import RequirementsFile +from util.run import Program MD_DIRS = [('.', False), (DOC_MODULE.root_dir, True), (PYTHON_MODULE.root_dir, True)] YAML_DIRS = [('.', False), ('.github', True)] -def __isort(directory: str, enforce_changes: bool = False): - args = ['--settings-path', '.', '--virtual-env', 'venv', '--skip-gitignore'] - - if not enforce_changes: - args.append('--check') +class Yapf(Program): + """ + Allows to run the external program "yapf". + """ - run_program('isort', *args, directory) + def __init__(self, directory: str, enforce_changes: bool = False): + """ + :param directory: The path to the directory, the program should be applied to + :param enforce_changes: True, if changes should be applied to files, False otherwise + """ + super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'yapf', '-r', '-p', '--style=.style.yapf', + '--exclude', '**/build/*.py', '-i' if enforce_changes else '--diff', directory) -def __yapf(directory: str, enforce_changes: bool = False): - run_program('yapf', '-r', '-p', '--style=.style.yapf', '--exclude', '**/build/*.py', - '-i' if enforce_changes else '--diff', directory) +class Isort(Program): + """ + Allows to run the external program "isort". + """ + def __init__(self, directory: str, enforce_changes: bool = False): + """ + :param directory: The path to the directory, the program should be applied to + :param enforce_changes: True, if changes should be applied to files, False otherwise + """ + super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'isort', directory, '--settings-path', '.', + '--virtual-env', 'venv', '--skip-gitignore') + self.add_conditional_arguments(not enforce_changes, '--check') -def __pylint(directory: str): - run_program('pylint', '--jobs=0', '--recursive=y', '--ignore=build', '--rcfile=.pylintrc', '--score=n', directory) +class Pylint(Program): + """ + Allows to run the external program "pylint". + """ -def __clang_format(directory: str, enforce_changes: bool = False): - cpp_header_files = glob(path.join(directory, '**', '*.hpp'), recursive=True) - cpp_source_files = glob(path.join(directory, '**', '*.cpp'), recursive=True) - args = ['--style=file'] + def __init__(self, directory: str): + """ + :param directory: The path to the directory, the program should be applied to + """ + super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'pylint', directory, '--jobs=0', + '--recursive=y', '--ignore=build', '--rcfile=.pylintrc', '--score=n') - if enforce_changes: - args.append('-i') - else: - args.append('-n') - args.append('--Werror') - run_program('clang-format', *args, *cpp_header_files, *cpp_source_files) +class ClangFormat(Program): + """ + Allows to run the external program "clang-format". + """ + def __init__(self, directory: str, enforce_changes: bool = False): + """ + :param directory: The path to the directory, the program should be applied to + :param enforce_changes: True, if changes should be applied to files, False otherwise + """ + super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'clang-format', '--style=file') + self.add_conditional_arguments(enforce_changes, '-i') + self.add_conditional_arguments(not enforce_changes, '--dry-run', '--Werror') + self.add_arguments(*glob(path.join(directory, '**', '*.hpp'), recursive=True)) + self.add_arguments(*glob(path.join(directory, '**', '*.cpp'), recursive=True)) -def __cpplint(directory: str): - run_program('cpplint', '--quiet', '--recursive', directory) +class Cpplint(Program): + """ + Allows to run the external program "cpplint". + """ -def __mdformat(directory: str, recursive: bool = False, enforce_changes: bool = False): - suffix_md = '*.md' - glob_path = path.join(directory, '**', '**', suffix_md) if recursive else path.join(directory, suffix_md) - md_files = glob(glob_path, recursive=recursive) - args = ['--number', '--wrap', 'no', '--end-of-line', 'lf'] + def __init__(self, directory: str): + """ + :param directory: The path to the directory, the program should be applied to + """ + super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'cpplint', directory, '--quiet', + '--recursive') - if not enforce_changes: - args.append('--check') - run_program('mdformat', *args, *md_files, additional_dependencies=['mdformat-myst']) +class Mdformat(Program): + """ + Allows to run the external program "mdformat". + """ + def __init__(self, directory: str, recursive: bool = False, enforce_changes: bool = False): + """ + :param directory: The path to the directory, the program should be applied to + :param recursive: True, if the program should be applied to subdirectories, False otherwise + :param enforce_changes: True, if changes should be applied to files, False otherwise + """ + super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'mdformat', '--number', '--wrap', 'no', + '--end-of-line', 'lf') + self.add_conditional_arguments(not enforce_changes, '--check') + suffix_md = '*.md' + glob_path = path.join(directory, '**', '**', suffix_md) if recursive else path.join(directory, suffix_md) + self.add_arguments(*glob(glob_path, recursive=recursive)) + self.add_dependencies('mdformat-myst') -def __yamlfix(directory: str, recursive: bool = False, enforce_changes: bool = False): - glob_path = path.join(directory, '**', '*') if recursive else path.join(directory, '*') - glob_path_hidden = path.join(directory, '**', '.*') if recursive else path.join(directory, '.*') - yaml_files = [ - file for file in glob(glob_path) + glob(glob_path_hidden) - if path.basename(file).endswith('.yml') or path.basename(file).endswith('.yaml') - ] - args = ['--config-file', '.yamlfix.toml'] - if not enforce_changes: - args.append('--check') +class Yamlfix(Program): + """ + Allows to run the external program "yamlfix". + """ - run_program('yamlfix', *args, *yaml_files, print_args=True) + def __init__(self, directory: str, recursive: bool = False, enforce_changes: bool = False): + """ + :param directory: The path to the directory, the program should be applied to + :param recursive: True, if the program should be applied to subdirectories, False otherwise + :param enforce_changes: True, if changes should be applied to files, False otherwise + """ + super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'yamlfix', '--config-file', '.yamlfix.toml') + self.add_conditional_arguments(not enforce_changes, '--check') + glob_path = path.join(directory, '**', '*') if recursive else path.join(directory, '*') + glob_path_hidden = path.join(directory, '**', '.*') if recursive else path.join(directory, '.*') + yaml_files = [ + file for file in glob(glob_path) + glob(glob_path_hidden) + if path.basename(file).endswith('.yml') or path.basename(file).endswith('.yaml') + ] + self.add_arguments(yaml_files) + self.print_arguments(True) def check_python_code_style(**_): @@ -84,9 +137,9 @@ def check_python_code_style(**_): for module in [BUILD_MODULE, PYTHON_MODULE]: directory = module.root_dir print('Checking Python code style in directory "' + directory + '"...') - __isort(directory) - __yapf(directory) - __pylint(directory) + Isort(directory).run() + Yapf(directory).run() + Pylint(directory).run() def enforce_python_code_style(**_): @@ -96,8 +149,8 @@ def enforce_python_code_style(**_): for module in [BUILD_MODULE, PYTHON_MODULE, DOC_MODULE]: directory = module.root_dir print('Formatting Python code in directory "' + directory + '"...') - __isort(directory, enforce_changes=True) - __yapf(directory, enforce_changes=True) + Isort(directory, enforce_changes=True).run() + Yapf(directory, enforce_changes=True).run() def check_cpp_code_style(**_): @@ -106,11 +159,11 @@ def check_cpp_code_style(**_): """ root_dir = CPP_MODULE.root_dir print('Checking C++ code style in directory "' + root_dir + '"...') - __clang_format(root_dir) + ClangFormat(root_dir).run() for subproject in CPP_MODULE.find_subprojects(): for directory in [subproject.include_dir, subproject.src_dir]: - __cpplint(directory) + Cpplint(directory).run() def enforce_cpp_code_style(**_): @@ -119,7 +172,7 @@ def enforce_cpp_code_style(**_): """ root_dir = CPP_MODULE.root_dir print('Formatting C++ code in directory "' + root_dir + '"...') - __clang_format(root_dir, enforce_changes=True) + ClangFormat(root_dir, enforce_changes=True).run() def check_md_code_style(**_): @@ -128,7 +181,7 @@ def check_md_code_style(**_): """ for directory, recursive in MD_DIRS: print('Checking Markdown code style in the directory "' + directory + '"...') - __mdformat(directory, recursive=recursive) + Mdformat(directory, recursive=recursive).run() def enforce_md_code_style(**_): @@ -137,7 +190,7 @@ def enforce_md_code_style(**_): """ for directory, recursive in MD_DIRS: print('Formatting Markdown files in the directory "' + directory + '"...') - __mdformat(directory, recursive=recursive, enforce_changes=True) + Mdformat(directory, recursive=recursive, enforce_changes=True).run() def check_yaml_code_style(**_): @@ -146,7 +199,7 @@ def check_yaml_code_style(**_): """ for directory, recursive in YAML_DIRS: print('Checking YAML files in the directory "' + directory + '"...') - __yamlfix(directory, recursive=recursive) + Yamlfix(directory, recursive=recursive).run() def enforce_yaml_code_style(**_): @@ -155,4 +208,4 @@ def enforce_yaml_code_style(**_): """ for directory, recursive in YAML_DIRS: print('Formatting YAML files in the directory "' + directory + '"...') - __yamlfix(directory, recursive=recursive, enforce_changes=True) + Yamlfix(directory, recursive=recursive, enforce_changes=True).run() diff --git a/scons/compilation.py b/scons/compilation.py index e6add19c0..858f79791 100644 --- a/scons/compilation.py +++ b/scons/compilation.py @@ -6,9 +6,10 @@ from os import environ from typing import List, Optional -from modules import CPP_MODULE, PYTHON_MODULE -from run import run_program +from modules import BUILD_MODULE, CPP_MODULE, PYTHON_MODULE from util.env import get_env +from util.pip import RequirementsFile +from util.run import Program class BuildOptions: @@ -64,22 +65,29 @@ def add(self, name: str, subpackage: Optional[str] = None) -> 'BuildOptions': self.build_options.append(BuildOptions.BuildOption(name=name, subpackage=subpackage)) return self - def to_args(self) -> List[str]: + def as_arguments(self) -> List[str]: """ Returns a list of arguments to be passed to the command "meson configure" for setting the build options. :return: A list of arguments """ - args = [] + arguments = [] for build_option in self.build_options: value = build_option.value if value: - args.append('-D') - args.append(build_option.key + '=' + value) + arguments.append('-D') + arguments.append(build_option.key + '=' + value) - return args + return arguments + + def __bool__(self) -> bool: + for build_option in self.build_options: + if build_option.value: + return True + + return False CPP_BUILD_OPTIONS = BuildOptions() \ @@ -93,45 +101,88 @@ def to_args(self) -> List[str]: .add(name='subprojects') -def __meson_setup(root_dir: str, - build_dir: str, - build_options: BuildOptions = BuildOptions(), - dependencies: Optional[List[str]] = None): - print('Setting up build directory "' + build_dir + '"...') - args = build_options.to_args() - run_program('meson', 'setup', *args, build_dir, root_dir, print_args=True, additional_dependencies=dependencies) +class MesonSetup(Program): + """ + Allows to run the external program "meson setup". + """ + def __init__(self, build_directory: str, source_directory: str, build_options: BuildOptions = BuildOptions()): + """ + :param build_directory: The path to the build directory + :param source_directory: The path to the source directory + :param build_options: The build options to be used + """ + super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'meson', 'setup', + *build_options.as_arguments(), build_directory, source_directory) + self.print_arguments(True) -def __meson_configure(build_dir: str, build_options: BuildOptions): - args = build_options.to_args() - if args: - print('Configuring build options according to environment variables...') - run_program('meson', 'configure', *args, build_dir, print_args=True) +class MesonConfigure(Program): + """ + Allows to run the external program "meson configure". + """ + def __init__(self, build_directory: str, build_options: BuildOptions = BuildOptions()): + """ + :param build_directory: The path to the build directory + :param build_options: The build options to be used + """ + super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'meson', 'configure', + *build_options.as_arguments(), build_directory) + self.print_arguments(True) + self.build_options = build_options -def __meson_compile(build_dir: str): - run_program('meson', 'compile', '-C', build_dir, print_args=True) + def run(self): + if self.build_options: + print('Configuring build options according to environment variables...') + super().run() -def __meson_install(build_dir: str): - run_program('meson', 'install', '--no-rebuild', '--only-changed', '-C', build_dir, print_args=True) +class MesonCompile(Program): + """ + Allows to run the external program "meson compile". + """ + + def __init__(self, build_directory: str): + """ + :param build_directory: The path to the build directory + """ + super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'meson', 'compile', '-C', build_directory) + self.print_arguments(True) + + +class MesonInstall(Program): + """ + Allows to run the external program "meson install". + """ + + def __init__(self, build_directory: str): + """ + :param build_directory: The path to the build directory + """ + super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'meson', 'install', '--no-rebuild', + '--only-changed', '-C', build_directory) + self.print_arguments(True) def setup_cpp(**_): """ Sets up the build system for compiling the C++ code. """ - __meson_setup(CPP_MODULE.root_dir, CPP_MODULE.build_dir, CPP_BUILD_OPTIONS, dependencies=['ninja']) + MesonSetup(build_directory=CPP_MODULE.build_dir, + source_directory=CPP_MODULE.root_dir, + build_options=CPP_BUILD_OPTIONS) \ + .add_dependencies('ninja') \ + .run() def compile_cpp(**_): """ Compiles the C++ code. """ - __meson_configure(CPP_MODULE.build_dir, CPP_BUILD_OPTIONS) + MesonConfigure(CPP_MODULE.build_dir, CPP_BUILD_OPTIONS).run() print('Compiling C++ code...') - __meson_compile(CPP_MODULE.build_dir) + MesonCompile(CPP_MODULE.build_dir).run() def install_cpp(**_): @@ -139,23 +190,27 @@ def install_cpp(**_): Installs shared libraries into the source tree. """ print('Installing shared libraries into source tree...') - __meson_install(CPP_MODULE.build_dir) + MesonInstall(CPP_MODULE.build_dir).run() def setup_cython(**_): """ Sets up the build system for compiling the Cython code. """ - __meson_setup(PYTHON_MODULE.root_dir, PYTHON_MODULE.build_dir, CYTHON_BUILD_OPTIONS, dependencies=['cython']) + MesonSetup(build_directory=PYTHON_MODULE.build_dir, + source_directory=PYTHON_MODULE.root_dir, + build_options=CYTHON_BUILD_OPTIONS) \ + .add_dependencies('cython') \ + .run() def compile_cython(**_): """ Compiles the Cython code. """ - __meson_configure(PYTHON_MODULE.build_dir, CYTHON_BUILD_OPTIONS) + MesonConfigure(PYTHON_MODULE.build_dir, CYTHON_BUILD_OPTIONS) print('Compiling Cython code...') - __meson_compile(PYTHON_MODULE.build_dir) + MesonCompile(PYTHON_MODULE.build_dir).run() def install_cython(**_): @@ -163,4 +218,4 @@ def install_cython(**_): Installs extension modules into the source tree. """ print('Installing extension modules into source tree...') - __meson_install(PYTHON_MODULE.build_dir) + MesonInstall(PYTHON_MODULE.build_dir).run() diff --git a/scons/documentation.py b/scons/documentation.py index 0f0db244f..8786049e8 100644 --- a/scons/documentation.py +++ b/scons/documentation.py @@ -6,10 +6,11 @@ from os import environ, makedirs, path, remove from typing import List -from modules import CPP_MODULE, DOC_MODULE, PYTHON_MODULE -from run import run_program +from modules import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE from util.env import set_env from util.io import read_file, write_file +from util.pip import RequirementsFile +from util.run import Program def __doxygen(project_name: str, input_dir: str, output_dir: str): @@ -19,38 +20,29 @@ def __doxygen(project_name: str, input_dir: str, output_dir: str): set_env(env, 'DOXYGEN_INPUT_DIR', input_dir) set_env(env, 'DOXYGEN_OUTPUT_DIR', output_dir) set_env(env, 'DOXYGEN_PREDEFINED', 'MLRL' + project_name.upper() + '_API=') - run_program('doxygen', DOC_MODULE.doxygen_config_file, print_args=True, install_program=False, env=env) + Program(RequirementsFile(BUILD_MODULE.requirements_file), 'doxygen', DOC_MODULE.doxygen_config_file) \ + .print_arguments(False) \ + .install_program(False) \ + .use_environment(env) \ + .run() def __breathe_apidoc(source_dir: str, output_dir: str, project: str): - run_program('breathe-apidoc', - '--members', - '--project', - project, - '-g', - 'file', - '-o', - output_dir, - source_dir, - print_args=True, - additional_dependencies=['breathe'], - requirements_file=DOC_MODULE.requirements_file, - install_program=False) + Program(RequirementsFile(DOC_MODULE.requirements_file), 'breathe-apidoc', '--members', '--project', project, '-g', + 'file', '-o', output_dir, source_dir) \ + .print_arguments(True) \ + .add_dependencies('breathe') \ + .install_program(False) \ + .run() def __sphinx_apidoc(source_dir: str, output_dir: str): - run_program('sphinx-apidoc', - '--separate', - '--module-first', - '--no-toc', - '-o', - output_dir, - source_dir, - '*.so*', - print_args=True, - additional_dependencies=['sphinx'], - requirements_file=DOC_MODULE.requirements_file, - install_program=False) + Program(RequirementsFile(DOC_MODULE.requirements_file), 'sphinx-apidoc', '--separate', '--module-first', '--no-toc', + '-o', output_dir, source_dir, '*.so*') \ + .print_arguments(True) \ + .add_dependencies('sphinx') \ + .install_program(False) \ + .run() root_rst_file = path.join(output_dir, 'mlrl.rst') @@ -59,22 +51,12 @@ def __sphinx_apidoc(source_dir: str, output_dir: str): def __sphinx_build(source_dir: str, output_dir: str): - run_program('sphinx-build', - '--jobs', - 'auto', - source_dir, - output_dir, - print_args=True, - additional_dependencies=[ - 'furo', - 'myst-parser', - 'sphinxext-opengraph', - 'sphinx-inline-tabs', - 'sphinx-copybutton', - 'sphinx-favicon', - ], - requirements_file=DOC_MODULE.requirements_file, - install_program=False) + Program(RequirementsFile(DOC_MODULE.requirements_file), 'sphinx-build', '--jobs', 'auto', source_dir, output_dir) \ + .print_arguments(True) \ + .add_dependencies('furo', 'myst-parser', 'sphinxext-opengraph', 'sphinx-inline-tabs', 'sphinx-copybutton', + 'sphinx-favicon',) \ + .install_program(False) \ + .run() def __read_tocfile_template(directory: str) -> List[str]: diff --git a/scons/packaging.py b/scons/packaging.py index 926dc9167..2d51c650d 100644 --- a/scons/packaging.py +++ b/scons/packaging.py @@ -5,28 +5,24 @@ """ from typing import List -from modules import PYTHON_MODULE -from run import run_python_program +from modules import BUILD_MODULE, PYTHON_MODULE +from util.pip import RequirementsFile +from util.run import PythonModule def __build_python_wheel(package_dir: str): - run_python_program('build', - '--no-isolation', - '--wheel', - package_dir, - print_args=True, - additional_dependencies=['wheel', 'setuptools']) + PythonModule(RequirementsFile(BUILD_MODULE.requirements_file), 'build', '--no-isolation', '--wheel', package_dir) \ + .print_arguments(True) \ + .add_dependencies('wheel', 'setuptools') \ + .run() def __install_python_wheels(wheels: List[str]): - run_python_program('pip', - 'install', - '--force-reinstall', - '--no-deps', - '--disable-pip-version-check', - *wheels, - print_args=True, - install_program=False) + PythonModule(RequirementsFile(BUILD_MODULE.requirements_file), 'pip', 'install', '--force-reinstall', '--no-deps', + '--disable-pip-version-check', *wheels) \ + .print_arguments(True) \ + .install_program(False) \ + .run() # pylint: disable=unused-argument diff --git a/scons/run.py b/scons/run.py deleted file mode 100644 index ad05a09b7..000000000 --- a/scons/run.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Provides utility functions for running external programs during the build process. -""" -from typing import List, Optional - -from dependencies import install_dependencies -from modules import BUILD_MODULE -from util.cmd import Command - - -def run_program(program: str, - *args, - print_args: bool = False, - additional_dependencies: Optional[List[str]] = None, - requirements_file: str = BUILD_MODULE.requirements_file, - install_program: bool = True, - env=None): - """ - Runs an external program that has been installed into the virtual environment. - - :param program: The name of the program to be run - :param args: Optional arguments that should be passed to the program - :param print_args: True, if the arguments should be included in log statements, False otherwise - :param additional_dependencies: The names of dependencies that should be installed before running the program - :param requirements_file: The path of the requirements.txt file that specifies the dependency versions - :param install_program: True, if the program should be installed before being run, False otherwise - :param env: The environment variables to be passed to the program - """ - dependencies = [] - - if install_program: - dependencies.append(program) - - if additional_dependencies: - dependencies.extend(additional_dependencies) - - install_dependencies(requirements_file, *dependencies) - Command(program, *args).print_arguments(print_args).use_environment(env).run() - - -def run_python_program(program: str, - *args, - print_args: bool = False, - additional_dependencies: Optional[List[str]] = None, - requirements_file: str = BUILD_MODULE.requirements_file, - install_program: bool = True, - env=None): - """ - Runs an external Python program. - - :param program: The name of the program to be run - :param args: Optional arguments that should be passed to the program - :param print_args: True, if the arguments should be included in log statements, False otherwise - :param additional_dependencies: The names of dependencies that should be installed before running the program - :param requirements_file: The path of the requirements.txt file that specifies the dependency versions - :param install_program: True, if the program should be installed before being run, False otherwise - :param env: The environment variable to be passed to the program - """ - dependencies = [] - - if install_program: - dependencies.append(program) - - if additional_dependencies: - dependencies.extend(additional_dependencies) - - run_program('python', - '-m', - program, - *args, - print_args=print_args, - additional_dependencies=dependencies, - requirements_file=requirements_file, - install_program=False, - env=env) diff --git a/scons/testing.py b/scons/testing.py index 0fdcb1fcf..9289a7b4f 100644 --- a/scons/testing.py +++ b/scons/testing.py @@ -5,46 +5,26 @@ """ from os import environ, path -from modules import CPP_MODULE, PYTHON_MODULE -from run import run_program, run_python_program +from modules import BUILD_MODULE, CPP_MODULE, PYTHON_MODULE from util.env import get_env_bool - - -def __meson_test(build_dir: str): - run_program('meson', 'test', '-C', build_dir, '-v', print_args=True) - - -def __python_unittest(directory: str, fail_fast: bool = False): - args = [ - 'discover', - '--verbose', - '--start-directory', - directory, - '--output', - path.join(PYTHON_MODULE.build_dir, 'test-results'), - ] - - if fail_fast: - args.append('--failfast') - - run_python_program('xmlrunner', - *args, - print_args=True, - install_program=False, - additional_dependencies=['unittest-xml-reporting']) +from util.pip import RequirementsFile +from util.run import Program, PythonModule def tests_cpp(**_): """ Runs all automated tests of C++ code. """ - __meson_test(CPP_MODULE.build_dir) + Program(RequirementsFile(BUILD_MODULE.requirements_file), 'meson', 'test', '-C', CPP_MODULE.build_dir, '-v') \ + .print_arguments(True) \ + .run() def tests_python(**_): """ Runs all automated tests of Python code. """ + output_directory = path.join(PYTHON_MODULE.build_dir, 'test-results') fail_fast = get_env_bool(environ, 'FAIL_FAST') for subproject in PYTHON_MODULE.find_subprojects(): @@ -52,4 +32,10 @@ def tests_python(**_): if path.isdir(test_dir): print('Running automated tests for subpackage "' + subproject.name + '"...') - __python_unittest(test_dir, fail_fast=fail_fast) + PythonModule(RequirementsFile(BUILD_MODULE.requirements_file), 'xmlrunner', 'discover', '--verbose', + '--start-directory', test_dir, '--output', output_directory) \ + .add_conditional_arguments(fail_fast, '--failfast') \ + .print_arguments(True) \ + .install_program(False) \ + .add_dependencies('unittest-xml-reporting') \ + .run() diff --git a/scons/util/run.py b/scons/util/run.py new file mode 100644 index 000000000..fb11226a0 --- /dev/null +++ b/scons/util/run.py @@ -0,0 +1,93 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for running external programs during the build process. +""" +from subprocess import CompletedProcess + +from util.cmd import Command +from util.pip import Pip + + +class Program(Command): + """ + Allows to run an external program. + """ + + class RunOptions(Command.RunOptions): + """ + Allows to customize options for running an external program. + """ + + def __init__(self, requirements_file: RequirementsFile): + """ + :param requirements_file: The requirements file that should be used for looking up dependency versions + """ + super().__init__() + self.requirements_file = requirements_file + self.install_program = True + self.dependencies = set() + + def run(self, command: Command, capture_output: bool) -> CompletedProcess: + dependencies = [] + + if self.install_program: + dependencies.append(command.command) + + dependencies.extend(self.dependencies) + Pip(self.requirements_file).install_packages(*dependencies) + return super().run(command, capture_output) + + def __init__(self, requirements_file: RequirementsFile, program: str, *arguments: str): + """ + :param requirements_file: The requirements file that should be used for looking up dependency versions + :param program: The name of the program to be run + :param arguments: Optional arguments to be passed to the program + """ + super().__init__(program, *arguments, run_options=Program.RunOptions(requirements_file)) + + def install_program(self, install_program: bool) -> 'Program': + """ + Sets whether the program should be installed via pip before being run or not. + + :param install_program: True, if the program should be installed before being run, False otherwise + :return: The `Program` itself + """ + self.run_options.install_program = install_program + return self + + def add_dependencies(self, *dependencies: str) -> 'Program': + """ + Adds one or several Python packages that should be installed before running the program. + + :param dependencies: The names of the Python packages to be added + :return: The `Program` itself + """ + self.run_options.dependencies.update(dependencies) + return self + + +class PythonModule(Program): + """ + Allows to run a Python module. + """ + + def __init__(self, requirements_file: RequirementsFile, module: str, *arguments: str): + """ + :param requirements_file: The requirements file that should be used for looking up dependency versions + :param module: The name of the module to be run + :param arguments: Optional arguments to be passed to the module + """ + super().__init__(requirements_file, 'python', '-m', module, *arguments) + self.module = module + self.install_program(True) + + def install_program(self, install_program: bool) -> Program: + super().install_program(False) + + if install_program: + super().add_dependencies(self.module) + else: + self.run_options.dependencies.remove(self.module) + + return self From aba685c11d3f24ea79e0c13a2ebdcdb86f2dfdb4 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 21 Nov 2024 01:03:35 +0100 Subject: [PATCH 016/114] Add class BuildUnit. --- scons/code_style.py | 22 ++++++++---------- scons/compilation.py | 14 +++++------- scons/dependencies.py | 2 +- scons/documentation.py | 16 ++++++------- scons/modules.py | 13 +++++------ scons/packaging.py | 8 +++---- scons/testing.py | 9 ++++---- scons/util/pip.py | 7 +++--- scons/{ => util}/requirements.txt | 0 scons/util/run.py | 37 +++++++++++++++++++------------ scons/util/units.py | 33 +++++++++++++++++++++++++++ 11 files changed, 95 insertions(+), 66 deletions(-) rename scons/{ => util}/requirements.txt (100%) create mode 100644 scons/util/units.py diff --git a/scons/code_style.py b/scons/code_style.py index eadda4050..39c18a7cc 100644 --- a/scons/code_style.py +++ b/scons/code_style.py @@ -7,7 +7,6 @@ from os import path from modules import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE -from util.pip import RequirementsFile from util.run import Program MD_DIRS = [('.', False), (DOC_MODULE.root_dir, True), (PYTHON_MODULE.root_dir, True)] @@ -25,8 +24,8 @@ def __init__(self, directory: str, enforce_changes: bool = False): :param directory: The path to the directory, the program should be applied to :param enforce_changes: True, if changes should be applied to files, False otherwise """ - super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'yapf', '-r', '-p', '--style=.style.yapf', - '--exclude', '**/build/*.py', '-i' if enforce_changes else '--diff', directory) + super().__init__('yapf', '-r', '-p', '--style=.style.yapf', '--exclude', '**/build/*.py', + '-i' if enforce_changes else '--diff', directory) class Isort(Program): @@ -39,8 +38,7 @@ def __init__(self, directory: str, enforce_changes: bool = False): :param directory: The path to the directory, the program should be applied to :param enforce_changes: True, if changes should be applied to files, False otherwise """ - super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'isort', directory, '--settings-path', '.', - '--virtual-env', 'venv', '--skip-gitignore') + super().__init__('isort', directory, '--settings-path', '.', '--virtual-env', 'venv', '--skip-gitignore') self.add_conditional_arguments(not enforce_changes, '--check') @@ -53,8 +51,8 @@ def __init__(self, directory: str): """ :param directory: The path to the directory, the program should be applied to """ - super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'pylint', directory, '--jobs=0', - '--recursive=y', '--ignore=build', '--rcfile=.pylintrc', '--score=n') + super().__init__('pylint', directory, '--jobs=0', '--recursive=y', '--ignore=build', '--rcfile=.pylintrc', + '--score=n') class ClangFormat(Program): @@ -67,7 +65,7 @@ def __init__(self, directory: str, enforce_changes: bool = False): :param directory: The path to the directory, the program should be applied to :param enforce_changes: True, if changes should be applied to files, False otherwise """ - super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'clang-format', '--style=file') + super().__init__('clang-format', '--style=file') self.add_conditional_arguments(enforce_changes, '-i') self.add_conditional_arguments(not enforce_changes, '--dry-run', '--Werror') self.add_arguments(*glob(path.join(directory, '**', '*.hpp'), recursive=True)) @@ -83,8 +81,7 @@ def __init__(self, directory: str): """ :param directory: The path to the directory, the program should be applied to """ - super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'cpplint', directory, '--quiet', - '--recursive') + super().__init__('cpplint', directory, '--quiet', '--recursive') class Mdformat(Program): @@ -98,8 +95,7 @@ def __init__(self, directory: str, recursive: bool = False, enforce_changes: boo :param recursive: True, if the program should be applied to subdirectories, False otherwise :param enforce_changes: True, if changes should be applied to files, False otherwise """ - super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'mdformat', '--number', '--wrap', 'no', - '--end-of-line', 'lf') + super().__init__('mdformat', '--number', '--wrap', 'no', '--end-of-line', 'lf') self.add_conditional_arguments(not enforce_changes, '--check') suffix_md = '*.md' glob_path = path.join(directory, '**', '**', suffix_md) if recursive else path.join(directory, suffix_md) @@ -118,7 +114,7 @@ def __init__(self, directory: str, recursive: bool = False, enforce_changes: boo :param recursive: True, if the program should be applied to subdirectories, False otherwise :param enforce_changes: True, if changes should be applied to files, False otherwise """ - super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'yamlfix', '--config-file', '.yamlfix.toml') + super().__init__('yamlfix', '--config-file', '.yamlfix.toml') self.add_conditional_arguments(not enforce_changes, '--check') glob_path = path.join(directory, '**', '*') if recursive else path.join(directory, '*') glob_path_hidden = path.join(directory, '**', '.*') if recursive else path.join(directory, '.*') diff --git a/scons/compilation.py b/scons/compilation.py index 858f79791..9d7517b43 100644 --- a/scons/compilation.py +++ b/scons/compilation.py @@ -6,9 +6,8 @@ from os import environ from typing import List, Optional -from modules import BUILD_MODULE, CPP_MODULE, PYTHON_MODULE +from modules import CPP_MODULE, PYTHON_MODULE from util.env import get_env -from util.pip import RequirementsFile from util.run import Program @@ -112,8 +111,7 @@ def __init__(self, build_directory: str, source_directory: str, build_options: B :param source_directory: The path to the source directory :param build_options: The build options to be used """ - super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'meson', 'setup', - *build_options.as_arguments(), build_directory, source_directory) + super().__init__('meson', 'setup', *build_options.as_arguments(), build_directory, source_directory) self.print_arguments(True) @@ -127,8 +125,7 @@ def __init__(self, build_directory: str, build_options: BuildOptions = BuildOpti :param build_directory: The path to the build directory :param build_options: The build options to be used """ - super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'meson', 'configure', - *build_options.as_arguments(), build_directory) + super().__init__('meson', 'configure', *build_options.as_arguments(), build_directory) self.print_arguments(True) self.build_options = build_options @@ -147,7 +144,7 @@ def __init__(self, build_directory: str): """ :param build_directory: The path to the build directory """ - super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'meson', 'compile', '-C', build_directory) + super().__init__('meson', 'compile', '-C', build_directory) self.print_arguments(True) @@ -160,8 +157,7 @@ def __init__(self, build_directory: str): """ :param build_directory: The path to the build directory """ - super().__init__(RequirementsFile(BUILD_MODULE.requirements_file), 'meson', 'install', '--no-rebuild', - '--only-changed', '-C', build_directory) + super().__init__('meson', 'install', '--no-rebuild', '--only-changed', '-C', build_directory) self.print_arguments(True) diff --git a/scons/dependencies.py b/scons/dependencies.py index f0a1ccf09..e49528cd9 100644 --- a/scons/dependencies.py +++ b/scons/dependencies.py @@ -51,7 +51,7 @@ def __install_module_dependencies(module: Module, *dependencies: str): requirements_file = module.requirements_file if path.isfile(requirements_file): - Pip(RequirementsFile(requirements_file)).install_packages(*(dependencies) + Pip(module).install_packages(*dependencies) def __print_table(header: List[str], rows: List[List[str]]): diff --git a/scons/documentation.py b/scons/documentation.py index 8786049e8..5a56873af 100644 --- a/scons/documentation.py +++ b/scons/documentation.py @@ -6,10 +6,9 @@ from os import environ, makedirs, path, remove from typing import List -from modules import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE +from modules import CPP_MODULE, DOC_MODULE, PYTHON_MODULE from util.env import set_env from util.io import read_file, write_file -from util.pip import RequirementsFile from util.run import Program @@ -20,7 +19,7 @@ def __doxygen(project_name: str, input_dir: str, output_dir: str): set_env(env, 'DOXYGEN_INPUT_DIR', input_dir) set_env(env, 'DOXYGEN_OUTPUT_DIR', output_dir) set_env(env, 'DOXYGEN_PREDEFINED', 'MLRL' + project_name.upper() + '_API=') - Program(RequirementsFile(BUILD_MODULE.requirements_file), 'doxygen', DOC_MODULE.doxygen_config_file) \ + Program('doxygen', DOC_MODULE.doxygen_config_file) \ .print_arguments(False) \ .install_program(False) \ .use_environment(env) \ @@ -28,8 +27,8 @@ def __doxygen(project_name: str, input_dir: str, output_dir: str): def __breathe_apidoc(source_dir: str, output_dir: str, project: str): - Program(RequirementsFile(DOC_MODULE.requirements_file), 'breathe-apidoc', '--members', '--project', project, '-g', - 'file', '-o', output_dir, source_dir) \ + Program('breathe-apidoc', '--members', '--project', project, '-g', 'file', '-o', output_dir, source_dir) \ + .set_build_unit(DOC_MODULE) \ .print_arguments(True) \ .add_dependencies('breathe') \ .install_program(False) \ @@ -37,8 +36,8 @@ def __breathe_apidoc(source_dir: str, output_dir: str, project: str): def __sphinx_apidoc(source_dir: str, output_dir: str): - Program(RequirementsFile(DOC_MODULE.requirements_file), 'sphinx-apidoc', '--separate', '--module-first', '--no-toc', - '-o', output_dir, source_dir, '*.so*') \ + Program('sphinx-apidoc', '--separate', '--module-first', '--no-toc', '-o', output_dir, source_dir, '*.so*') \ + .set_build_unit(DOC_MODULE) \ .print_arguments(True) \ .add_dependencies('sphinx') \ .install_program(False) \ @@ -51,7 +50,8 @@ def __sphinx_apidoc(source_dir: str, output_dir: str): def __sphinx_build(source_dir: str, output_dir: str): - Program(RequirementsFile(DOC_MODULE.requirements_file), 'sphinx-build', '--jobs', 'auto', source_dir, output_dir) \ + Program('sphinx-build', '--jobs', 'auto', source_dir, output_dir) \ + .set_build_unit(DOC_MODULE) \ .print_arguments(True) \ .add_dependencies('furo', 'myst-parser', 'sphinxext-opengraph', 'sphinx-inline-tabs', 'sphinx-copybutton', 'sphinx-favicon',) \ diff --git a/scons/modules.py b/scons/modules.py index f8f4fcd2a..d36ad7aa1 100644 --- a/scons/modules.py +++ b/scons/modules.py @@ -9,6 +9,7 @@ from typing import Callable, List, Optional from util.env import get_env_array +from util.units import BuildUnit def find_files_recursively(directory: str, @@ -36,11 +37,14 @@ def find_files_recursively(directory: str, return result -class Module(ABC): +class Module(BuildUnit, ABC): """ An abstract base class for all classes that provide access to directories and files that belong to a module. """ + def __init__(self): + super().__init__(path.join(self.root_dir, 'requirements.txt')) + @property @abstractmethod def root_dir(self) -> str: @@ -55,13 +59,6 @@ def build_dir(self) -> str: """ return path.join(self.root_dir, 'build') - @property - def requirements_file(self) -> str: - """ - The path to the requirements.txt file that specifies dependencies required by a module. - """ - return path.join(self.root_dir, 'requirements.txt') - class SourceModule(Module, ABC): """ diff --git a/scons/packaging.py b/scons/packaging.py index 2d51c650d..565e5ce67 100644 --- a/scons/packaging.py +++ b/scons/packaging.py @@ -5,21 +5,19 @@ """ from typing import List -from modules import BUILD_MODULE, PYTHON_MODULE -from util.pip import RequirementsFile +from modules import PYTHON_MODULE from util.run import PythonModule def __build_python_wheel(package_dir: str): - PythonModule(RequirementsFile(BUILD_MODULE.requirements_file), 'build', '--no-isolation', '--wheel', package_dir) \ + PythonModule('build', '--no-isolation', '--wheel', package_dir) \ .print_arguments(True) \ .add_dependencies('wheel', 'setuptools') \ .run() def __install_python_wheels(wheels: List[str]): - PythonModule(RequirementsFile(BUILD_MODULE.requirements_file), 'pip', 'install', '--force-reinstall', '--no-deps', - '--disable-pip-version-check', *wheels) \ + PythonModule('pip', 'install', '--force-reinstall', '--no-deps', '--disable-pip-version-check', *wheels) \ .print_arguments(True) \ .install_program(False) \ .run() diff --git a/scons/testing.py b/scons/testing.py index 9289a7b4f..21b62830f 100644 --- a/scons/testing.py +++ b/scons/testing.py @@ -5,9 +5,8 @@ """ from os import environ, path -from modules import BUILD_MODULE, CPP_MODULE, PYTHON_MODULE +from modules import CPP_MODULE, PYTHON_MODULE from util.env import get_env_bool -from util.pip import RequirementsFile from util.run import Program, PythonModule @@ -15,7 +14,7 @@ def tests_cpp(**_): """ Runs all automated tests of C++ code. """ - Program(RequirementsFile(BUILD_MODULE.requirements_file), 'meson', 'test', '-C', CPP_MODULE.build_dir, '-v') \ + Program('meson', 'test', '-C', CPP_MODULE.build_dir, '-v') \ .print_arguments(True) \ .run() @@ -32,8 +31,8 @@ def tests_python(**_): if path.isdir(test_dir): print('Running automated tests for subpackage "' + subproject.name + '"...') - PythonModule(RequirementsFile(BUILD_MODULE.requirements_file), 'xmlrunner', 'discover', '--verbose', - '--start-directory', test_dir, '--output', output_directory) \ + PythonModule('xmlrunner', 'discover', '--verbose', '--start-directory', test_dir, '--output', + output_directory) \ .add_conditional_arguments(fail_fast, '--failfast') \ .print_arguments(True) \ .install_program(False) \ diff --git a/scons/util/pip.py b/scons/util/pip.py index 8d93ea908..8737d9c95 100644 --- a/scons/util/pip.py +++ b/scons/util/pip.py @@ -10,6 +10,7 @@ from util.cmd import Command as Cmd from util.io import read_file +from util.units import BuildUnit @dataclass @@ -189,11 +190,11 @@ def __install_requirement(requirement: Requirement, dry_run: bool = False): except RuntimeError: Pip.__install_requirement(requirement) - def __init__(self, requirements_file: RequirementsFile): + def __init__(self, build_unit: BuildUnit = BuildUnit()): """ - :param requirements_file: The requirements file that should be used for looking up package versions + :param build_unit: The build unit for which packages should be installed """ - self.requirements_file = requirements_file + self.requirements_file = RequirementsFile(build_unit.requirements_file) def install_packages(self, *package_names: str, accept_missing: bool = False): """ diff --git a/scons/requirements.txt b/scons/util/requirements.txt similarity index 100% rename from scons/requirements.txt rename to scons/util/requirements.txt diff --git a/scons/util/run.py b/scons/util/run.py index fb11226a0..d1be1c827 100644 --- a/scons/util/run.py +++ b/scons/util/run.py @@ -7,6 +7,7 @@ from util.cmd import Command from util.pip import Pip +from util.units import BuildUnit class Program(Command): @@ -19,12 +20,12 @@ class RunOptions(Command.RunOptions): Allows to customize options for running an external program. """ - def __init__(self, requirements_file: RequirementsFile): + def __init__(self, build_unit: BuildUnit = BuildUnit()): """ - :param requirements_file: The requirements file that should be used for looking up dependency versions + :param build_unit: The build unit from which the program should be run """ super().__init__() - self.requirements_file = requirements_file + self.build_unit = build_unit self.install_program = True self.dependencies = set() @@ -35,16 +36,25 @@ def run(self, command: Command, capture_output: bool) -> CompletedProcess: dependencies.append(command.command) dependencies.extend(self.dependencies) - Pip(self.requirements_file).install_packages(*dependencies) + Pip(self.build_unit).install_packages(*dependencies) return super().run(command, capture_output) - def __init__(self, requirements_file: RequirementsFile, program: str, *arguments: str): + def __init__(self, program: str, *arguments: str): """ - :param requirements_file: The requirements file that should be used for looking up dependency versions - :param program: The name of the program to be run - :param arguments: Optional arguments to be passed to the program + :param program: The name of the program to be run + :param arguments: Optional arguments to be passed to the program """ - super().__init__(program, *arguments, run_options=Program.RunOptions(requirements_file)) + super().__init__(program, *arguments, run_options=Program.RunOptions()) + + def set_build_unit(self, build_unit: BuildUnit) -> 'Program': + """ + Sets the build unit from which the program should be run. + + :param build_unit: The build unit to be set + :return: The `Program` itself + """ + self.run_options.build_unit = build_unit + return self def install_program(self, install_program: bool) -> 'Program': """ @@ -72,13 +82,12 @@ class PythonModule(Program): Allows to run a Python module. """ - def __init__(self, requirements_file: RequirementsFile, module: str, *arguments: str): + def __init__(self, module: str, *arguments: str): """ - :param requirements_file: The requirements file that should be used for looking up dependency versions - :param module: The name of the module to be run - :param arguments: Optional arguments to be passed to the module + :param module: The name of the module to be run + :param arguments: Optional arguments to be passed to the module """ - super().__init__(requirements_file, 'python', '-m', module, *arguments) + super().__init__('python', '-m', module, *arguments) self.module = module self.install_program(True) diff --git a/scons/util/units.py b/scons/util/units.py new file mode 100644 index 000000000..df82f8fee --- /dev/null +++ b/scons/util/units.py @@ -0,0 +1,33 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that provide information about independent units of the build system. +""" +from dataclasses import dataclass +from os import path + + +@dataclass +class BuildUnit: + """ + An independent unit of the build system that may come with its own built-time dependencies. + + Attributes: + root_directory: The path to the root directory of this unit + requirements_file: The path to the requirements file that specifies the dependencies required by this unit + """ + + def __init__(self, root_directory: str = path.join('scons', 'util')): + self.root_directory = root_directory + self.requirements_file: str = path.join(root_directory, 'requirements.txt') + + @staticmethod + def by_name(unit_name: str, *subdirectories: str) -> 'BuildUnit': + """ + Creates and returns a `BuildUnit` with a specific name. + + :param unit_name: The name of the build unit + :param subdirectories: Optional subdirectories + :return: The `BuildUnit` that has been created + """ + return BuildUnit(path.join('scons', unit_name, *subdirectories)) From 2fc1866a532efec4737778bc7fe4562fe746c7bb Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 21 Nov 2024 01:27:28 +0100 Subject: [PATCH 017/114] Remove function "install_build_dependencies". --- build | 2 +- build.bat | 2 +- scons/dependencies.py | 13 ++----------- scons/github_actions.py | 8 ++++---- 4 files changed, 8 insertions(+), 17 deletions(-) diff --git a/build b/build index 1b9f3ccea..bf7fbff16 100755 --- a/build +++ b/build @@ -22,7 +22,7 @@ fi if [ -d "$VENV_DIR" ]; then . $VENV_DIR/bin/activate - python3 -c "import sys; sys.path.append('$SCONS_DIR'); import dependencies; dependencies.install_build_dependencies('scons')" + python3 -c "import sys; sys.path.append('$SCONS_DIR'); from util.pip import Pip; Pip().install_packages('scons')" scons --silent --file $SCONS_DIR/sconstruct.py $@ deactivate fi diff --git a/build.bat b/build.bat index fba6b9952..1b006dc3a 100644 --- a/build.bat +++ b/build.bat @@ -20,7 +20,7 @@ if not exist "%VENV_DIR%" ( if exist "%VENV_DIR%" ( call %VENV_DIR%\Scripts\activate || exit - .\%VENV_DIR%\Scripts\python -c "import sys;sys.path.append('%SCONS_DIR%');import dependencies;dependencies.install_build_dependencies('scons')" || exit + .\%VENV_DIR%\Scripts\python -c "import sys;sys.path.append('%SCONS_DIR%');from util.pip import Pip;Pip().install_packages('scons')" || exit .\%VENV_DIR%\Scripts\python -m SCons --silent --file %SCONS_DIR%\sconstruct.py %* || exit call deactivate || exit ) diff --git a/scons/dependencies.py b/scons/dependencies.py index e49528cd9..44f19c1f5 100644 --- a/scons/dependencies.py +++ b/scons/dependencies.py @@ -8,7 +8,7 @@ from os import path from typing import List -from modules import ALL_MODULES, BUILD_MODULE, CPP_MODULE, PYTHON_MODULE, Module +from modules import ALL_MODULES, CPP_MODULE, PYTHON_MODULE, Module from util.pip import Package, Pip, RequirementsFile @@ -55,21 +55,12 @@ def __install_module_dependencies(module: Module, *dependencies: str): def __print_table(header: List[str], rows: List[List[str]]): - install_build_dependencies('tabulate') + Pip().install_packages('tabulate') # pylint: disable=import-outside-toplevel from tabulate import tabulate print(tabulate(rows, headers=header)) -def install_build_dependencies(*dependencies: str): - """ - Installs one or several dependencies that are required by the build system. - - :param dependencies: The names of the dependencies that should be installed - """ - __install_module_dependencies(BUILD_MODULE, *dependencies) - - def install_runtime_dependencies(**_): """ Installs all runtime dependencies that are required by the Python and C++ module. diff --git a/scons/github_actions.py b/scons/github_actions.py index 5940c4078..6f0abebb9 100644 --- a/scons/github_actions.py +++ b/scons/github_actions.py @@ -11,9 +11,9 @@ from os import environ, path from typing import List, Optional, Set -from dependencies import install_build_dependencies from util.env import get_env from util.io import read_file, write_file +from util.pip import Pip ENV_GITHUB_TOKEN = 'GITHUB_TOKEN' @@ -172,7 +172,7 @@ def __hash__(self): def __read_workflow(workflow_file: str) -> Workflow: - install_build_dependencies('pyyaml') + Pip().install_packages('pyyaml') # pylint: disable=import-outside-toplevel import yaml with read_file(workflow_file) as file: @@ -231,7 +231,7 @@ def __parse_workflows(*workflow_files: str) -> Set[Workflow]: def __query_latest_action_version(action: Action, github_token: Optional[str] = None) -> Optional[ActionVersion]: repository_name = action.repository - install_build_dependencies('pygithub') + Pip().install_packages('pygithub') # pylint: disable=import-outside-toplevel from github import Auth, Github, UnknownObjectException @@ -282,7 +282,7 @@ def __parse_all_workflows() -> Set[Workflow]: def __print_table(header: List[str], rows: List[List[str]]): - install_build_dependencies('tabulate') + Pip().install_packages('tabulate') # pylint: disable=import-outside-toplevel from tabulate import tabulate print(tabulate(rows, headers=header)) From 84db22dbca0cc1217074753114ffee8f3b68f9b7 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 21 Nov 2024 23:03:06 +0100 Subject: [PATCH 018/114] Move class BuildOptions into a separate file. --- scons/build_options.py | 113 +++++++++++++++++++++++++++++++++++++++++ scons/compilation.py | 94 +++------------------------------- 2 files changed, 119 insertions(+), 88 deletions(-) create mode 100644 scons/build_options.py diff --git a/scons/build_options.py b/scons/build_options.py new file mode 100644 index 000000000..7d14aaa63 --- /dev/null +++ b/scons/build_options.py @@ -0,0 +1,113 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that allow to configure build options. +""" +from abc import ABC, abstractmethod +from os import environ +from typing import Iterable, List, Optional + +from util.env import get_env + + +class BuildOption(ABC): + """ + An abstract base class for all build options. + """ + + def __init__(self, name: str, subpackage: Optional[str]): + """ + name: The name of the build option + subpackage: The subpackage, the build option corresponds to, or None, if it is a global option + """ + self.name = name + self.subpackage = subpackage + + @property + def key(self) -> str: + """ + The key to be used for setting the build option. + """ + return (self.subpackage + ':' if self.subpackage else '') + self.name + + @property + @abstractmethod + def value(self) -> Optional[str]: + """ + Returns the value of the build option. + + :return: The value or None, if no value is set + """ + + def __eq__(self, other: 'BuildOption') -> bool: + return self.key == other.key + + def __hash__(self) -> int: + return hash(self.key) + + def __bool__(self) -> bool: + return self.value is not None + + +class EnvBuildOption(BuildOption): + """ + A build option, whose value is obtained from an environment variable. + """ + + def __init__(self, name: str, subpackage: Optional[str] = None): + super().__init__(name, subpackage) + + @property + def value(self) -> Optional[str]: + value = get_env(environ, self.name.upper(), None) + + if value: + value = value.strip() + + return value + + +class BuildOptions(Iterable): + """ + Stores multiple build options. + """ + + def __init__(self): + self.build_options = set() + + def add(self, build_option: BuildOption) -> 'BuildOptions': + """ + Adds a build option. + + :param build_option: The build option to be added + :return: The `BuildOptions` itself + """ + self.build_options.add(build_option) + return self + + def as_arguments(self) -> List[str]: + """ + Returns a list of arguments to be passed to the command "meson configure" for setting the build options. + + :return: A list of arguments + """ + arguments = [] + + for build_option in self: + value = build_option.value + + if value: + arguments.append('-D') + arguments.append(build_option.key + '=' + value) + + return arguments + + def __iter__(self): + return iter(self.build_options) + + def __bool__(self) -> bool: + for build_option in self.build_options: + if build_option: + return True + + return False diff --git a/scons/compilation.py b/scons/compilation.py index 9d7517b43..8eaaeab87 100644 --- a/scons/compilation.py +++ b/scons/compilation.py @@ -3,101 +3,19 @@ Provides utility functions for compiling C++ and Cython code. """ -from os import environ -from typing import List, Optional - +from build_options import BuildOptions, EnvBuildOption from modules import CPP_MODULE, PYTHON_MODULE -from util.env import get_env from util.run import Program - -class BuildOptions: - """ - Allows to obtain build options from environment variables. - """ - - class BuildOption: - """ - A single build option. - """ - - def __init__(self, name: str, subpackage: Optional[str] = None): - """ - :param name: The name of the build option - :param subpackage: The subpackage, the build option corresponds to, or None, if it is a global option - """ - self.name = name - self.subpackage = subpackage - - @property - def key(self) -> str: - """ - The key to be used for setting the build option. - """ - return (self.subpackage + ':' if self.subpackage else '') + self.name - - @property - def value(self) -> Optional[str]: - """ - Returns the value to be set for the build option. - - :return: The value to be set or None, if no value should be set - """ - value = get_env(environ, self.name.upper(), None) - - if value: - value = value.strip() - - return value - - def __init__(self): - self.build_options = [] - - def add(self, name: str, subpackage: Optional[str] = None) -> 'BuildOptions': - """ - Adds a build option. - - :param name: The name of the build option - :param subpackage: The subpackage, the build option corresponds to, or None, if it is a global option - :return: The `BuildOptions` itself - """ - self.build_options.append(BuildOptions.BuildOption(name=name, subpackage=subpackage)) - return self - - def as_arguments(self) -> List[str]: - """ - Returns a list of arguments to be passed to the command "meson configure" for setting the build options. - - :return: A list of arguments - """ - arguments = [] - - for build_option in self.build_options: - value = build_option.value - - if value: - arguments.append('-D') - arguments.append(build_option.key + '=' + value) - - return arguments - - def __bool__(self) -> bool: - for build_option in self.build_options: - if build_option.value: - return True - - return False - - CPP_BUILD_OPTIONS = BuildOptions() \ - .add(name='subprojects') \ - .add(name='test_support', subpackage='common') \ - .add(name='multi_threading_support', subpackage='common') \ - .add(name='gpu_support', subpackage='common') + .add(EnvBuildOption(name='subprojects')) \ + .add(EnvBuildOption(name='test_support', subpackage='common')) \ + .add(EnvBuildOption(name='multi_threading_support', subpackage='common')) \ + .add(EnvBuildOption(name='gpu_support', subpackage='common')) CYTHON_BUILD_OPTIONS = BuildOptions() \ - .add(name='subprojects') + .add(EnvBuildOption(name='subprojects')) class MesonSetup(Program): From 8c55dadf99480a18721d0b495cb29a76aa424c13 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 21 Nov 2024 23:17:50 +0100 Subject: [PATCH 019/114] Move classes MesonSetup, MesonConfigure, MesonCompile and MesonInstall into a separate file. --- scons/build_options.py | 19 +------- scons/compilation.py | 63 +-------------------------- scons/meson.py | 99 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 80 deletions(-) create mode 100644 scons/meson.py diff --git a/scons/build_options.py b/scons/build_options.py index 7d14aaa63..0cab43c8d 100644 --- a/scons/build_options.py +++ b/scons/build_options.py @@ -5,7 +5,7 @@ """ from abc import ABC, abstractmethod from os import environ -from typing import Iterable, List, Optional +from typing import Iterable, Optional from util.env import get_env @@ -85,23 +85,6 @@ def add(self, build_option: BuildOption) -> 'BuildOptions': self.build_options.add(build_option) return self - def as_arguments(self) -> List[str]: - """ - Returns a list of arguments to be passed to the command "meson configure" for setting the build options. - - :return: A list of arguments - """ - arguments = [] - - for build_option in self: - value = build_option.value - - if value: - arguments.append('-D') - arguments.append(build_option.key + '=' + value) - - return arguments - def __iter__(self): return iter(self.build_options) diff --git a/scons/compilation.py b/scons/compilation.py index 8eaaeab87..f55ae86a2 100644 --- a/scons/compilation.py +++ b/scons/compilation.py @@ -4,8 +4,8 @@ Provides utility functions for compiling C++ and Cython code. """ from build_options import BuildOptions, EnvBuildOption +from meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup from modules import CPP_MODULE, PYTHON_MODULE -from util.run import Program CPP_BUILD_OPTIONS = BuildOptions() \ .add(EnvBuildOption(name='subprojects')) \ @@ -18,67 +18,6 @@ .add(EnvBuildOption(name='subprojects')) -class MesonSetup(Program): - """ - Allows to run the external program "meson setup". - """ - - def __init__(self, build_directory: str, source_directory: str, build_options: BuildOptions = BuildOptions()): - """ - :param build_directory: The path to the build directory - :param source_directory: The path to the source directory - :param build_options: The build options to be used - """ - super().__init__('meson', 'setup', *build_options.as_arguments(), build_directory, source_directory) - self.print_arguments(True) - - -class MesonConfigure(Program): - """ - Allows to run the external program "meson configure". - """ - - def __init__(self, build_directory: str, build_options: BuildOptions = BuildOptions()): - """ - :param build_directory: The path to the build directory - :param build_options: The build options to be used - """ - super().__init__('meson', 'configure', *build_options.as_arguments(), build_directory) - self.print_arguments(True) - self.build_options = build_options - - def run(self): - if self.build_options: - print('Configuring build options according to environment variables...') - super().run() - - -class MesonCompile(Program): - """ - Allows to run the external program "meson compile". - """ - - def __init__(self, build_directory: str): - """ - :param build_directory: The path to the build directory - """ - super().__init__('meson', 'compile', '-C', build_directory) - self.print_arguments(True) - - -class MesonInstall(Program): - """ - Allows to run the external program "meson install". - """ - - def __init__(self, build_directory: str): - """ - :param build_directory: The path to the build directory - """ - super().__init__('meson', 'install', '--no-rebuild', '--only-changed', '-C', build_directory) - self.print_arguments(True) - - def setup_cpp(**_): """ Sets up the build system for compiling the C++ code. diff --git a/scons/meson.py b/scons/meson.py new file mode 100644 index 000000000..6d9482158 --- /dev/null +++ b/scons/meson.py @@ -0,0 +1,99 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that allow to run the external program "meson". +""" +from abc import ABC + +from build_options import BuildOptions +from util.run import Program +from typing import List + + +def build_options_as_meson_arguments(build_options: BuildOptions) -> List[str]: + """ + Returns a list of arguments that can be passed to meson for setting build options. + + :param build_options: The build options + :return: A list of arguments + """ + arguments = [] + + for build_option in build_options: + if build_option: + arguments.append('-D') + arguments.append(build_option.key + '=' + build_option.value) + + return arguments + + +class Meson(Program, ABC): + """ + An abstract base class for all classes that allow to run the external program "meson". + """ + + def __init__(self, meson_command: str, *arguments: str): + """ + :param program: The meson command to be run + :param arguments: Optional arguments to be passed to meson + """ + super().__init__('meson', meson_command, *arguments) + self.print_arguments(True) + + +class MesonSetup(Meson): + """ + Allows to run the external program "meson setup". + """ + + def __init__(self, build_directory: str, source_directory: str, build_options: BuildOptions = BuildOptions()): + """ + :param build_directory: The path to the build directory + :param source_directory: The path to the source directory + :param build_options: The build options to be used + """ + super().__init__('setup', *build_options_as_meson_arguments(build_options), build_directory, source_directory) + + +class MesonConfigure(Meson): + """ + Allows to run the external program "meson configure". + """ + + def __init__(self, build_directory: str, build_options: BuildOptions = BuildOptions()): + """ + :param build_directory: The path to the build directory + :param build_options: The build options to be used + """ + super().__init__('configure', *build_options_as_meson_arguments(build_options), build_directory) + self.build_options = build_options + + def _should_be_skipped(self) -> bool: + return not self.build_options + + def _before(self): + print('Configuring build options according to environment variables...') + + +class MesonCompile(Meson): + """ + Allows to run the external program "meson compile". + """ + + def __init__(self, build_directory: str): + """ + :param build_directory: The path to the build directory + """ + super().__init__('compile', '-C', build_directory) + + +class MesonInstall(Meson): + """ + Allows to run the external program "meson install". + """ + + def __init__(self, build_directory: str): + """ + :param build_directory: The path to the build directory + """ + super().__init__('install', '--no-rebuild', '--only-changed', '-C', build_directory) From 58b412e439a569fca379c68f243e1b002e85d3ac Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 21 Nov 2024 23:20:27 +0100 Subject: [PATCH 020/114] Split file compilation.py into files cpp.py and cython.py. --- scons/compilation.py | 74 -------------------------------------------- scons/cpp.py | 42 +++++++++++++++++++++++++ scons/cython.py | 39 +++++++++++++++++++++++ scons/meson.py | 2 +- scons/sconstruct.py | 4 ++- 5 files changed, 85 insertions(+), 76 deletions(-) delete mode 100644 scons/compilation.py create mode 100644 scons/cpp.py create mode 100644 scons/cython.py diff --git a/scons/compilation.py b/scons/compilation.py deleted file mode 100644 index f55ae86a2..000000000 --- a/scons/compilation.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Provides utility functions for compiling C++ and Cython code. -""" -from build_options import BuildOptions, EnvBuildOption -from meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup -from modules import CPP_MODULE, PYTHON_MODULE - -CPP_BUILD_OPTIONS = BuildOptions() \ - .add(EnvBuildOption(name='subprojects')) \ - .add(EnvBuildOption(name='test_support', subpackage='common')) \ - .add(EnvBuildOption(name='multi_threading_support', subpackage='common')) \ - .add(EnvBuildOption(name='gpu_support', subpackage='common')) - - -CYTHON_BUILD_OPTIONS = BuildOptions() \ - .add(EnvBuildOption(name='subprojects')) - - -def setup_cpp(**_): - """ - Sets up the build system for compiling the C++ code. - """ - MesonSetup(build_directory=CPP_MODULE.build_dir, - source_directory=CPP_MODULE.root_dir, - build_options=CPP_BUILD_OPTIONS) \ - .add_dependencies('ninja') \ - .run() - - -def compile_cpp(**_): - """ - Compiles the C++ code. - """ - MesonConfigure(CPP_MODULE.build_dir, CPP_BUILD_OPTIONS).run() - print('Compiling C++ code...') - MesonCompile(CPP_MODULE.build_dir).run() - - -def install_cpp(**_): - """ - Installs shared libraries into the source tree. - """ - print('Installing shared libraries into source tree...') - MesonInstall(CPP_MODULE.build_dir).run() - - -def setup_cython(**_): - """ - Sets up the build system for compiling the Cython code. - """ - MesonSetup(build_directory=PYTHON_MODULE.build_dir, - source_directory=PYTHON_MODULE.root_dir, - build_options=CYTHON_BUILD_OPTIONS) \ - .add_dependencies('cython') \ - .run() - - -def compile_cython(**_): - """ - Compiles the Cython code. - """ - MesonConfigure(PYTHON_MODULE.build_dir, CYTHON_BUILD_OPTIONS) - print('Compiling Cython code...') - MesonCompile(PYTHON_MODULE.build_dir).run() - - -def install_cython(**_): - """ - Installs extension modules into the source tree. - """ - print('Installing extension modules into source tree...') - MesonInstall(PYTHON_MODULE.build_dir).run() diff --git a/scons/cpp.py b/scons/cpp.py new file mode 100644 index 000000000..5c2795337 --- /dev/null +++ b/scons/cpp.py @@ -0,0 +1,42 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for compiling C++ code. +""" +from build_options import BuildOptions, EnvBuildOption +from meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup +from modules import CPP_MODULE + +BUILD_OPTIONS = BuildOptions() \ + .add(EnvBuildOption(name='subprojects')) \ + .add(EnvBuildOption(name='test_support', subpackage='common')) \ + .add(EnvBuildOption(name='multi_threading_support', subpackage='common')) \ + .add(EnvBuildOption(name='gpu_support', subpackage='common')) + + +def setup_cpp(**_): + """ + Sets up the build system for compiling the C++ code. + """ + MesonSetup(build_directory=CPP_MODULE.build_dir, + source_directory=CPP_MODULE.root_dir, + build_options=BUILD_OPTIONS) \ + .add_dependencies('ninja') \ + .run() + + +def compile_cpp(**_): + """ + Compiles the C++ code. + """ + MesonConfigure(CPP_MODULE.build_dir, BUILD_OPTIONS).run() + print('Compiling C++ code...') + MesonCompile(CPP_MODULE.build_dir).run() + + +def install_cpp(**_): + """ + Installs shared libraries into the source tree. + """ + print('Installing shared libraries into source tree...') + MesonInstall(CPP_MODULE.build_dir).run() diff --git a/scons/cython.py b/scons/cython.py new file mode 100644 index 000000000..f0ad013ae --- /dev/null +++ b/scons/cython.py @@ -0,0 +1,39 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for compiling Cython code. +""" +from build_options import BuildOptions, EnvBuildOption +from meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup +from modules import PYTHON_MODULE + +BUILD_OPTIONS = BuildOptions() \ + .add(EnvBuildOption(name='subprojects')) + + +def setup_cython(**_): + """ + Sets up the build system for compiling the Cython code. + """ + MesonSetup(build_directory=PYTHON_MODULE.build_dir, + source_directory=PYTHON_MODULE.root_dir, + build_options=BUILD_OPTIONS) \ + .add_dependencies('cython') \ + .run() + + +def compile_cython(**_): + """ + Compiles the Cython code. + """ + MesonConfigure(PYTHON_MODULE.build_dir, BUILD_OPTIONS) + print('Compiling Cython code...') + MesonCompile(PYTHON_MODULE.build_dir).run() + + +def install_cython(**_): + """ + Installs extension modules into the source tree. + """ + print('Installing extension modules into source tree...') + MesonInstall(PYTHON_MODULE.build_dir).run() diff --git a/scons/meson.py b/scons/meson.py index 6d9482158..6e6796599 100644 --- a/scons/meson.py +++ b/scons/meson.py @@ -4,10 +4,10 @@ Provides classes that allow to run the external program "meson". """ from abc import ABC +from typing import List from build_options import BuildOptions from util.run import Program -from typing import List def build_options_as_meson_arguments(build_options: BuildOptions) -> List[str]: diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 0da020c6b..ab29ab3d1 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -9,7 +9,7 @@ from code_style import check_cpp_code_style, check_md_code_style, check_python_code_style, check_yaml_code_style, \ enforce_cpp_code_style, enforce_md_code_style, enforce_python_code_style, enforce_yaml_code_style -from compilation import compile_cpp, compile_cython, install_cpp, install_cython, setup_cpp, setup_cython +from cython import compile_cython, install_cython, setup_cython from dependencies import check_dependency_versions, install_runtime_dependencies from documentation import apidoc_cpp, apidoc_cpp_tocfile, apidoc_python, apidoc_python_tocfile, doc from github_actions import check_github_actions, update_github_actions @@ -21,6 +21,8 @@ from util.reflection import import_source_file from util.targets import Target +from cpp import compile_cpp, install_cpp, setup_cpp + from SCons.Script import COMMAND_LINE_TARGETS from SCons.Script.SConscript import SConsEnvironment From c9a132e5638c3e9c5a49a0ceecc877754585b2d7 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 21 Nov 2024 23:28:35 +0100 Subject: [PATCH 021/114] Move file build_options.py into subdirectory "compilation". --- scons/compilation/__init__.py | 0 scons/{ => compilation}/build_options.py | 0 scons/cpp.py | 2 +- scons/cython.py | 2 +- scons/meson.py | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 scons/compilation/__init__.py rename scons/{ => compilation}/build_options.py (100%) diff --git a/scons/compilation/__init__.py b/scons/compilation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scons/build_options.py b/scons/compilation/build_options.py similarity index 100% rename from scons/build_options.py rename to scons/compilation/build_options.py diff --git a/scons/cpp.py b/scons/cpp.py index 5c2795337..b1cdb7e98 100644 --- a/scons/cpp.py +++ b/scons/cpp.py @@ -3,7 +3,7 @@ Provides utility functions for compiling C++ code. """ -from build_options import BuildOptions, EnvBuildOption +from compilation.build_options import BuildOptions, EnvBuildOption from meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup from modules import CPP_MODULE diff --git a/scons/cython.py b/scons/cython.py index f0ad013ae..b54ccd9f3 100644 --- a/scons/cython.py +++ b/scons/cython.py @@ -3,7 +3,7 @@ Provides utility functions for compiling Cython code. """ -from build_options import BuildOptions, EnvBuildOption +from compilation.build_options import BuildOptions, EnvBuildOption from meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup from modules import PYTHON_MODULE diff --git a/scons/meson.py b/scons/meson.py index 6e6796599..5764b7086 100644 --- a/scons/meson.py +++ b/scons/meson.py @@ -6,7 +6,7 @@ from abc import ABC from typing import List -from build_options import BuildOptions +from compilation.build_options import BuildOptions from util.run import Program From da355ee5a93db4d0ded4ad398b4c9f56558e4037 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 21 Nov 2024 23:29:15 +0100 Subject: [PATCH 022/114] Move file meson.py into subdirectory "compilation". --- scons/{ => compilation}/meson.py | 0 scons/cpp.py | 2 +- scons/cython.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename scons/{ => compilation}/meson.py (100%) diff --git a/scons/meson.py b/scons/compilation/meson.py similarity index 100% rename from scons/meson.py rename to scons/compilation/meson.py diff --git a/scons/cpp.py b/scons/cpp.py index b1cdb7e98..0701da4e2 100644 --- a/scons/cpp.py +++ b/scons/cpp.py @@ -4,7 +4,7 @@ Provides utility functions for compiling C++ code. """ from compilation.build_options import BuildOptions, EnvBuildOption -from meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup +from compilation.meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup from modules import CPP_MODULE BUILD_OPTIONS = BuildOptions() \ diff --git a/scons/cython.py b/scons/cython.py index b54ccd9f3..446c92722 100644 --- a/scons/cython.py +++ b/scons/cython.py @@ -4,7 +4,7 @@ Provides utility functions for compiling Cython code. """ from compilation.build_options import BuildOptions, EnvBuildOption -from meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup +from compilation.meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup from modules import PYTHON_MODULE BUILD_OPTIONS = BuildOptions() \ From 754a69c4de1a300c9b22becc7c7892ef62798daa Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Fri, 22 Nov 2024 17:26:54 +0100 Subject: [PATCH 023/114] Add enum Language. --- scons/util/languages.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 scons/util/languages.py diff --git a/scons/util/languages.py b/scons/util/languages.py new file mode 100644 index 000000000..7be34fd1a --- /dev/null +++ b/scons/util/languages.py @@ -0,0 +1,17 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that help to distinguish between different programming languages. +""" +from enum import Enum + + +class Language(Enum): + """ + Different programming languages. + """ + CPP = {'cpp', 'hpp'} + PYTHON = {'py'} + CYTHON = {'pyx', 'pxd'} + MARKDOWN = {'md'} + YAML = {'yaml', 'yml'} From 4cec8f8465ba240526830ab3a437156a6ff19ac4 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Fri, 22 Nov 2024 00:22:27 +0100 Subject: [PATCH 024/114] Add classes Module and ModuleRegistry. --- scons/util/modules.py | 63 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 scons/util/modules.py diff --git a/scons/util/modules.py b/scons/util/modules.py new file mode 100644 index 000000000..61ef8060b --- /dev/null +++ b/scons/util/modules.py @@ -0,0 +1,63 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that provide information about files and directories that belong to individual modules of the project +to be dealt with by the build system. +""" +from abc import ABC +from functools import reduce +from typing import List + + +class Module(ABC): + """ + An abstract base class for all modules. + """ + + class Filter(ABC): + """ + An abstract base class for all classes that allow to filter modules. + """ + + def matches(self, module: 'Module') -> bool: + """ + Returns whether the filter matches a given module or not. + + :param module: The module to be matched + :return: True, if the filter matches the given module, False otherwise + """ + + def match(self, module_filter: Filter) -> List['Module']: + """ + Returns a list that contains all submodules in this module that match a given filter. + + :param module_filter: The filter + :return: A list that contains all matching submodules + """ + return [self] if module_filter.matches(self) else [] + + +class ModuleRegistry: + """ + Allows to look up modules that have previously been registered. + """ + + def __init__(self): + self.modules = [] + + def register(self, module: Module): + """ + Registers a new module. + + :param module: The module to be registered + """ + self.modules.append(module) + + def lookup(self, module_filter: Module.Filter) -> List[Module]: + """ + Looks up and returns all modules that match a given filter. + + :param module_filter: The filter + :return: A list that contains all modules matching the given filter + """ + return list(reduce(lambda aggr, module: aggr + module.match(module_filter), self.modules, [])) From a82b03e809e9674636a7eb54b9c9b064d1eaa194 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 23 Nov 2024 18:31:07 +0100 Subject: [PATCH 025/114] Add class FileSearch. --- scons/util/files.py | 94 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/scons/util/files.py b/scons/util/files.py index c7f5caf67..01c2fb219 100644 --- a/scons/util/files.py +++ b/scons/util/files.py @@ -8,6 +8,8 @@ from os import path from typing import Callable, List +from util.languages import Language + class DirectorySearch: """ @@ -75,3 +77,95 @@ def filter_file(file: str) -> bool: result.extend(subdirectories) return result + + +class FileSearch: + """ + Allows to search for files. + """ + + def __init__(self): + self.hidden = False + self.file_patterns = {'*'} + self.directory_search = DirectorySearch() + + def set_recursive(self, recursive: bool) -> 'FileSearch': + """ + Sets whether the search should be recursive or not. + + :param recursive: True, if the search should be recursive, False otherwise + :return: The `FileSearch` itself + """ + self.directory_search.set_recursive(recursive) + return self + + def exclude_subdirectories(self, *excludes: DirectorySearch.Filter) -> 'FileSearch': + """ + Sets one or several filters that should be used for excluding subdirectories. Does only have an effect if the + search is recursive. + + :param excludes: The filters to be set + :return: The `FileSearch` itself + """ + self.directory_search.exclude(*excludes) + return self + + def exclude_subdirectories_by_name(self, *names: str) -> 'FileSearch': + """ + Sets one or several filters that should be used for excluding subdirectories by their names. Does only have an + effect if the search is recursive. + + :param names: The names of the subdirectories to be excluded + :return: The `FileSearch` itself + """ + self.directory_search.exclude_by_name(*names) + return self + + def set_hidden(self, hidden: bool) -> 'FileSearch': + """ + Sets whether hidden files should be included or not. + + :param hidden: True, if hidden files should be included, False otherwise + """ + self.hidden = hidden + return self + + def set_suffixes(self, *suffixes: str) -> 'FileSearch': + """ + Sets the suffixes of the files that should be included. + + :param suffixes: The suffixes of the files that should be included (without starting dot) + :return: The `FileSearch` itself + """ + self.file_patterns = {'*.' + suffix for suffix in suffixes} + return self + + def set_languages(self, *languages: Language) -> 'FileSearch': + """ + Sets the suffixes of the files that should be included. + + :param languages: The languages of the files that should be included + :return: The `FileSearch` itself + """ + return self.set_suffixes(*reduce(lambda aggr, language: aggr | language.value, languages, set())) + + def list(self, *directories: str) -> List[str]: + """ + Lists all files that can be found in given directories. + + :param directories: The directories to search for files + :return: A list that contains all files that have been found + """ + result = [] + subdirectories = self.directory_search.list(*directories) if self.directory_search.recursive else [] + + for directory in list(directories) + subdirectories: + for file_pattern in self.file_patterns: + files = [file for file in glob(path.join(directory, file_pattern)) if path.isfile(file)] + + if self.hidden: + files.extend([file for file in glob(path.join(directory, '.' + file_pattern)) if path.isfile(file)]) + + result.extend(files) + + return result From ba5d47803f6dc93a1f079e0085274f6e52fd2a69 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 21 Nov 2024 23:44:50 +0100 Subject: [PATCH 026/114] Add class CodeModule. --- scons/code_style/__init__.py | 0 scons/code_style/modules.py | 52 ++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 scons/code_style/__init__.py create mode 100644 scons/code_style/modules.py diff --git a/scons/code_style/__init__.py b/scons/code_style/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scons/code_style/modules.py b/scons/code_style/modules.py new file mode 100644 index 000000000..cf545255e --- /dev/null +++ b/scons/code_style/modules.py @@ -0,0 +1,52 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that provide access to directories and files that belong to individual modules. +""" +from typing import List + +from util.files import FileSearch +from util.languages import Language +from util.modules import Module + + +class CodeModule(Module): + """ + A module that contains source code. + """ + + class Filter(Module.Filter): + """ + A filter that matches code modules. + """ + + def __init__(self, *languages: Language): + """ + :param languages: The languages of the code modules to be matched or None, if no restrictions should be + imposed on the languages + """ + self.languages = set(languages) + + def matches(self, module: Module) -> bool: + return isinstance(module, CodeModule) and (not self.languages or module.language in self.languages) + + def __init__(self, + language: Language, + root_directory: str, + file_search: FileSearch = FileSearch().set_recursive(True)): + """ + :param language: The programming language of the source code that belongs to the module + :param root_directory: The path to the module's root directory + :param file_search: The `FileSearch` that should be used to search for source files + """ + self.language = language + self.root_directory = root_directory + self.file_search = file_search + + def find_source_files(self) -> List[str]: + """ + Finds and returns all source files that belong to the module. + + :return: A list that contains the paths of the source files that have been found + """ + return self.file_search.set_languages(self.language).list(self.root_directory) From 4eadc0da3f666d94099ee8754ccbc24716329348 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 21 Nov 2024 23:55:03 +0100 Subject: [PATCH 027/114] Add class CompilationModule. --- scons/compilation/modules.py | 45 ++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 scons/compilation/modules.py diff --git a/scons/compilation/modules.py b/scons/compilation/modules.py new file mode 100644 index 000000000..1590059ed --- /dev/null +++ b/scons/compilation/modules.py @@ -0,0 +1,45 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that provide access to directories and files that belong to individual modules. +""" +from os import path + +from util.languages import Language +from util.modules import Module + + +class CompilationModule(Module): + """ + A module that contains source code that must be compiled. + """ + + class Filter(Module.Filter): + """ + A filter that matches modules that contain source code that must be compiled. + """ + + def __init__(self, *languages: Language): + """ + :param languages: The languages of the source code contained by the modules to be matched or None, if no + restrictions should be imposed on the languages + """ + self.languages = set(languages) + + def matches(self, module: Module) -> bool: + return isinstance(module, CompilationModule) and (not self.languages or module.language in self.languages) + + def __init__(self, language: Language, root_directory: str): + """ + :param language: The programming language of the source code that belongs to the module + :param root_directory: The path to the module's root directory + """ + self.language = language + self.root_directory = root_directory + + @property + def build_directory(self) -> str: + """ + The path to the directory, where build files should be stored. + """ + return path.join(self.root_directory, 'build') From 6bc5292ad5ad14f0c3ae112cb6bba86f2b674ca0 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Fri, 22 Nov 2024 18:41:51 +0100 Subject: [PATCH 028/114] Add class PhonyTarget.Builder. --- scons/sconstruct.py | 4 +- scons/util/targets.py | 78 +++++++++++++++++++++++++++++++++--- scons/versioning/__init__.py | 28 ++++++------- 3 files changed, 89 insertions(+), 21 deletions(-) diff --git a/scons/sconstruct.py b/scons/sconstruct.py index ab29ab3d1..552900698 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -18,6 +18,7 @@ from testing import tests_cpp, tests_python from util.files import DirectorySearch from util.format import format_iterable +from util.modules import ModuleRegistry from util.reflection import import_source_file from util.targets import Target @@ -81,6 +82,7 @@ def __print_if_clean(environment, message: str): # Register build targets... env = SConsEnvironment() +module_registry = ModuleRegistry() for subdirectory in DirectorySearch().set_recursive(True).list(BUILD_MODULE.root_dir): init_file = path.join(subdirectory, '__init__.py') @@ -88,7 +90,7 @@ def __print_if_clean(environment, message: str): if path.isfile(init_file): for build_target in getattr(import_source_file(init_file), 'TARGETS', []): if isinstance(build_target, Target): - build_target.register(env) + build_target.register(env, module_registry) VALID_TARGETS.add(build_target.name) # Raise an error if any invalid targets are given... diff --git a/scons/util/targets.py b/scons/util/targets.py index 23c3791d1..31e5990e2 100644 --- a/scons/util/targets.py +++ b/scons/util/targets.py @@ -6,6 +6,8 @@ from abc import ABC, abstractmethod from typing import Callable +from util.modules import ModuleRegistry + from SCons.Script.SConscript import SConsEnvironment as Environment @@ -21,11 +23,12 @@ def __init__(self, name: str): self.name = name @abstractmethod - def register(self, environment: Environment): + def register(self, environment: Environment, module_registry: ModuleRegistry): """ Must be implemented by subclasses in order to register the target. - :param environment: The environment, the target should be registered at + :param environment: The environment, the target should be registered at + :param module_registry: The `ModuleRegistry` that can be used by the target for looking up modules """ @@ -34,13 +37,76 @@ class PhonyTarget(Target): A phony target, which executes a certain action and does not produce any output files. """ - def __init__(self, name: str, action: Callable[[], None]): + Function = Callable[[], None] + + class Runnable(ABC): + """ + An abstract base class for all classes that can be run via a phony target. + """ + + def run(self, modules: ModuleRegistry): + """ + Must be implemented by subclasses in order to run the target. + + :param modules: A `ModuleRegistry` that can be used by the target for looking up modules + """ + + class Builder: + """ + A builder that allows to configure and create phony targets. + """ + + def __init__(self, name: str): + """ + :param name: The name of the target + """ + self.name = name + self.function = None + self.runnable = None + + def set_function(self, function: 'PhonyTarget.Function') -> 'PhonyTarget.Builder': + """ + Sets a function to be run by the target. + + :param function: The function to be run + :return: The `PhonyTarget.Builder` itself + """ + self.function = function + return self + + def set_runnable(self, runnable: 'PhonyTarget.Runnable') -> 'PhonyTarget.Builder': + """ + Sets a runnable to be run by the target. + + :param runnable: The runnable to be run + :return: The `PhonyTarget.Builder` itself + """ + self.runnable = runnable + return self + + def build(self) -> 'PhonyTarget': + """ + Creates and returns the phony target that has been configured via the builder. + + :return: The phony target that has been created + """ + + def action(module_registry: ModuleRegistry): + if self.function: + self.function() + + if self.runnable: + self.runnable.run(module_registry) + + return PhonyTarget(self.name, action) + + def __init__(self, name: str, action: Callable[[ModuleRegistry], None]): """ :param name: The name of the target - :param action: The function to be executed by the target + :param action: The action to be executed by the target """ super().__init__(name) self.action = action - def register(self, environment: Environment): - return environment.AlwaysBuild(environment.Alias(self.name, None, lambda **_: self.action())) + def register(self, environment: Environment, module_registry: ModuleRegistry): + environment.AlwaysBuild(environment.Alias(self.name, None, action=lambda **_: self.action(module_registry))) diff --git a/scons/versioning/__init__.py b/scons/versioning/__init__.py index d433bc3ad..b79fcb6a4 100644 --- a/scons/versioning/__init__.py +++ b/scons/versioning/__init__.py @@ -9,24 +9,24 @@ TARGETS = [ # Targets for updating the project's version - PhonyTarget('increment_development_version', action=increment_development_version), - PhonyTarget('reset_development_version', action=reset_development_version), - PhonyTarget('apply_development_version', action=apply_development_version), - PhonyTarget('increment_patch_version', action=increment_patch_version), - PhonyTarget('increment_minor_version', action=increment_minor_version), - PhonyTarget('increment_major_version', action=increment_major_version), + PhonyTarget.Builder('increment_development_version').set_function(increment_development_version).build(), + PhonyTarget.Builder('reset_development_version').set_function(reset_development_version).build(), + PhonyTarget.Builder('apply_development_version').set_function(apply_development_version).build(), + PhonyTarget.Builder('increment_patch_version').set_function(increment_patch_version).build(), + PhonyTarget.Builder('increment_minor_version').set_function(increment_minor_version).build(), + PhonyTarget.Builder('increment_major_version').set_function(increment_major_version).build(), # Targets for validating changelogs - PhonyTarget('validate_changelog_bugfix', action=validate_changelog_bugfix), - PhonyTarget('validate_changelog_feature', action=validate_changelog_feature), - PhonyTarget('validate_changelog_main', action=validate_changelog_main), + PhonyTarget.Builder('validate_changelog_bugfix').set_function(validate_changelog_bugfix).build(), + PhonyTarget.Builder('validate_changelog_feature').set_function(validate_changelog_feature).build(), + PhonyTarget.Builder('validate_changelog_main').set_function(validate_changelog_main).build(), # Targets for updating the project's changelog - PhonyTarget('update_changelog_bugfix', action=update_changelog_bugfix), - PhonyTarget('update_changelog_feature', action=update_changelog_feature), - PhonyTarget('update_changelog_main', action=update_changelog_main), + PhonyTarget.Builder('update_changelog_bugfix').set_function(update_changelog_bugfix).build(), + PhonyTarget.Builder('update_changelog_feature').set_function(update_changelog_feature).build(), + PhonyTarget.Builder('update_changelog_main').set_function(update_changelog_main).build(), # Targets for printing information about the project - PhonyTarget(name='print_version', action=print_current_version), - PhonyTarget(name='print_latest_changelog', action=print_latest_changelog) + PhonyTarget.Builder('print_version').set_function(print_current_version).build(), + PhonyTarget.Builder('print_latest_changelog').set_function(print_latest_changelog).build() ] From 7f1608329a4e8596f3d81f783a67f89232c84ff3 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 23 Nov 2024 00:10:22 +0100 Subject: [PATCH 029/114] Add class TargetBuilder. --- scons/util/targets.py | 82 +++++++++++++++++++++++++++--------- scons/versioning/__init__.py | 41 ++++++++---------- 2 files changed, 80 insertions(+), 43 deletions(-) diff --git a/scons/util/targets.py b/scons/util/targets.py index 31e5990e2..85f5c32c2 100644 --- a/scons/util/targets.py +++ b/scons/util/targets.py @@ -4,9 +4,10 @@ Provides base classes for defining individual targets of the build process. """ from abc import ABC, abstractmethod -from typing import Callable +from typing import Callable, List from util.modules import ModuleRegistry +from util.units import BuildUnit from SCons.Script.SConscript import SConsEnvironment as Environment @@ -16,6 +17,19 @@ class Target(ABC): An abstract base class for all targets of the build system. """ + class Builder(ABC): + """ + An abstract base class for all builders that allow to configure and create targets. + """ + + @abstractmethod + def build(self) -> 'Target': + """ + Creates and returns the target that has been configured via the builder. + + :return: The target that has been created + """ + def __init__(self, name: str): """ :param name: The name of the target @@ -44,59 +58,57 @@ class Runnable(ABC): An abstract base class for all classes that can be run via a phony target. """ - def run(self, modules: ModuleRegistry): + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): """ Must be implemented by subclasses in order to run the target. - :param modules: A `ModuleRegistry` that can be used by the target for looking up modules + :param build_unit: The build unit, the target belongs to + :param modules: A `ModuleRegistry` that can be used by the target for looking up modules """ - class Builder: + class Builder(Target.Builder): """ A builder that allows to configure and create phony targets. """ - def __init__(self, name: str): + def __init__(self, target_builder: 'TargetBuilder', name: str): """ - :param name: The name of the target + :param target_builder: The `TargetBuilder`, this builder has been created from + :param name: The name of the target """ + self.target_builder = target_builder self.name = name self.function = None self.runnable = None - def set_function(self, function: 'PhonyTarget.Function') -> 'PhonyTarget.Builder': + def set_function(self, function: 'PhonyTarget.Function') -> 'TargetBuilder': """ Sets a function to be run by the target. :param function: The function to be run - :return: The `PhonyTarget.Builder` itself + :return: The `TargetBuilder`, this builder has been created from """ self.function = function - return self + return self.target_builder - def set_runnable(self, runnable: 'PhonyTarget.Runnable') -> 'PhonyTarget.Builder': + def set_runnable(self, runnable: 'PhonyTarget.Runnable') -> 'TargetBuilder': """ Sets a runnable to be run by the target. :param runnable: The runnable to be run - :return: The `PhonyTarget.Builder` itself + :return: The `TargetBuilder`, this builder has been created from """ self.runnable = runnable - return self + return self.target_builder - def build(self) -> 'PhonyTarget': - """ - Creates and returns the phony target that has been configured via the builder. - - :return: The phony target that has been created - """ + def build(self) -> Target: def action(module_registry: ModuleRegistry): if self.function: self.function() if self.runnable: - self.runnable.run(module_registry) + self.runnable.run(self.target_builder.build_unit, module_registry) return PhonyTarget(self.name, action) @@ -110,3 +122,35 @@ def __init__(self, name: str, action: Callable[[ModuleRegistry], None]): def register(self, environment: Environment, module_registry: ModuleRegistry): environment.AlwaysBuild(environment.Alias(self.name, None, action=lambda **_: self.action(module_registry))) + + +class TargetBuilder: + """ + A builder that allows to configure and create multiple targets. + """ + + def __init__(self, build_unit: BuildUnit = BuildUnit()): + """ + :param build_unit: The build unit, the targets belong to + """ + self.build_unit = build_unit + self.target_builders = [] + + def add_phony_target(self, name: str) -> PhonyTarget.Builder: + """ + Adds a phony target. + + :param name: The name of the target + :return: A `PhonyTarget.Builder` that allows to configure the target + """ + target_builder = PhonyTarget.Builder(self, name) + self.target_builders.append(target_builder) + return target_builder + + def build(self) -> List[Target]: + """ + Creates and returns the targets that have been configured via the builder. + + :return: A list that stores the targets that have been created + """ + return [target_builder.build() for target_builder in self.target_builders] diff --git a/scons/versioning/__init__.py b/scons/versioning/__init__.py index b79fcb6a4..504db8357 100644 --- a/scons/versioning/__init__.py +++ b/scons/versioning/__init__.py @@ -1,32 +1,25 @@ """ Defines build targets for updating the project's version and changelog. """ -from util.targets import PhonyTarget +from util.targets import PhonyTarget, TargetBuilder from versioning.changelog import print_latest_changelog, update_changelog_bugfix, update_changelog_feature, \ update_changelog_main, validate_changelog_bugfix, validate_changelog_feature, validate_changelog_main from versioning.versioning import apply_development_version, increment_development_version, increment_major_version, \ increment_minor_version, increment_patch_version, print_current_version, reset_development_version -TARGETS = [ - # Targets for updating the project's version - PhonyTarget.Builder('increment_development_version').set_function(increment_development_version).build(), - PhonyTarget.Builder('reset_development_version').set_function(reset_development_version).build(), - PhonyTarget.Builder('apply_development_version').set_function(apply_development_version).build(), - PhonyTarget.Builder('increment_patch_version').set_function(increment_patch_version).build(), - PhonyTarget.Builder('increment_minor_version').set_function(increment_minor_version).build(), - PhonyTarget.Builder('increment_major_version').set_function(increment_major_version).build(), - - # Targets for validating changelogs - PhonyTarget.Builder('validate_changelog_bugfix').set_function(validate_changelog_bugfix).build(), - PhonyTarget.Builder('validate_changelog_feature').set_function(validate_changelog_feature).build(), - PhonyTarget.Builder('validate_changelog_main').set_function(validate_changelog_main).build(), - - # Targets for updating the project's changelog - PhonyTarget.Builder('update_changelog_bugfix').set_function(update_changelog_bugfix).build(), - PhonyTarget.Builder('update_changelog_feature').set_function(update_changelog_feature).build(), - PhonyTarget.Builder('update_changelog_main').set_function(update_changelog_main).build(), - - # Targets for printing information about the project - PhonyTarget.Builder('print_version').set_function(print_current_version).build(), - PhonyTarget.Builder('print_latest_changelog').set_function(print_latest_changelog).build() -] +TARGETS = TargetBuilder() \ + .add_phony_target('increment_development_version').set_function(increment_development_version) \ + .add_phony_target('reset_development_version').set_function(reset_development_version) \ + .add_phony_target('apply_development_version').set_function(apply_development_version) \ + .add_phony_target('increment_patch_version').set_function(increment_patch_version) \ + .add_phony_target('increment_minor_version').set_function(increment_minor_version) \ + .add_phony_target('increment_major_version').set_function(increment_major_version) \ + .add_phony_target('validate_changelog_bugfix').set_function(validate_changelog_bugfix) \ + .add_phony_target('validate_changelog_feature').set_function(validate_changelog_feature) \ + .add_phony_target('validate_changelog_main').set_function(validate_changelog_main) \ + .add_phony_target('update_changelog_bugfix').set_function(update_changelog_bugfix) \ + .add_phony_target('update_changelog_feature').set_function(update_changelog_feature) \ + .add_phony_target('update_changelog_main').set_function(update_changelog_main) \ + .add_phony_target('print_version').set_function(print_current_version) \ + .add_phony_target('print_latest_changelog').set_function(print_latest_changelog) \ + .build() From 82c93a5b311cfd48f34a85ffe314fde2638888f9 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 23 Nov 2024 01:01:03 +0100 Subject: [PATCH 030/114] Split file code_style.py into files code_style_cpp.py, code_style_python.py, code_style_yaml.py and code_style_md.py. --- scons/code_style.py | 207 ------------------------------------- scons/code_style_cpp.py | 61 +++++++++++ scons/code_style_md.py | 49 +++++++++ scons/code_style_python.py | 75 ++++++++++++++ scons/code_style_yaml.py | 52 ++++++++++ scons/sconstruct.py | 6 +- 6 files changed, 241 insertions(+), 209 deletions(-) delete mode 100644 scons/code_style.py create mode 100644 scons/code_style_cpp.py create mode 100644 scons/code_style_md.py create mode 100644 scons/code_style_python.py create mode 100644 scons/code_style_yaml.py diff --git a/scons/code_style.py b/scons/code_style.py deleted file mode 100644 index 39c18a7cc..000000000 --- a/scons/code_style.py +++ /dev/null @@ -1,207 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Provides utility functions for checking and enforcing code style definitions. -""" -from glob import glob -from os import path - -from modules import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE -from util.run import Program - -MD_DIRS = [('.', False), (DOC_MODULE.root_dir, True), (PYTHON_MODULE.root_dir, True)] - -YAML_DIRS = [('.', False), ('.github', True)] - - -class Yapf(Program): - """ - Allows to run the external program "yapf". - """ - - def __init__(self, directory: str, enforce_changes: bool = False): - """ - :param directory: The path to the directory, the program should be applied to - :param enforce_changes: True, if changes should be applied to files, False otherwise - """ - super().__init__('yapf', '-r', '-p', '--style=.style.yapf', '--exclude', '**/build/*.py', - '-i' if enforce_changes else '--diff', directory) - - -class Isort(Program): - """ - Allows to run the external program "isort". - """ - - def __init__(self, directory: str, enforce_changes: bool = False): - """ - :param directory: The path to the directory, the program should be applied to - :param enforce_changes: True, if changes should be applied to files, False otherwise - """ - super().__init__('isort', directory, '--settings-path', '.', '--virtual-env', 'venv', '--skip-gitignore') - self.add_conditional_arguments(not enforce_changes, '--check') - - -class Pylint(Program): - """ - Allows to run the external program "pylint". - """ - - def __init__(self, directory: str): - """ - :param directory: The path to the directory, the program should be applied to - """ - super().__init__('pylint', directory, '--jobs=0', '--recursive=y', '--ignore=build', '--rcfile=.pylintrc', - '--score=n') - - -class ClangFormat(Program): - """ - Allows to run the external program "clang-format". - """ - - def __init__(self, directory: str, enforce_changes: bool = False): - """ - :param directory: The path to the directory, the program should be applied to - :param enforce_changes: True, if changes should be applied to files, False otherwise - """ - super().__init__('clang-format', '--style=file') - self.add_conditional_arguments(enforce_changes, '-i') - self.add_conditional_arguments(not enforce_changes, '--dry-run', '--Werror') - self.add_arguments(*glob(path.join(directory, '**', '*.hpp'), recursive=True)) - self.add_arguments(*glob(path.join(directory, '**', '*.cpp'), recursive=True)) - - -class Cpplint(Program): - """ - Allows to run the external program "cpplint". - """ - - def __init__(self, directory: str): - """ - :param directory: The path to the directory, the program should be applied to - """ - super().__init__('cpplint', directory, '--quiet', '--recursive') - - -class Mdformat(Program): - """ - Allows to run the external program "mdformat". - """ - - def __init__(self, directory: str, recursive: bool = False, enforce_changes: bool = False): - """ - :param directory: The path to the directory, the program should be applied to - :param recursive: True, if the program should be applied to subdirectories, False otherwise - :param enforce_changes: True, if changes should be applied to files, False otherwise - """ - super().__init__('mdformat', '--number', '--wrap', 'no', '--end-of-line', 'lf') - self.add_conditional_arguments(not enforce_changes, '--check') - suffix_md = '*.md' - glob_path = path.join(directory, '**', '**', suffix_md) if recursive else path.join(directory, suffix_md) - self.add_arguments(*glob(glob_path, recursive=recursive)) - self.add_dependencies('mdformat-myst') - - -class Yamlfix(Program): - """ - Allows to run the external program "yamlfix". - """ - - def __init__(self, directory: str, recursive: bool = False, enforce_changes: bool = False): - """ - :param directory: The path to the directory, the program should be applied to - :param recursive: True, if the program should be applied to subdirectories, False otherwise - :param enforce_changes: True, if changes should be applied to files, False otherwise - """ - super().__init__('yamlfix', '--config-file', '.yamlfix.toml') - self.add_conditional_arguments(not enforce_changes, '--check') - glob_path = path.join(directory, '**', '*') if recursive else path.join(directory, '*') - glob_path_hidden = path.join(directory, '**', '.*') if recursive else path.join(directory, '.*') - yaml_files = [ - file for file in glob(glob_path) + glob(glob_path_hidden) - if path.basename(file).endswith('.yml') or path.basename(file).endswith('.yaml') - ] - self.add_arguments(yaml_files) - self.print_arguments(True) - - -def check_python_code_style(**_): - """ - Checks if the Python source files adhere to the code style definitions. If this is not the case, an error is raised. - """ - for module in [BUILD_MODULE, PYTHON_MODULE]: - directory = module.root_dir - print('Checking Python code style in directory "' + directory + '"...') - Isort(directory).run() - Yapf(directory).run() - Pylint(directory).run() - - -def enforce_python_code_style(**_): - """ - Enforces the Python source files to adhere to the code style definitions. - """ - for module in [BUILD_MODULE, PYTHON_MODULE, DOC_MODULE]: - directory = module.root_dir - print('Formatting Python code in directory "' + directory + '"...') - Isort(directory, enforce_changes=True).run() - Yapf(directory, enforce_changes=True).run() - - -def check_cpp_code_style(**_): - """ - Checks if the C++ source files adhere to the code style definitions. If this is not the case, an error is raised. - """ - root_dir = CPP_MODULE.root_dir - print('Checking C++ code style in directory "' + root_dir + '"...') - ClangFormat(root_dir).run() - - for subproject in CPP_MODULE.find_subprojects(): - for directory in [subproject.include_dir, subproject.src_dir]: - Cpplint(directory).run() - - -def enforce_cpp_code_style(**_): - """ - Enforces the C++ source files to adhere to the code style definitions. - """ - root_dir = CPP_MODULE.root_dir - print('Formatting C++ code in directory "' + root_dir + '"...') - ClangFormat(root_dir, enforce_changes=True).run() - - -def check_md_code_style(**_): - """ - Checks if the Markdown files adhere to the code style definitions. If this is not the case, an error is raised. - """ - for directory, recursive in MD_DIRS: - print('Checking Markdown code style in the directory "' + directory + '"...') - Mdformat(directory, recursive=recursive).run() - - -def enforce_md_code_style(**_): - """ - Enforces the Markdown files to adhere to the code style definitions. - """ - for directory, recursive in MD_DIRS: - print('Formatting Markdown files in the directory "' + directory + '"...') - Mdformat(directory, recursive=recursive, enforce_changes=True).run() - - -def check_yaml_code_style(**_): - """ - Checks if the YAML files adhere to the code style definitions. If this is not the case, an error is raised. - """ - for directory, recursive in YAML_DIRS: - print('Checking YAML files in the directory "' + directory + '"...') - Yamlfix(directory, recursive=recursive).run() - - -def enforce_yaml_code_style(**_): - """ - Enforces the YAML files to adhere to the code style definitions. - """ - for directory, recursive in YAML_DIRS: - print('Formatting YAML files in the directory "' + directory + '"...') - Yamlfix(directory, recursive=recursive, enforce_changes=True).run() diff --git a/scons/code_style_cpp.py b/scons/code_style_cpp.py new file mode 100644 index 000000000..55da571fb --- /dev/null +++ b/scons/code_style_cpp.py @@ -0,0 +1,61 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for checking and enforcing code style definitions. +""" +from glob import glob +from os import path + +from modules import CPP_MODULE +from util.run import Program + + +class ClangFormat(Program): + """ + Allows to run the external program "clang-format". + """ + + def __init__(self, directory: str, enforce_changes: bool = False): + """ + :param directory: The path to the directory, the program should be applied to + :param enforce_changes: True, if changes should be applied to files, False otherwise + """ + super().__init__('clang-format', '--style=file') + self.add_conditional_arguments(enforce_changes, '-i') + self.add_conditional_arguments(not enforce_changes, '--dry-run', '--Werror') + self.add_arguments(*glob(path.join(directory, '**', '*.hpp'), recursive=True)) + self.add_arguments(*glob(path.join(directory, '**', '*.cpp'), recursive=True)) + + +class Cpplint(Program): + """ + Allows to run the external program "cpplint". + """ + + def __init__(self, directory: str): + """ + :param directory: The path to the directory, the program should be applied to + """ + super().__init__('cpplint', directory, '--quiet', '--recursive') + + +def check_cpp_code_style(**_): + """ + Checks if the C++ source files adhere to the code style definitions. If this is not the case, an error is raised. + """ + root_dir = CPP_MODULE.root_dir + print('Checking C++ code style in directory "' + root_dir + '"...') + ClangFormat(root_dir).run() + + for subproject in CPP_MODULE.find_subprojects(): + for directory in [subproject.include_dir, subproject.src_dir]: + Cpplint(directory).run() + + +def enforce_cpp_code_style(**_): + """ + Enforces the C++ source files to adhere to the code style definitions. + """ + root_dir = CPP_MODULE.root_dir + print('Formatting C++ code in directory "' + root_dir + '"...') + ClangFormat(root_dir, enforce_changes=True).run() diff --git a/scons/code_style_md.py b/scons/code_style_md.py new file mode 100644 index 000000000..984d4f7c3 --- /dev/null +++ b/scons/code_style_md.py @@ -0,0 +1,49 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for checking and enforcing code style definitions. +""" +from glob import glob +from os import path + +from modules import DOC_MODULE, PYTHON_MODULE +from util.run import Program + +MD_DIRS = [('.', False), (DOC_MODULE.root_dir, True), (PYTHON_MODULE.root_dir, True)] + + +class Mdformat(Program): + """ + Allows to run the external program "mdformat". + """ + + def __init__(self, directory: str, recursive: bool = False, enforce_changes: bool = False): + """ + :param directory: The path to the directory, the program should be applied to + :param recursive: True, if the program should be applied to subdirectories, False otherwise + :param enforce_changes: True, if changes should be applied to files, False otherwise + """ + super().__init__('mdformat', '--number', '--wrap', 'no', '--end-of-line', 'lf') + self.add_conditional_arguments(not enforce_changes, '--check') + suffix_md = '*.md' + glob_path = path.join(directory, '**', '**', suffix_md) if recursive else path.join(directory, suffix_md) + self.add_arguments(*glob(glob_path, recursive=recursive)) + self.add_dependencies('mdformat-myst') + + +def check_md_code_style(**_): + """ + Checks if the Markdown files adhere to the code style definitions. If this is not the case, an error is raised. + """ + for directory, recursive in MD_DIRS: + print('Checking Markdown code style in the directory "' + directory + '"...') + Mdformat(directory, recursive=recursive).run() + + +def enforce_md_code_style(**_): + """ + Enforces the Markdown files to adhere to the code style definitions. + """ + for directory, recursive in MD_DIRS: + print('Formatting Markdown files in the directory "' + directory + '"...') + Mdformat(directory, recursive=recursive, enforce_changes=True).run() diff --git a/scons/code_style_python.py b/scons/code_style_python.py new file mode 100644 index 000000000..b57e644fb --- /dev/null +++ b/scons/code_style_python.py @@ -0,0 +1,75 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for checking and enforcing code style definitions. +""" +from modules import BUILD_MODULE, DOC_MODULE, PYTHON_MODULE +from util.run import Program + +MD_DIRS = [('.', False), (DOC_MODULE.root_dir, True), (PYTHON_MODULE.root_dir, True)] + +YAML_DIRS = [('.', False), ('.github', True)] + + +class Yapf(Program): + """ + Allows to run the external program "yapf". + """ + + def __init__(self, directory: str, enforce_changes: bool = False): + """ + :param directory: The path to the directory, the program should be applied to + :param enforce_changes: True, if changes should be applied to files, False otherwise + """ + super().__init__('yapf', '-r', '-p', '--style=.style.yapf', '--exclude', '**/build/*.py', + '-i' if enforce_changes else '--diff', directory) + + +class Isort(Program): + """ + Allows to run the external program "isort". + """ + + def __init__(self, directory: str, enforce_changes: bool = False): + """ + :param directory: The path to the directory, the program should be applied to + :param enforce_changes: True, if changes should be applied to files, False otherwise + """ + super().__init__('isort', directory, '--settings-path', '.', '--virtual-env', 'venv', '--skip-gitignore') + self.add_conditional_arguments(not enforce_changes, '--check') + + +class Pylint(Program): + """ + Allows to run the external program "pylint". + """ + + def __init__(self, directory: str): + """ + :param directory: The path to the directory, the program should be applied to + """ + super().__init__('pylint', directory, '--jobs=0', '--recursive=y', '--ignore=build', '--rcfile=.pylintrc', + '--score=n') + + +def check_python_code_style(**_): + """ + Checks if the Python source files adhere to the code style definitions. If this is not the case, an error is raised. + """ + for module in [BUILD_MODULE, PYTHON_MODULE]: + directory = module.root_dir + print('Checking Python code style in directory "' + directory + '"...') + Isort(directory).run() + Yapf(directory).run() + Pylint(directory).run() + + +def enforce_python_code_style(**_): + """ + Enforces the Python source files to adhere to the code style definitions. + """ + for module in [BUILD_MODULE, PYTHON_MODULE, DOC_MODULE]: + directory = module.root_dir + print('Formatting Python code in directory "' + directory + '"...') + Isort(directory, enforce_changes=True).run() + Yapf(directory, enforce_changes=True).run() diff --git a/scons/code_style_yaml.py b/scons/code_style_yaml.py new file mode 100644 index 000000000..93a5c394a --- /dev/null +++ b/scons/code_style_yaml.py @@ -0,0 +1,52 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for checking and enforcing code style definitions. +""" +from glob import glob +from os import path + +from util.run import Program + +YAML_DIRS = [('.', False), ('.github', True)] + + +class Yamlfix(Program): + """ + Allows to run the external program "yamlfix". + """ + + def __init__(self, directory: str, recursive: bool = False, enforce_changes: bool = False): + """ + :param directory: The path to the directory, the program should be applied to + :param recursive: True, if the program should be applied to subdirectories, False otherwise + :param enforce_changes: True, if changes should be applied to files, False otherwise + """ + super().__init__('yamlfix', '--config-file', '.yamlfix.toml') + self.add_conditional_arguments(not enforce_changes, '--check') + glob_path = path.join(directory, '**', '*') if recursive else path.join(directory, '*') + glob_path_hidden = path.join(directory, '**', '.*') if recursive else path.join(directory, '.*') + yaml_files = [ + file for file in glob(glob_path) + glob(glob_path_hidden) + if path.basename(file).endswith('.yml') or path.basename(file).endswith('.yaml') + ] + self.add_arguments(yaml_files) + self.print_arguments(True) + + +def check_yaml_code_style(**_): + """ + Checks if the YAML files adhere to the code style definitions. If this is not the case, an error is raised. + """ + for directory, recursive in YAML_DIRS: + print('Checking YAML files in the directory "' + directory + '"...') + Yamlfix(directory, recursive=recursive).run() + + +def enforce_yaml_code_style(**_): + """ + Enforces the YAML files to adhere to the code style definitions. + """ + for directory, recursive in YAML_DIRS: + print('Formatting YAML files in the directory "' + directory + '"...') + Yamlfix(directory, recursive=recursive, enforce_changes=True).run() diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 552900698..4d7a3df8f 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -7,8 +7,10 @@ from os import path -from code_style import check_cpp_code_style, check_md_code_style, check_python_code_style, check_yaml_code_style, \ - enforce_cpp_code_style, enforce_md_code_style, enforce_python_code_style, enforce_yaml_code_style +from code_style_cpp import check_cpp_code_style, enforce_cpp_code_style +from code_style_md import check_md_code_style, enforce_md_code_style +from code_style_python import check_python_code_style, enforce_python_code_style +from code_style_yaml import check_yaml_code_style, enforce_yaml_code_style from cython import compile_cython, install_cython, setup_cython from dependencies import check_dependency_versions, install_runtime_dependencies from documentation import apidoc_cpp, apidoc_cpp_tocfile, apidoc_python, apidoc_python_tocfile, doc From 85f7607edd88176efe8f4f9f48a165b39ff766a7 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 23 Nov 2024 01:06:26 +0100 Subject: [PATCH 031/114] Rename file modules.py to modules_old.py. --- scons/code_style_cpp.py | 2 +- scons/code_style_md.py | 2 +- scons/code_style_python.py | 2 +- scons/cpp.py | 2 +- scons/cython.py | 2 +- scons/dependencies.py | 2 +- scons/documentation.py | 2 +- scons/{modules.py => modules_old.py} | 0 scons/packaging.py | 2 +- scons/sconstruct.py | 2 +- scons/testing.py | 2 +- 11 files changed, 10 insertions(+), 10 deletions(-) rename scons/{modules.py => modules_old.py} (100%) diff --git a/scons/code_style_cpp.py b/scons/code_style_cpp.py index 55da571fb..db3cd3643 100644 --- a/scons/code_style_cpp.py +++ b/scons/code_style_cpp.py @@ -6,7 +6,7 @@ from glob import glob from os import path -from modules import CPP_MODULE +from modules_old import CPP_MODULE from util.run import Program diff --git a/scons/code_style_md.py b/scons/code_style_md.py index 984d4f7c3..2e391e8ca 100644 --- a/scons/code_style_md.py +++ b/scons/code_style_md.py @@ -6,7 +6,7 @@ from glob import glob from os import path -from modules import DOC_MODULE, PYTHON_MODULE +from modules_old import DOC_MODULE, PYTHON_MODULE from util.run import Program MD_DIRS = [('.', False), (DOC_MODULE.root_dir, True), (PYTHON_MODULE.root_dir, True)] diff --git a/scons/code_style_python.py b/scons/code_style_python.py index b57e644fb..0303f6e5f 100644 --- a/scons/code_style_python.py +++ b/scons/code_style_python.py @@ -3,7 +3,7 @@ Provides utility functions for checking and enforcing code style definitions. """ -from modules import BUILD_MODULE, DOC_MODULE, PYTHON_MODULE +from modules_old import BUILD_MODULE, DOC_MODULE, PYTHON_MODULE from util.run import Program MD_DIRS = [('.', False), (DOC_MODULE.root_dir, True), (PYTHON_MODULE.root_dir, True)] diff --git a/scons/cpp.py b/scons/cpp.py index 0701da4e2..ee74f4aee 100644 --- a/scons/cpp.py +++ b/scons/cpp.py @@ -5,7 +5,7 @@ """ from compilation.build_options import BuildOptions, EnvBuildOption from compilation.meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup -from modules import CPP_MODULE +from modules_old import CPP_MODULE BUILD_OPTIONS = BuildOptions() \ .add(EnvBuildOption(name='subprojects')) \ diff --git a/scons/cython.py b/scons/cython.py index 446c92722..80afba425 100644 --- a/scons/cython.py +++ b/scons/cython.py @@ -5,7 +5,7 @@ """ from compilation.build_options import BuildOptions, EnvBuildOption from compilation.meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup -from modules import PYTHON_MODULE +from modules_old import PYTHON_MODULE BUILD_OPTIONS = BuildOptions() \ .add(EnvBuildOption(name='subprojects')) diff --git a/scons/dependencies.py b/scons/dependencies.py index 44f19c1f5..96c132953 100644 --- a/scons/dependencies.py +++ b/scons/dependencies.py @@ -8,7 +8,7 @@ from os import path from typing import List -from modules import ALL_MODULES, CPP_MODULE, PYTHON_MODULE, Module +from modules_old import ALL_MODULES, CPP_MODULE, PYTHON_MODULE, Module from util.pip import Package, Pip, RequirementsFile diff --git a/scons/documentation.py b/scons/documentation.py index 5a56873af..d81e8554d 100644 --- a/scons/documentation.py +++ b/scons/documentation.py @@ -6,7 +6,7 @@ from os import environ, makedirs, path, remove from typing import List -from modules import CPP_MODULE, DOC_MODULE, PYTHON_MODULE +from modules_old import CPP_MODULE, DOC_MODULE, PYTHON_MODULE from util.env import set_env from util.io import read_file, write_file from util.run import Program diff --git a/scons/modules.py b/scons/modules_old.py similarity index 100% rename from scons/modules.py rename to scons/modules_old.py diff --git a/scons/packaging.py b/scons/packaging.py index 565e5ce67..87c3ed6d0 100644 --- a/scons/packaging.py +++ b/scons/packaging.py @@ -5,7 +5,7 @@ """ from typing import List -from modules import PYTHON_MODULE +from modules_old import PYTHON_MODULE from util.run import PythonModule diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 4d7a3df8f..96e6d7a26 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -15,7 +15,7 @@ from dependencies import check_dependency_versions, install_runtime_dependencies from documentation import apidoc_cpp, apidoc_cpp_tocfile, apidoc_python, apidoc_python_tocfile, doc from github_actions import check_github_actions, update_github_actions -from modules import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE +from modules_old import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE from packaging import build_python_wheel, install_python_wheels from testing import tests_cpp, tests_python from util.files import DirectorySearch diff --git a/scons/testing.py b/scons/testing.py index 21b62830f..d6516d56f 100644 --- a/scons/testing.py +++ b/scons/testing.py @@ -5,7 +5,7 @@ """ from os import environ, path -from modules import CPP_MODULE, PYTHON_MODULE +from modules_old import CPP_MODULE, PYTHON_MODULE from util.env import get_env_bool from util.run import Program, PythonModule From 8aea45da4ee23db0f5b25dd9d51987cd0368b574 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 23 Nov 2024 01:25:19 +0100 Subject: [PATCH 032/114] Dynamically register modules. --- scons/sconstruct.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 96e6d7a26..fbd47c9c0 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -20,7 +20,7 @@ from testing import tests_cpp, tests_python from util.files import DirectorySearch from util.format import format_iterable -from util.modules import ModuleRegistry +from util.modules import Module, ModuleRegistry from util.reflection import import_source_file from util.targets import Target @@ -82,9 +82,19 @@ def __print_if_clean(environment, message: str): DEFAULT_TARGET = TARGET_NAME_INSTALL_WHEELS +# Register modules... +module_registry = ModuleRegistry() + +for subdirectory in DirectorySearch().set_recursive(True).list(BUILD_MODULE.root_dir): + init_file = path.join(subdirectory, '__init__.py') + + if path.isfile(init_file): + for module in getattr(import_source_file(init_file), 'MODULES', []): + if isinstance(module, Module): + module_registry.register(module) + # Register build targets... env = SConsEnvironment() -module_registry = ModuleRegistry() for subdirectory in DirectorySearch().set_recursive(True).list(BUILD_MODULE.root_dir): init_file = path.join(subdirectory, '__init__.py') From 10d8bc2daa10183a2219c15a820e8a5d659f3601 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 23 Nov 2024 01:29:26 +0100 Subject: [PATCH 033/114] Move classes ClangFormat and CppLint into separate files. --- scons/code_style/cpp/__init__.py | 0 scons/code_style/cpp/clang_format.py | 26 +++++++++++++++++++ scons/code_style/cpp/cpplint.py | 18 ++++++++++++++ scons/code_style_cpp.py | 37 +++------------------------- 4 files changed, 47 insertions(+), 34 deletions(-) create mode 100644 scons/code_style/cpp/__init__.py create mode 100644 scons/code_style/cpp/clang_format.py create mode 100644 scons/code_style/cpp/cpplint.py diff --git a/scons/code_style/cpp/__init__.py b/scons/code_style/cpp/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scons/code_style/cpp/clang_format.py b/scons/code_style/cpp/clang_format.py new file mode 100644 index 000000000..0b1fb1cba --- /dev/null +++ b/scons/code_style/cpp/clang_format.py @@ -0,0 +1,26 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that allow to run the external program "clang-format". +""" +from glob import glob +from os import path + +from util.run import Program + + +class ClangFormat(Program): + """ + Allows to run the external program "clang-format". + """ + + def __init__(self, directory: str, enforce_changes: bool = False): + """ + :param directory: The path to the directory, the program should be applied to + :param enforce_changes: True, if changes should be applied to files, False otherwise + """ + super().__init__('clang-format', '--style=file') + self.add_conditional_arguments(enforce_changes, '-i') + self.add_conditional_arguments(not enforce_changes, '--dry-run', '--Werror') + self.add_arguments(*glob(path.join(directory, '**', '*.hpp'), recursive=True)) + self.add_arguments(*glob(path.join(directory, '**', '*.cpp'), recursive=True)) diff --git a/scons/code_style/cpp/cpplint.py b/scons/code_style/cpp/cpplint.py new file mode 100644 index 000000000..ae9daedc2 --- /dev/null +++ b/scons/code_style/cpp/cpplint.py @@ -0,0 +1,18 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that allow to run the external program "cpplint". +""" +from util.run import Program + + +class CppLint(Program): + """ + Allows to run the external program "cpplint". + """ + + def __init__(self, directory: str): + """ + :param directory: The path to the directory, the program should be applied to + """ + super().__init__('cpplint', directory, '--quiet', '--recursive') diff --git a/scons/code_style_cpp.py b/scons/code_style_cpp.py index db3cd3643..60e0dd709 100644 --- a/scons/code_style_cpp.py +++ b/scons/code_style_cpp.py @@ -3,40 +3,9 @@ Provides utility functions for checking and enforcing code style definitions. """ -from glob import glob -from os import path - +from code_style.cpp.clang_format import ClangFormat +from code_style.cpp.cpplint import CppLint from modules_old import CPP_MODULE -from util.run import Program - - -class ClangFormat(Program): - """ - Allows to run the external program "clang-format". - """ - - def __init__(self, directory: str, enforce_changes: bool = False): - """ - :param directory: The path to the directory, the program should be applied to - :param enforce_changes: True, if changes should be applied to files, False otherwise - """ - super().__init__('clang-format', '--style=file') - self.add_conditional_arguments(enforce_changes, '-i') - self.add_conditional_arguments(not enforce_changes, '--dry-run', '--Werror') - self.add_arguments(*glob(path.join(directory, '**', '*.hpp'), recursive=True)) - self.add_arguments(*glob(path.join(directory, '**', '*.cpp'), recursive=True)) - - -class Cpplint(Program): - """ - Allows to run the external program "cpplint". - """ - - def __init__(self, directory: str): - """ - :param directory: The path to the directory, the program should be applied to - """ - super().__init__('cpplint', directory, '--quiet', '--recursive') def check_cpp_code_style(**_): @@ -49,7 +18,7 @@ def check_cpp_code_style(**_): for subproject in CPP_MODULE.find_subprojects(): for directory in [subproject.include_dir, subproject.src_dir]: - Cpplint(directory).run() + CppLint(directory).run() def enforce_cpp_code_style(**_): From 677104e590be542250dc187c4f61c648756120c2 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 23 Nov 2024 01:33:32 +0100 Subject: [PATCH 034/114] Move class MdFormat into a separate file. --- scons/code_style/markdown/__init__.py | 0 scons/code_style/markdown/mdformat.py | 28 +++++++++++++++++++++++++++ scons/code_style_md.py | 28 +++------------------------ 3 files changed, 31 insertions(+), 25 deletions(-) create mode 100644 scons/code_style/markdown/__init__.py create mode 100644 scons/code_style/markdown/mdformat.py diff --git a/scons/code_style/markdown/__init__.py b/scons/code_style/markdown/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scons/code_style/markdown/mdformat.py b/scons/code_style/markdown/mdformat.py new file mode 100644 index 000000000..b2c47d562 --- /dev/null +++ b/scons/code_style/markdown/mdformat.py @@ -0,0 +1,28 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that allow to run the external program "mdformat". +""" +from glob import glob +from os import path + +from util.run import Program + + +class MdFormat(Program): + """ + Allows to run the external program "mdformat". + """ + + def __init__(self, directory: str, recursive: bool = False, enforce_changes: bool = False): + """ + :param directory: The path to the directory, the program should be applied to + :param recursive: True, if the program should be applied to subdirectories, False otherwise + :param enforce_changes: True, if changes should be applied to files, False otherwise + """ + super().__init__('mdformat', '--number', '--wrap', 'no', '--end-of-line', 'lf') + self.add_conditional_arguments(not enforce_changes, '--check') + suffix_md = '*.md' + glob_path = path.join(directory, '**', '**', suffix_md) if recursive else path.join(directory, suffix_md) + self.add_arguments(*glob(glob_path, recursive=recursive)) + self.add_dependencies('mdformat-myst') diff --git a/scons/code_style_md.py b/scons/code_style_md.py index 2e391e8ca..8ba161557 100644 --- a/scons/code_style_md.py +++ b/scons/code_style_md.py @@ -3,41 +3,19 @@ Provides utility functions for checking and enforcing code style definitions. """ -from glob import glob -from os import path - +from code_style.markdown.mdformat import MdFormat from modules_old import DOC_MODULE, PYTHON_MODULE -from util.run import Program MD_DIRS = [('.', False), (DOC_MODULE.root_dir, True), (PYTHON_MODULE.root_dir, True)] -class Mdformat(Program): - """ - Allows to run the external program "mdformat". - """ - - def __init__(self, directory: str, recursive: bool = False, enforce_changes: bool = False): - """ - :param directory: The path to the directory, the program should be applied to - :param recursive: True, if the program should be applied to subdirectories, False otherwise - :param enforce_changes: True, if changes should be applied to files, False otherwise - """ - super().__init__('mdformat', '--number', '--wrap', 'no', '--end-of-line', 'lf') - self.add_conditional_arguments(not enforce_changes, '--check') - suffix_md = '*.md' - glob_path = path.join(directory, '**', '**', suffix_md) if recursive else path.join(directory, suffix_md) - self.add_arguments(*glob(glob_path, recursive=recursive)) - self.add_dependencies('mdformat-myst') - - def check_md_code_style(**_): """ Checks if the Markdown files adhere to the code style definitions. If this is not the case, an error is raised. """ for directory, recursive in MD_DIRS: print('Checking Markdown code style in the directory "' + directory + '"...') - Mdformat(directory, recursive=recursive).run() + MdFormat(directory, recursive=recursive).run() def enforce_md_code_style(**_): @@ -46,4 +24,4 @@ def enforce_md_code_style(**_): """ for directory, recursive in MD_DIRS: print('Formatting Markdown files in the directory "' + directory + '"...') - Mdformat(directory, recursive=recursive, enforce_changes=True).run() + MdFormat(directory, recursive=recursive, enforce_changes=True).run() From 8feb7bdb8e29b6977ecc5a1410e0e8113f859339 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 23 Nov 2024 01:36:00 +0100 Subject: [PATCH 035/114] Move class YamlFix into a separate file. --- scons/code_style/yaml/__init__.py | 0 scons/code_style/yaml/yamlfix.py | 32 +++++++++++++++++++++++++++++++ scons/code_style_yaml.py | 32 +++---------------------------- 3 files changed, 35 insertions(+), 29 deletions(-) create mode 100644 scons/code_style/yaml/__init__.py create mode 100644 scons/code_style/yaml/yamlfix.py diff --git a/scons/code_style/yaml/__init__.py b/scons/code_style/yaml/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scons/code_style/yaml/yamlfix.py b/scons/code_style/yaml/yamlfix.py new file mode 100644 index 000000000..10025dea0 --- /dev/null +++ b/scons/code_style/yaml/yamlfix.py @@ -0,0 +1,32 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that allow to run the external program "yamlfix". +""" +from glob import glob +from os import path + +from util.run import Program + + +class YamlFix(Program): + """ + Allows to run the external program "yamlfix". + """ + + def __init__(self, directory: str, recursive: bool = False, enforce_changes: bool = False): + """ + :param directory: The path to the directory, the program should be applied to + :param recursive: True, if the program should be applied to subdirectories, False otherwise + :param enforce_changes: True, if changes should be applied to files, False otherwise + """ + super().__init__('yamlfix', '--config-file', '.yamlfix.toml') + self.add_conditional_arguments(not enforce_changes, '--check') + glob_path = path.join(directory, '**', '*') if recursive else path.join(directory, '*') + glob_path_hidden = path.join(directory, '**', '.*') if recursive else path.join(directory, '.*') + yaml_files = [ + file for file in glob(glob_path) + glob(glob_path_hidden) + if path.basename(file).endswith('.yml') or path.basename(file).endswith('.yaml') + ] + self.add_arguments(yaml_files) + self.print_arguments(True) diff --git a/scons/code_style_yaml.py b/scons/code_style_yaml.py index 93a5c394a..7ddf7cc90 100644 --- a/scons/code_style_yaml.py +++ b/scons/code_style_yaml.py @@ -3,44 +3,18 @@ Provides utility functions for checking and enforcing code style definitions. """ -from glob import glob -from os import path - -from util.run import Program +from code_style.yaml.yamlfix import YamlFix YAML_DIRS = [('.', False), ('.github', True)] -class Yamlfix(Program): - """ - Allows to run the external program "yamlfix". - """ - - def __init__(self, directory: str, recursive: bool = False, enforce_changes: bool = False): - """ - :param directory: The path to the directory, the program should be applied to - :param recursive: True, if the program should be applied to subdirectories, False otherwise - :param enforce_changes: True, if changes should be applied to files, False otherwise - """ - super().__init__('yamlfix', '--config-file', '.yamlfix.toml') - self.add_conditional_arguments(not enforce_changes, '--check') - glob_path = path.join(directory, '**', '*') if recursive else path.join(directory, '*') - glob_path_hidden = path.join(directory, '**', '.*') if recursive else path.join(directory, '.*') - yaml_files = [ - file for file in glob(glob_path) + glob(glob_path_hidden) - if path.basename(file).endswith('.yml') or path.basename(file).endswith('.yaml') - ] - self.add_arguments(yaml_files) - self.print_arguments(True) - - def check_yaml_code_style(**_): """ Checks if the YAML files adhere to the code style definitions. If this is not the case, an error is raised. """ for directory, recursive in YAML_DIRS: print('Checking YAML files in the directory "' + directory + '"...') - Yamlfix(directory, recursive=recursive).run() + YamlFix(directory, recursive=recursive).run() def enforce_yaml_code_style(**_): @@ -49,4 +23,4 @@ def enforce_yaml_code_style(**_): """ for directory, recursive in YAML_DIRS: print('Formatting YAML files in the directory "' + directory + '"...') - Yamlfix(directory, recursive=recursive, enforce_changes=True).run() + YamlFix(directory, recursive=recursive, enforce_changes=True).run() From 841d9c20b37713e4d5884f9c2b949b5f070dd9d6 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 23 Nov 2024 01:41:02 +0100 Subject: [PATCH 036/114] Move classes Yapf, ISort and PyLint into separate files. --- scons/code_style/python/__init__.py | 0 scons/code_style/python/isort.py | 20 +++++++++++ scons/code_style/python/pylint.py | 19 ++++++++++ scons/code_style/python/yapf.py | 20 +++++++++++ scons/code_style_python.py | 55 ++++------------------------- 5 files changed, 65 insertions(+), 49 deletions(-) create mode 100644 scons/code_style/python/__init__.py create mode 100644 scons/code_style/python/isort.py create mode 100644 scons/code_style/python/pylint.py create mode 100644 scons/code_style/python/yapf.py diff --git a/scons/code_style/python/__init__.py b/scons/code_style/python/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scons/code_style/python/isort.py b/scons/code_style/python/isort.py new file mode 100644 index 000000000..f63f0949e --- /dev/null +++ b/scons/code_style/python/isort.py @@ -0,0 +1,20 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that allow to run the external program "isort". +""" +from util.run import Program + + +class ISort(Program): + """ + Allows to run the external program "isort". + """ + + def __init__(self, directory: str, enforce_changes: bool = False): + """ + :param directory: The path to the directory, the program should be applied to + :param enforce_changes: True, if changes should be applied to files, False otherwise + """ + super().__init__('isort', directory, '--settings-path', '.', '--virtual-env', 'venv', '--skip-gitignore') + self.add_conditional_arguments(not enforce_changes, '--check') diff --git a/scons/code_style/python/pylint.py b/scons/code_style/python/pylint.py new file mode 100644 index 000000000..23d6763c0 --- /dev/null +++ b/scons/code_style/python/pylint.py @@ -0,0 +1,19 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that allow to run the external program "isort". +""" +from util.run import Program + + +class PyLint(Program): + """ + Allows to run the external program "pylint". + """ + + def __init__(self, directory: str): + """ + :param directory: The path to the directory, the program should be applied to + """ + super().__init__('pylint', directory, '--jobs=0', '--recursive=y', '--ignore=build', '--rcfile=.pylintrc', + '--score=n') diff --git a/scons/code_style/python/yapf.py b/scons/code_style/python/yapf.py new file mode 100644 index 000000000..8be2bf7e0 --- /dev/null +++ b/scons/code_style/python/yapf.py @@ -0,0 +1,20 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that allow to run the external program "yapf". +""" +from util.run import Program + + +class Yapf(Program): + """ + Allows to run the external program "yapf". + """ + + def __init__(self, directory: str, enforce_changes: bool = False): + """ + :param directory: The path to the directory, the program should be applied to + :param enforce_changes: True, if changes should be applied to files, False otherwise + """ + super().__init__('yapf', '-r', '-p', '--style=.style.yapf', '--exclude', '**/build/*.py', + '-i' if enforce_changes else '--diff', directory) diff --git a/scons/code_style_python.py b/scons/code_style_python.py index 0303f6e5f..66bad6436 100644 --- a/scons/code_style_python.py +++ b/scons/code_style_python.py @@ -3,53 +3,10 @@ Provides utility functions for checking and enforcing code style definitions. """ +from code_style.python.isort import ISort +from code_style.python.pylint import PyLint +from code_style.python.yapf import Yapf from modules_old import BUILD_MODULE, DOC_MODULE, PYTHON_MODULE -from util.run import Program - -MD_DIRS = [('.', False), (DOC_MODULE.root_dir, True), (PYTHON_MODULE.root_dir, True)] - -YAML_DIRS = [('.', False), ('.github', True)] - - -class Yapf(Program): - """ - Allows to run the external program "yapf". - """ - - def __init__(self, directory: str, enforce_changes: bool = False): - """ - :param directory: The path to the directory, the program should be applied to - :param enforce_changes: True, if changes should be applied to files, False otherwise - """ - super().__init__('yapf', '-r', '-p', '--style=.style.yapf', '--exclude', '**/build/*.py', - '-i' if enforce_changes else '--diff', directory) - - -class Isort(Program): - """ - Allows to run the external program "isort". - """ - - def __init__(self, directory: str, enforce_changes: bool = False): - """ - :param directory: The path to the directory, the program should be applied to - :param enforce_changes: True, if changes should be applied to files, False otherwise - """ - super().__init__('isort', directory, '--settings-path', '.', '--virtual-env', 'venv', '--skip-gitignore') - self.add_conditional_arguments(not enforce_changes, '--check') - - -class Pylint(Program): - """ - Allows to run the external program "pylint". - """ - - def __init__(self, directory: str): - """ - :param directory: The path to the directory, the program should be applied to - """ - super().__init__('pylint', directory, '--jobs=0', '--recursive=y', '--ignore=build', '--rcfile=.pylintrc', - '--score=n') def check_python_code_style(**_): @@ -59,9 +16,9 @@ def check_python_code_style(**_): for module in [BUILD_MODULE, PYTHON_MODULE]: directory = module.root_dir print('Checking Python code style in directory "' + directory + '"...') - Isort(directory).run() + ISort(directory).run() Yapf(directory).run() - Pylint(directory).run() + PyLint(directory).run() def enforce_python_code_style(**_): @@ -71,5 +28,5 @@ def enforce_python_code_style(**_): for module in [BUILD_MODULE, PYTHON_MODULE, DOC_MODULE]: directory = module.root_dir print('Formatting Python code in directory "' + directory + '"...') - Isort(directory, enforce_changes=True).run() + ISort(directory, enforce_changes=True).run() Yapf(directory, enforce_changes=True).run() From 8cccf144a4c845417719f337845b6d9797c1a1a3 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sun, 24 Nov 2024 21:25:54 +0100 Subject: [PATCH 037/114] Allow to set multiple functions or runnables via a PhonyTarget.Builder. --- scons/util/targets.py | 28 ++++++++++++++-------------- scons/versioning/__init__.py | 28 ++++++++++++++-------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/scons/util/targets.py b/scons/util/targets.py index 85f5c32c2..6bba31676 100644 --- a/scons/util/targets.py +++ b/scons/util/targets.py @@ -78,37 +78,37 @@ def __init__(self, target_builder: 'TargetBuilder', name: str): """ self.target_builder = target_builder self.name = name - self.function = None - self.runnable = None + self.functions = [] + self.runnables = [] - def set_function(self, function: 'PhonyTarget.Function') -> 'TargetBuilder': + def set_functions(self, *functions: 'PhonyTarget.Function') -> 'TargetBuilder': """ - Sets a function to be run by the target. + Sets one or several functions to be run by the target. - :param function: The function to be run + :param functions: The functions to be set :return: The `TargetBuilder`, this builder has been created from """ - self.function = function + self.functions = list(functions) return self.target_builder - def set_runnable(self, runnable: 'PhonyTarget.Runnable') -> 'TargetBuilder': + def set_runnables(self, *runnables: 'PhonyTarget.Runnable') -> 'TargetBuilder': """ - Sets a runnable to be run by the target. + Sets one or several `Runnable` objects to be run by the target. - :param runnable: The runnable to be run + :param runnables: The `Runnable` objects to be set :return: The `TargetBuilder`, this builder has been created from """ - self.runnable = runnable + self.runnables = list(runnables) return self.target_builder def build(self) -> Target: def action(module_registry: ModuleRegistry): - if self.function: - self.function() + for function in self.functions: + function() - if self.runnable: - self.runnable.run(self.target_builder.build_unit, module_registry) + for runnable in self.runnables: + runnable.run(self.target_builder.build_unit, module_registry) return PhonyTarget(self.name, action) diff --git a/scons/versioning/__init__.py b/scons/versioning/__init__.py index 504db8357..22abce0c2 100644 --- a/scons/versioning/__init__.py +++ b/scons/versioning/__init__.py @@ -8,18 +8,18 @@ increment_minor_version, increment_patch_version, print_current_version, reset_development_version TARGETS = TargetBuilder() \ - .add_phony_target('increment_development_version').set_function(increment_development_version) \ - .add_phony_target('reset_development_version').set_function(reset_development_version) \ - .add_phony_target('apply_development_version').set_function(apply_development_version) \ - .add_phony_target('increment_patch_version').set_function(increment_patch_version) \ - .add_phony_target('increment_minor_version').set_function(increment_minor_version) \ - .add_phony_target('increment_major_version').set_function(increment_major_version) \ - .add_phony_target('validate_changelog_bugfix').set_function(validate_changelog_bugfix) \ - .add_phony_target('validate_changelog_feature').set_function(validate_changelog_feature) \ - .add_phony_target('validate_changelog_main').set_function(validate_changelog_main) \ - .add_phony_target('update_changelog_bugfix').set_function(update_changelog_bugfix) \ - .add_phony_target('update_changelog_feature').set_function(update_changelog_feature) \ - .add_phony_target('update_changelog_main').set_function(update_changelog_main) \ - .add_phony_target('print_version').set_function(print_current_version) \ - .add_phony_target('print_latest_changelog').set_function(print_latest_changelog) \ + .add_phony_target('increment_development_version').set_functions(increment_development_version) \ + .add_phony_target('reset_development_version').set_functions(reset_development_version) \ + .add_phony_target('apply_development_version').set_functions(apply_development_version) \ + .add_phony_target('increment_patch_version').set_functions(increment_patch_version) \ + .add_phony_target('increment_minor_version').set_functions(increment_minor_version) \ + .add_phony_target('increment_major_version').set_functions(increment_major_version) \ + .add_phony_target('validate_changelog_bugfix').set_functions(validate_changelog_bugfix) \ + .add_phony_target('validate_changelog_feature').set_functions(validate_changelog_feature) \ + .add_phony_target('validate_changelog_main').set_functions(validate_changelog_main) \ + .add_phony_target('update_changelog_bugfix').set_functions(update_changelog_bugfix) \ + .add_phony_target('update_changelog_feature').set_functions(update_changelog_feature) \ + .add_phony_target('update_changelog_main').set_functions(update_changelog_main) \ + .add_phony_target('print_version').set_functions(print_current_version) \ + .add_phony_target('print_latest_changelog').set_functions(print_latest_changelog) \ .build() From 24daa60b511535adc1e276d4eba7ea4bce2e9947 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Mon, 25 Nov 2024 01:06:03 +0100 Subject: [PATCH 038/114] Allow targets to depend on others. --- scons/sconstruct.py | 20 +++++----- scons/util/targets.py | 91 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 93 insertions(+), 18 deletions(-) diff --git a/scons/sconstruct.py b/scons/sconstruct.py index fbd47c9c0..95f940379 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -22,16 +22,15 @@ from util.format import format_iterable from util.modules import Module, ModuleRegistry from util.reflection import import_source_file -from util.targets import Target +from util.targets import Target, TargetRegistry from cpp import compile_cpp, install_cpp, setup_cpp from SCons.Script import COMMAND_LINE_TARGETS -from SCons.Script.SConscript import SConsEnvironment -def __create_phony_target(environment, target, action=None): - return environment.AlwaysBuild(environment.Alias(target, None, action)) +def __create_phony_target(environment, target_name, action=None): + return environment.AlwaysBuild(environment.Alias(target_name, None, action)) def __print_if_clean(environment, message: str): @@ -94,16 +93,19 @@ def __print_if_clean(environment, message: str): module_registry.register(module) # Register build targets... -env = SConsEnvironment() +target_registry = TargetRegistry(module_registry) +env = target_registry.environment for subdirectory in DirectorySearch().set_recursive(True).list(BUILD_MODULE.root_dir): init_file = path.join(subdirectory, '__init__.py') if path.isfile(init_file): - for build_target in getattr(import_source_file(init_file), 'TARGETS', []): - if isinstance(build_target, Target): - build_target.register(env, module_registry) - VALID_TARGETS.add(build_target.name) + for target in getattr(import_source_file(init_file), 'TARGETS', []): + if isinstance(target, Target): + target_registry.add_target(target) + VALID_TARGETS.add(target.name) + +target_registry.register() # Raise an error if any invalid targets are given... invalid_targets = [target for target in COMMAND_LINE_TARGETS if target not in VALID_TARGETS] diff --git a/scons/util/targets.py b/scons/util/targets.py index 6bba31676..10eb717af 100644 --- a/scons/util/targets.py +++ b/scons/util/targets.py @@ -3,8 +3,10 @@ Provides base classes for defining individual targets of the build process. """ +import sys + from abc import ABC, abstractmethod -from typing import Callable, List +from typing import Any, Callable, List, Set from util.modules import ModuleRegistry from util.units import BuildUnit @@ -22,6 +24,19 @@ class Builder(ABC): An abstract base class for all builders that allow to configure and create targets. """ + def __init__(self): + self.dependencies = set() + + def depends_on(self, *target_names: str) -> 'Target.Builder': + """ + Adds on or several targets, this target should depend on. + + :param target_names: The names of the targets, this target should depend on + :return: The `Target.Builder` itself + """ + self.dependencies.update(target_names) + return self + @abstractmethod def build(self) -> 'Target': """ @@ -30,19 +45,22 @@ def build(self) -> 'Target': :return: The target that has been created """ - def __init__(self, name: str): + def __init__(self, name: str, dependencies: Set[str]): """ - :param name: The name of the target + :param name: The name of the target + :param dependencies: The name of the targets, this target depends on """ self.name = name + self.dependencies = dependencies @abstractmethod - def register(self, environment: Environment, module_registry: ModuleRegistry): + def register(self, environment: Environment, module_registry: ModuleRegistry) -> Any: """ Must be implemented by subclasses in order to register the target. :param environment: The environment, the target should be registered at :param module_registry: The `ModuleRegistry` that can be used by the target for looking up modules + :return: The scons target that has been created """ @@ -76,11 +94,20 @@ def __init__(self, target_builder: 'TargetBuilder', name: str): :param target_builder: The `TargetBuilder`, this builder has been created from :param name: The name of the target """ + super().__init__() self.target_builder = target_builder self.name = name self.functions = [] self.runnables = [] + def nop(self) -> 'TargetBuilder': + """ + Instructs the target to not execute any action. + + :return: The `TargetBuilder`, this builder has been created from + """ + return self.target_builder + def set_functions(self, *functions: 'PhonyTarget.Function') -> 'TargetBuilder': """ Sets one or several functions to be run by the target. @@ -110,18 +137,19 @@ def action(module_registry: ModuleRegistry): for runnable in self.runnables: runnable.run(self.target_builder.build_unit, module_registry) - return PhonyTarget(self.name, action) + return PhonyTarget(self.name, self.dependencies, action) - def __init__(self, name: str, action: Callable[[ModuleRegistry], None]): + def __init__(self, name: str, dependencies: Set[str], action: Callable[[ModuleRegistry], None]): """ :param name: The name of the target :param action: The action to be executed by the target """ - super().__init__(name) + super().__init__(name, dependencies) self.action = action - def register(self, environment: Environment, module_registry: ModuleRegistry): - environment.AlwaysBuild(environment.Alias(self.name, None, action=lambda **_: self.action(module_registry))) + def register(self, environment: Environment, module_registry: ModuleRegistry) -> Any: + return environment.AlwaysBuild( + environment.Alias(self.name, None, action=lambda **_: self.action(module_registry))) class TargetBuilder: @@ -154,3 +182,48 @@ def build(self) -> List[Target]: :return: A list that stores the targets that have been created """ return [target_builder.build() for target_builder in self.target_builders] + + +class TargetRegistry: + """ + Allows to register targets. + """ + + def __init__(self, module_registry: ModuleRegistry): + """ + :param module_registry: The `ModuleRegistry` that should be used by targets for looking up modules + """ + self.environment = Environment() + self.module_registry = module_registry + self.targets_by_name = {} + + def add_target(self, target: Target): + """ + Adds a new target to be registered. + + :param target: The target to be added + """ + self.targets_by_name[target.name] = target + + def register(self): + """ + Registers all targets that have previously been added. + """ + scons_targets_by_name = {} + + for target_name, target in self.targets_by_name.items(): + scons_targets_by_name[target_name] = target.register(self.environment, self.module_registry) + + for target_name, target in self.targets_by_name.items(): + scons_target = scons_targets_by_name[target_name] + scons_dependencies = [] + + for dependency in target.dependencies: + try: + scons_dependencies.append(scons_targets_by_name[dependency]) + except KeyError: + print('Dependency "' + dependency + '" of target "' + target_name + '" has not been registered') + sys.exit(-1) + + if scons_dependencies: + self.environment.Depends(scons_target, scons_dependencies) From 6e52e598eba82fae0dfdba9c8c76e9e8a79f9cf1 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 23 Nov 2024 19:26:18 +0100 Subject: [PATCH 039/114] Dynamically register targets and modules for checking and enforcing the code style. --- CPPLINT.cfg => cpp/.cpplint.cfg | 2 + cpp/subprojects/boosting/test/.cpplint.cfg | 1 + cpp/subprojects/common/test/.cpplint.cfg | 1 + cpp/subprojects/seco/test/.cpplint.cfg | 1 + doc/conf.py | 15 +++-- doc/developer_guide/coding_standards.md | 2 +- scons/code_style/__init__.py | 20 ++++++ .../code_style/cpp/.clang-format | 0 scons/code_style/cpp/__init__.py | 17 +++++ scons/code_style/cpp/clang_format.py | 14 ++-- scons/code_style/cpp/cpplint.py | 10 ++- scons/code_style/cpp/requirements.txt | 2 + scons/code_style/cpp/targets.py | 37 +++++++++++ scons/code_style/markdown/__init__.py | 17 +++++ scons/code_style/markdown/mdformat.py | 16 ++--- scons/code_style/markdown/requirements.txt | 2 + scons/code_style/markdown/targets.py | 35 ++++++++++ .../code_style/python/.isort.cfg | 1 - .../code_style/python/.pylintrc | 3 +- .../code_style/python/.style.yapf | 0 scons/code_style/python/__init__.py | 18 ++++++ scons/code_style/python/isort.py | 11 +++- scons/code_style/python/pylint.py | 14 ++-- scons/code_style/python/requirements.txt | 3 + scons/code_style/python/targets.py | 64 +++++++++++++++++++ scons/code_style/python/yapf.py | 14 ++-- .../code_style/yaml/.yamlfix.toml | 0 scons/code_style/yaml/__init__.py | 17 +++++ scons/code_style/yaml/requirements.txt | 1 + scons/code_style/yaml/targets.py | 35 ++++++++++ scons/code_style/yaml/yamlfix.py | 20 +++--- scons/code_style_cpp.py | 30 --------- scons/code_style_md.py | 27 -------- scons/code_style_python.py | 32 ---------- scons/code_style_yaml.py | 26 -------- scons/modules/__init__.py | 28 ++++++++ scons/sconstruct.py | 46 ++----------- scons/util/requirements.txt | 8 --- 38 files changed, 376 insertions(+), 214 deletions(-) rename CPPLINT.cfg => cpp/.cpplint.cfg (95%) create mode 100644 cpp/subprojects/boosting/test/.cpplint.cfg create mode 100644 cpp/subprojects/common/test/.cpplint.cfg create mode 100644 cpp/subprojects/seco/test/.cpplint.cfg rename .clang-format => scons/code_style/cpp/.clang-format (100%) create mode 100644 scons/code_style/cpp/requirements.txt create mode 100644 scons/code_style/cpp/targets.py create mode 100644 scons/code_style/markdown/requirements.txt create mode 100644 scons/code_style/markdown/targets.py rename .isort.cfg => scons/code_style/python/.isort.cfg (91%) rename .pylintrc => scons/code_style/python/.pylintrc (96%) rename .style.yapf => scons/code_style/python/.style.yapf (100%) create mode 100644 scons/code_style/python/requirements.txt create mode 100644 scons/code_style/python/targets.py rename .yamlfix.toml => scons/code_style/yaml/.yamlfix.toml (100%) create mode 100644 scons/code_style/yaml/requirements.txt create mode 100644 scons/code_style/yaml/targets.py delete mode 100644 scons/code_style_cpp.py delete mode 100644 scons/code_style_md.py delete mode 100644 scons/code_style_python.py delete mode 100644 scons/code_style_yaml.py create mode 100644 scons/modules/__init__.py diff --git a/CPPLINT.cfg b/cpp/.cpplint.cfg similarity index 95% rename from CPPLINT.cfg rename to cpp/.cpplint.cfg index c68d29bd4..732d9bec7 100644 --- a/CPPLINT.cfg +++ b/cpp/.cpplint.cfg @@ -1,3 +1,5 @@ +set noparent + filter=-build/include_subdir filter=-build/include_order filter=-build/include_what_you_use diff --git a/cpp/subprojects/boosting/test/.cpplint.cfg b/cpp/subprojects/boosting/test/.cpplint.cfg new file mode 100644 index 000000000..657274cdd --- /dev/null +++ b/cpp/subprojects/boosting/test/.cpplint.cfg @@ -0,0 +1 @@ +filter=-build/include diff --git a/cpp/subprojects/common/test/.cpplint.cfg b/cpp/subprojects/common/test/.cpplint.cfg new file mode 100644 index 000000000..657274cdd --- /dev/null +++ b/cpp/subprojects/common/test/.cpplint.cfg @@ -0,0 +1 @@ +filter=-build/include diff --git a/cpp/subprojects/seco/test/.cpplint.cfg b/cpp/subprojects/seco/test/.cpplint.cfg new file mode 100644 index 000000000..657274cdd --- /dev/null +++ b/cpp/subprojects/seco/test/.cpplint.cfg @@ -0,0 +1 @@ +filter=-build/include diff --git a/doc/conf.py b/doc/conf.py index 6148a575f..a7652d7ca 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -1,12 +1,13 @@ +""" +Configuration file for the Sphinx documentation builder. + +This file only contains a selection of the most common options. For a full list see the documentation: +https://www.sphinx-doc.org/en/master/usage/configuration.html +""" +# pylint: disable=redefined-builtin,invalid-name from os import listdir from pathlib import Path -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - # -- Path setup -------------------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, @@ -40,7 +41,7 @@ ] # Favicons -favicons = [{"href": 'favicon.svg'}] +favicons = [{'href': 'favicon.svg'}] # Intersphinx configuration intersphinx_mapping = { diff --git a/doc/developer_guide/coding_standards.md b/doc/developer_guide/coding_standards.md index 84203ebd0..dae978cb5 100644 --- a/doc/developer_guide/coding_standards.md +++ b/doc/developer_guide/coding_standards.md @@ -68,7 +68,7 @@ We aim to enforce a consistent code style across the entire project. For this pu - For formatting the C++ code, we use [clang-format](https://clang.llvm.org/docs/ClangFormat.html). The desired C++ code style is defined in the file `.clang-format` in the project's root directory. In addition, [cpplint](https://github.com/cpplint/cpplint) is used for static code analysis. It uses the configuration file `CPPLINT.cfg`. - We use [YAPF](https://github.com/google/yapf) to enforce the Python code style defined in the file `.style.yapf`. In addition, [isort](https://github.com/PyCQA/isort) is used to keep the ordering of imports in Python and Cython source files consistent according to the configuration file `.isort.cfg` and [pylint](https://pylint.org/) is used to check for common issues in the Python code according to the configuration file `.pylintrc`. - For applying a consistent style to Markdown files, including those used for writing the documentation, we use [mdformat](https://github.com/executablebooks/mdformat). -- We apply [yamlfix](https://github.com/lyz-code/yamlfix) to YAML files to enforce the code style defined in the file `.yamlfix.toml`. +- We apply [yamlfix](https://github.com/lyz-code/yamlfix) to YAML files to enforce the code style defined in the file `scons/code_style/yaml/.yamlfix.toml`. If you have modified the project's source code, you can check whether it adheres to our coding standards via the following command: diff --git a/scons/code_style/__init__.py b/scons/code_style/__init__.py index e69de29bb..2b4b8df00 100644 --- a/scons/code_style/__init__.py +++ b/scons/code_style/__init__.py @@ -0,0 +1,20 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Defines targets for checking and enforcing code style definitions. +""" +from code_style.cpp import FORMAT_CPP, TEST_FORMAT_CPP +from code_style.markdown import FORMAT_MARKDOWN, TEST_FORMAT_MARKDOWN +from code_style.python import FORMAT_PYTHON, TEST_FORMAT_PYTHON +from code_style.yaml import FORMAT_YAML, TEST_FORMAT_YAML +from util.targets import TargetBuilder +from util.units import BuildUnit + +TARGETS = TargetBuilder(BuildUnit.by_name('code_style')) \ + .add_phony_target('format') \ + .depends_on(FORMAT_PYTHON, FORMAT_CPP, FORMAT_MARKDOWN, FORMAT_YAML) \ + .nop() \ + .add_phony_target('test_format') \ + .depends_on(TEST_FORMAT_PYTHON, TEST_FORMAT_CPP, TEST_FORMAT_MARKDOWN, TEST_FORMAT_YAML) \ + .nop() \ + .build() diff --git a/.clang-format b/scons/code_style/cpp/.clang-format similarity index 100% rename from .clang-format rename to scons/code_style/cpp/.clang-format diff --git a/scons/code_style/cpp/__init__.py b/scons/code_style/cpp/__init__.py index e69de29bb..6f7900299 100644 --- a/scons/code_style/cpp/__init__.py +++ b/scons/code_style/cpp/__init__.py @@ -0,0 +1,17 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Defines targets for checking and enforcing code style definitions for C++ files. +""" +from code_style.cpp.targets import CheckCppCodeStyle, EnforceCppCodeStyle +from util.targets import PhonyTarget, TargetBuilder +from util.units import BuildUnit + +FORMAT_CPP = 'format_cpp' + +TEST_FORMAT_CPP = 'test_format_cpp' + +TARGETS = TargetBuilder(BuildUnit.by_name('code_style', 'cpp')) \ + .add_phony_target(FORMAT_CPP).set_runnables(EnforceCppCodeStyle()) \ + .add_phony_target(TEST_FORMAT_CPP).set_runnables(CheckCppCodeStyle()) \ + .build() diff --git a/scons/code_style/cpp/clang_format.py b/scons/code_style/cpp/clang_format.py index 0b1fb1cba..00e9cdab6 100644 --- a/scons/code_style/cpp/clang_format.py +++ b/scons/code_style/cpp/clang_format.py @@ -3,10 +3,11 @@ Provides classes that allow to run the external program "clang-format". """ -from glob import glob from os import path +from code_style.modules import CodeModule from util.run import Program +from util.units import BuildUnit class ClangFormat(Program): @@ -14,13 +15,14 @@ class ClangFormat(Program): Allows to run the external program "clang-format". """ - def __init__(self, directory: str, enforce_changes: bool = False): + def __init__(self, build_unit: BuildUnit, module: CodeModule, enforce_changes: bool = False): """ - :param directory: The path to the directory, the program should be applied to + :param build_unit: The build unit from which the program should be run + :param module: The module, the program should be applied to :param enforce_changes: True, if changes should be applied to files, False otherwise """ - super().__init__('clang-format', '--style=file') + super().__init__('clang-format', '--style=file:' + path.join(build_unit.root_directory, '.clang-format')) self.add_conditional_arguments(enforce_changes, '-i') self.add_conditional_arguments(not enforce_changes, '--dry-run', '--Werror') - self.add_arguments(*glob(path.join(directory, '**', '*.hpp'), recursive=True)) - self.add_arguments(*glob(path.join(directory, '**', '*.cpp'), recursive=True)) + self.add_arguments(*module.find_source_files()) + self.set_build_unit(build_unit) diff --git a/scons/code_style/cpp/cpplint.py b/scons/code_style/cpp/cpplint.py index ae9daedc2..43997a279 100644 --- a/scons/code_style/cpp/cpplint.py +++ b/scons/code_style/cpp/cpplint.py @@ -3,7 +3,9 @@ Provides classes that allow to run the external program "cpplint". """ +from code_style.modules import CodeModule from util.run import Program +from util.units import BuildUnit class CppLint(Program): @@ -11,8 +13,10 @@ class CppLint(Program): Allows to run the external program "cpplint". """ - def __init__(self, directory: str): + def __init__(self, build_unit: BuildUnit, module: CodeModule): """ - :param directory: The path to the directory, the program should be applied to + :param build_unit: The build unit from which the program should be run + :param module: The module, the program should be applied to """ - super().__init__('cpplint', directory, '--quiet', '--recursive') + super().__init__('cpplint', '--quiet', '--config=.cpplint.cfg', *module.find_source_files()) + self.set_build_unit(build_unit) diff --git a/scons/code_style/cpp/requirements.txt b/scons/code_style/cpp/requirements.txt new file mode 100644 index 000000000..3f5670a40 --- /dev/null +++ b/scons/code_style/cpp/requirements.txt @@ -0,0 +1,2 @@ +cpplint >= 2.0, < 2.1 +clang-format >= 19.1, < 19.2 diff --git a/scons/code_style/cpp/targets.py b/scons/code_style/cpp/targets.py new file mode 100644 index 000000000..dbaec1464 --- /dev/null +++ b/scons/code_style/cpp/targets.py @@ -0,0 +1,37 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Implements targets for checking and enforcing code style definitions for C++ files. +""" +from code_style.cpp.clang_format import ClangFormat +from code_style.cpp.cpplint import CppLint +from code_style.modules import CodeModule +from util.languages import Language +from util.modules import ModuleRegistry +from util.targets import PhonyTarget +from util.units import BuildUnit + +MODULE_FILTER = CodeModule.Filter(Language.CPP) + + +class CheckCppCodeStyle(PhonyTarget.Runnable): + """ + Checks if C++ source files adhere to the code style definitions. If this is not the case, an error is raised. + """ + + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + for module in modules.lookup(MODULE_FILTER): + print('Checking C++ code style in directory "' + module.root_directory + '"...') + ClangFormat(build_unit, module).run() + CppLint(build_unit, module).run() + + +class EnforceCppCodeStyle(PhonyTarget.Runnable): + """ + Enforces C++ source files to adhere to the code style definitions. + """ + + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + for module in modules.lookup(MODULE_FILTER): + print('Formatting C++ code in directory "' + module.root_directory + '"...') + ClangFormat(build_unit, module, enforce_changes=True).run() diff --git a/scons/code_style/markdown/__init__.py b/scons/code_style/markdown/__init__.py index e69de29bb..e5d546e13 100644 --- a/scons/code_style/markdown/__init__.py +++ b/scons/code_style/markdown/__init__.py @@ -0,0 +1,17 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Defines targets for checking and enforcing code style definitions for Markdown files. +""" +from code_style.markdown.targets import CheckMarkdownCodeStyle, EnforceMarkdownCodeStyle +from util.targets import PhonyTarget, TargetBuilder +from util.units import BuildUnit + +FORMAT_MARKDOWN = 'format_md' + +TEST_FORMAT_MARKDOWN = 'test_format_md' + +TARGETS = TargetBuilder(BuildUnit.by_name('code_style', 'markdown')) \ + .add_phony_target(FORMAT_MARKDOWN).set_runnables(EnforceMarkdownCodeStyle()) \ + .add_phony_target(TEST_FORMAT_MARKDOWN).set_runnables(CheckMarkdownCodeStyle()) \ + .build() diff --git a/scons/code_style/markdown/mdformat.py b/scons/code_style/markdown/mdformat.py index b2c47d562..5ec2c960c 100644 --- a/scons/code_style/markdown/mdformat.py +++ b/scons/code_style/markdown/mdformat.py @@ -3,10 +3,9 @@ Provides classes that allow to run the external program "mdformat". """ -from glob import glob -from os import path - +from code_style.modules import CodeModule from util.run import Program +from util.units import BuildUnit class MdFormat(Program): @@ -14,15 +13,14 @@ class MdFormat(Program): Allows to run the external program "mdformat". """ - def __init__(self, directory: str, recursive: bool = False, enforce_changes: bool = False): + def __init__(self, build_unit: BuildUnit, module: CodeModule, enforce_changes: bool = False): """ - :param directory: The path to the directory, the program should be applied to - :param recursive: True, if the program should be applied to subdirectories, False otherwise + :param build_unit: The build unit from which the program should be run + :param module: The module, the program should be applied to :param enforce_changes: True, if changes should be applied to files, False otherwise """ super().__init__('mdformat', '--number', '--wrap', 'no', '--end-of-line', 'lf') self.add_conditional_arguments(not enforce_changes, '--check') - suffix_md = '*.md' - glob_path = path.join(directory, '**', '**', suffix_md) if recursive else path.join(directory, suffix_md) - self.add_arguments(*glob(glob_path, recursive=recursive)) + self.add_arguments(*module.find_source_files()) + self.set_build_unit(build_unit) self.add_dependencies('mdformat-myst') diff --git a/scons/code_style/markdown/requirements.txt b/scons/code_style/markdown/requirements.txt new file mode 100644 index 000000000..8feb1dcae --- /dev/null +++ b/scons/code_style/markdown/requirements.txt @@ -0,0 +1,2 @@ +mdformat >= 0.7, < 0.8 +mdformat-myst >= 0.2, < 0.3 diff --git a/scons/code_style/markdown/targets.py b/scons/code_style/markdown/targets.py new file mode 100644 index 000000000..0274b37f4 --- /dev/null +++ b/scons/code_style/markdown/targets.py @@ -0,0 +1,35 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Implements targets for checking and enforcing code style definitions for Markdown files. +""" +from code_style.markdown.mdformat import MdFormat +from code_style.modules import CodeModule +from util.languages import Language +from util.modules import ModuleRegistry +from util.targets import PhonyTarget +from util.units import BuildUnit + +MODULE_FILTER = CodeModule.Filter(Language.MARKDOWN) + + +class CheckMarkdownCodeStyle(PhonyTarget.Runnable): + """ + Checks if Markdown files adhere to the code style definitions. If this is not the case, an error is raised. + """ + + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + for module in modules.lookup(MODULE_FILTER): + print('Checking Markdown code style in the directory "' + module.root_directory + '"...') + MdFormat(build_unit, module).run() + + +class EnforceMarkdownCodeStyle(PhonyTarget.Runnable): + """ + Enforces Markdown files to adhere to the code style definitions. + """ + + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + for module in modules.lookup(MODULE_FILTER): + print('Formatting Markdown files in the directory "' + module.root_directory + '"...') + MdFormat(build_unit, module, enforce_changes=True).run() diff --git a/.isort.cfg b/scons/code_style/python/.isort.cfg similarity index 91% rename from .isort.cfg rename to scons/code_style/python/.isort.cfg index e42dcea7d..08d9adbfb 100644 --- a/.isort.cfg +++ b/scons/code_style/python/.isort.cfg @@ -1,5 +1,4 @@ [settings] -supported_extensions=py,pxd,pyx line_length=120 group_by_package=true known_first_party=mlrl diff --git a/.pylintrc b/scons/code_style/python/.pylintrc similarity index 96% rename from .pylintrc rename to scons/code_style/python/.pylintrc index a72297b30..bab33ed6b 100644 --- a/.pylintrc +++ b/scons/code_style/python/.pylintrc @@ -39,7 +39,8 @@ disable=no-name-in-module, too-many-locals, too-many-lines, too-many-public-methods, - too-many-statements + too-many-statements, + wrong-import-order [STRING] diff --git a/.style.yapf b/scons/code_style/python/.style.yapf similarity index 100% rename from .style.yapf rename to scons/code_style/python/.style.yapf diff --git a/scons/code_style/python/__init__.py b/scons/code_style/python/__init__.py index e69de29bb..b30122718 100644 --- a/scons/code_style/python/__init__.py +++ b/scons/code_style/python/__init__.py @@ -0,0 +1,18 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Defines targets for checking and enforcing code style definitions for Python and Cython files. +""" +from code_style.python.targets import CheckCythonCodeStyle, CheckPythonCodeStyle, EnforceCythonCodeStyle, \ + EnforcePythonCodeStyle +from util.targets import PhonyTarget, TargetBuilder +from util.units import BuildUnit + +FORMAT_PYTHON = 'format_python' + +TEST_FORMAT_PYTHON = 'test_format_python' + +TARGETS = TargetBuilder(BuildUnit.by_name('code_style', 'python')) \ + .add_phony_target(FORMAT_PYTHON).set_runnables(EnforcePythonCodeStyle(), EnforceCythonCodeStyle()) \ + .add_phony_target(TEST_FORMAT_PYTHON).set_runnables(CheckPythonCodeStyle(), CheckCythonCodeStyle()) \ + .build() diff --git a/scons/code_style/python/isort.py b/scons/code_style/python/isort.py index f63f0949e..5a6ea7121 100644 --- a/scons/code_style/python/isort.py +++ b/scons/code_style/python/isort.py @@ -3,7 +3,9 @@ Provides classes that allow to run the external program "isort". """ +from code_style.modules import CodeModule from util.run import Program +from util.units import BuildUnit class ISort(Program): @@ -11,10 +13,13 @@ class ISort(Program): Allows to run the external program "isort". """ - def __init__(self, directory: str, enforce_changes: bool = False): + def __init__(self, build_unit: BuildUnit, module: CodeModule, enforce_changes: bool = False): """ - :param directory: The path to the directory, the program should be applied to + :param build_unit: The build unit from which the program should be run + :param module: The module, the program should be applied to :param enforce_changes: True, if changes should be applied to files, False otherwise """ - super().__init__('isort', directory, '--settings-path', '.', '--virtual-env', 'venv', '--skip-gitignore') + super().__init__('isort', '--settings-path', build_unit.root_directory, '--virtual-env', 'venv', + '--skip-gitignore', *module.find_source_files()) self.add_conditional_arguments(not enforce_changes, '--check') + self.set_build_unit(build_unit) diff --git a/scons/code_style/python/pylint.py b/scons/code_style/python/pylint.py index 23d6763c0..55e276799 100644 --- a/scons/code_style/python/pylint.py +++ b/scons/code_style/python/pylint.py @@ -3,7 +3,11 @@ Provides classes that allow to run the external program "isort". """ +from os import path + +from code_style.modules import CodeModule from util.run import Program +from util.units import BuildUnit class PyLint(Program): @@ -11,9 +15,11 @@ class PyLint(Program): Allows to run the external program "pylint". """ - def __init__(self, directory: str): + def __init__(self, build_unit: BuildUnit, module: CodeModule): """ - :param directory: The path to the directory, the program should be applied to + :param build_unit: The build unit from which the program should be run + :param module: The module, the program should be applied to """ - super().__init__('pylint', directory, '--jobs=0', '--recursive=y', '--ignore=build', '--rcfile=.pylintrc', - '--score=n') + super().__init__('pylint', *module.find_source_files(), '--jobs=0', '--ignore=build', + '--rcfile=' + path.join(build_unit.root_directory, '.pylintrc'), '--score=n') + self.set_build_unit(build_unit) diff --git a/scons/code_style/python/requirements.txt b/scons/code_style/python/requirements.txt new file mode 100644 index 000000000..ef0b290c8 --- /dev/null +++ b/scons/code_style/python/requirements.txt @@ -0,0 +1,3 @@ +isort >= 5.13, < 5.14 +pylint >= 3.3, < 3.4 +yapf >= 0.43, < 0.44 diff --git a/scons/code_style/python/targets.py b/scons/code_style/python/targets.py new file mode 100644 index 000000000..cb52c9468 --- /dev/null +++ b/scons/code_style/python/targets.py @@ -0,0 +1,64 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Implements targets for checking and enforcing code style definitions for Python and Cython files. +""" +from code_style.modules import CodeModule +from code_style.python.isort import ISort +from code_style.python.pylint import PyLint +from code_style.python.yapf import Yapf +from util.languages import Language +from util.modules import ModuleRegistry +from util.targets import PhonyTarget +from util.units import BuildUnit + +PYTHON_MODULE_FILTER = CodeModule.Filter(Language.PYTHON) + +CYTHON_MODULE_FILTER = CodeModule.Filter(Language.CYTHON) + + +class CheckPythonCodeStyle(PhonyTarget.Runnable): + """ + Checks if Python source files adhere to the code style definitions. If this is not the case, an error is raised. + """ + + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + for module in modules.lookup(PYTHON_MODULE_FILTER): + print('Checking Python code style in directory "' + module.root_directory + '"...') + ISort(build_unit, module).run() + Yapf(build_unit, module).run() + PyLint(build_unit, module).run() + + +class EnforcePythonCodeStyle(PhonyTarget.Runnable): + """ + Enforces Python source files to adhere to the code style definitions. + """ + + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + for module in modules.lookup(PYTHON_MODULE_FILTER): + print('Formatting Python code in directory "' + module.root_directory + '"...') + ISort(build_unit, module, enforce_changes=True).run() + Yapf(build_unit, module, enforce_changes=True).run() + + +class CheckCythonCodeStyle(PhonyTarget.Runnable): + """ + Checks if Cython source files adhere to the code style definitions. If this is not the case, an error is raised. + """ + + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + for module in modules.lookup(CYTHON_MODULE_FILTER): + print('Checking Cython code style in directory "' + module.root_directory + '"...') + ISort(build_unit, module).run() + + +class EnforceCythonCodeStyle(PhonyTarget.Runnable): + """ + Enforces Cython source files to adhere to the code style definitions. + """ + + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + for module in modules.lookup(CYTHON_MODULE_FILTER): + print('Formatting Cython code in directory "' + module.root_directory + '"...') + ISort(build_unit, module, enforce_changes=True).run() diff --git a/scons/code_style/python/yapf.py b/scons/code_style/python/yapf.py index 8be2bf7e0..92f41ecce 100644 --- a/scons/code_style/python/yapf.py +++ b/scons/code_style/python/yapf.py @@ -3,7 +3,11 @@ Provides classes that allow to run the external program "yapf". """ +from os import path + +from code_style.modules import CodeModule from util.run import Program +from util.units import BuildUnit class Yapf(Program): @@ -11,10 +15,12 @@ class Yapf(Program): Allows to run the external program "yapf". """ - def __init__(self, directory: str, enforce_changes: bool = False): + def __init__(self, build_unit: BuildUnit, module: CodeModule, enforce_changes: bool = False): """ - :param directory: The path to the directory, the program should be applied to + :param build_unit: The build unit from which the program should be run + :param module: The module, the program should be applied to :param enforce_changes: True, if changes should be applied to files, False otherwise """ - super().__init__('yapf', '-r', '-p', '--style=.style.yapf', '--exclude', '**/build/*.py', - '-i' if enforce_changes else '--diff', directory) + super().__init__('yapf', '--parallel', '--style=' + path.join(build_unit.root_directory, '.style.yapf'), + '--in-place' if enforce_changes else '--diff', *module.find_source_files()) + self.set_build_unit(build_unit) diff --git a/.yamlfix.toml b/scons/code_style/yaml/.yamlfix.toml similarity index 100% rename from .yamlfix.toml rename to scons/code_style/yaml/.yamlfix.toml diff --git a/scons/code_style/yaml/__init__.py b/scons/code_style/yaml/__init__.py index e69de29bb..508bc96c2 100644 --- a/scons/code_style/yaml/__init__.py +++ b/scons/code_style/yaml/__init__.py @@ -0,0 +1,17 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Defines targets for checking and enforcing code style definitions for YAML files. +""" +from code_style.yaml.targets import CheckYamlCodeStyle, EnforceYamlCodeStyle +from util.targets import PhonyTarget, TargetBuilder +from util.units import BuildUnit + +FORMAT_YAML = 'format_yaml' + +TEST_FORMAT_YAML = 'test_format_yaml' + +TARGETS = TargetBuilder(BuildUnit.by_name('code_style', 'yaml')) \ + .add_phony_target(FORMAT_YAML).set_runnables(EnforceYamlCodeStyle()) \ + .add_phony_target(TEST_FORMAT_YAML).set_runnables(CheckYamlCodeStyle()) \ + .build() diff --git a/scons/code_style/yaml/requirements.txt b/scons/code_style/yaml/requirements.txt new file mode 100644 index 000000000..70c03b21f --- /dev/null +++ b/scons/code_style/yaml/requirements.txt @@ -0,0 +1 @@ +yamlfix >= 1.17, < 1.18 diff --git a/scons/code_style/yaml/targets.py b/scons/code_style/yaml/targets.py new file mode 100644 index 000000000..af1abc0fb --- /dev/null +++ b/scons/code_style/yaml/targets.py @@ -0,0 +1,35 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Implements targets for checking and enforcing code style definitions for YAML files. +""" +from code_style.modules import CodeModule +from code_style.yaml.yamlfix import YamlFix +from util.languages import Language +from util.modules import ModuleRegistry +from util.targets import PhonyTarget +from util.units import BuildUnit + +MODULE_FILTER = CodeModule.Filter(Language.YAML) + + +class CheckYamlCodeStyle(PhonyTarget.Runnable): + """ + Checks if YAML files adhere to the code style definitions. If this is not the case, an error is raised. + """ + + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + for module in modules.lookup(MODULE_FILTER): + print('Checking YAML files in the directory "' + module.root_directory + '"...') + YamlFix(build_unit, module).run() + + +class EnforceYamlCodeStyle(PhonyTarget.Runnable): + """ + Enforces YAML files to adhere to the code style definitions. + """ + + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + for module in modules.lookup(MODULE_FILTER): + print('Formatting YAML files in the directory "' + module.root_directory + '"...') + YamlFix(build_unit, module, enforce_changes=True).run() diff --git a/scons/code_style/yaml/yamlfix.py b/scons/code_style/yaml/yamlfix.py index 10025dea0..e437fe1e2 100644 --- a/scons/code_style/yaml/yamlfix.py +++ b/scons/code_style/yaml/yamlfix.py @@ -3,10 +3,11 @@ Provides classes that allow to run the external program "yamlfix". """ -from glob import glob from os import path +from code_style.modules import CodeModule from util.run import Program +from util.units import BuildUnit class YamlFix(Program): @@ -14,19 +15,14 @@ class YamlFix(Program): Allows to run the external program "yamlfix". """ - def __init__(self, directory: str, recursive: bool = False, enforce_changes: bool = False): + def __init__(self, build_unit: BuildUnit, module: CodeModule, enforce_changes: bool = False): """ - :param directory: The path to the directory, the program should be applied to - :param recursive: True, if the program should be applied to subdirectories, False otherwise + :param build_unit: The build unit from which the program should be run + :param module: The module, the program should be applied to :param enforce_changes: True, if changes should be applied to files, False otherwise """ - super().__init__('yamlfix', '--config-file', '.yamlfix.toml') + super().__init__('yamlfix', '--config-file', path.join(build_unit.root_directory, '.yamlfix.toml')) self.add_conditional_arguments(not enforce_changes, '--check') - glob_path = path.join(directory, '**', '*') if recursive else path.join(directory, '*') - glob_path_hidden = path.join(directory, '**', '.*') if recursive else path.join(directory, '.*') - yaml_files = [ - file for file in glob(glob_path) + glob(glob_path_hidden) - if path.basename(file).endswith('.yml') or path.basename(file).endswith('.yaml') - ] - self.add_arguments(yaml_files) + self.add_arguments(*module.find_source_files()) + self.set_build_unit(build_unit) self.print_arguments(True) diff --git a/scons/code_style_cpp.py b/scons/code_style_cpp.py deleted file mode 100644 index 60e0dd709..000000000 --- a/scons/code_style_cpp.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Provides utility functions for checking and enforcing code style definitions. -""" -from code_style.cpp.clang_format import ClangFormat -from code_style.cpp.cpplint import CppLint -from modules_old import CPP_MODULE - - -def check_cpp_code_style(**_): - """ - Checks if the C++ source files adhere to the code style definitions. If this is not the case, an error is raised. - """ - root_dir = CPP_MODULE.root_dir - print('Checking C++ code style in directory "' + root_dir + '"...') - ClangFormat(root_dir).run() - - for subproject in CPP_MODULE.find_subprojects(): - for directory in [subproject.include_dir, subproject.src_dir]: - CppLint(directory).run() - - -def enforce_cpp_code_style(**_): - """ - Enforces the C++ source files to adhere to the code style definitions. - """ - root_dir = CPP_MODULE.root_dir - print('Formatting C++ code in directory "' + root_dir + '"...') - ClangFormat(root_dir, enforce_changes=True).run() diff --git a/scons/code_style_md.py b/scons/code_style_md.py deleted file mode 100644 index 8ba161557..000000000 --- a/scons/code_style_md.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Provides utility functions for checking and enforcing code style definitions. -""" -from code_style.markdown.mdformat import MdFormat -from modules_old import DOC_MODULE, PYTHON_MODULE - -MD_DIRS = [('.', False), (DOC_MODULE.root_dir, True), (PYTHON_MODULE.root_dir, True)] - - -def check_md_code_style(**_): - """ - Checks if the Markdown files adhere to the code style definitions. If this is not the case, an error is raised. - """ - for directory, recursive in MD_DIRS: - print('Checking Markdown code style in the directory "' + directory + '"...') - MdFormat(directory, recursive=recursive).run() - - -def enforce_md_code_style(**_): - """ - Enforces the Markdown files to adhere to the code style definitions. - """ - for directory, recursive in MD_DIRS: - print('Formatting Markdown files in the directory "' + directory + '"...') - MdFormat(directory, recursive=recursive, enforce_changes=True).run() diff --git a/scons/code_style_python.py b/scons/code_style_python.py deleted file mode 100644 index 66bad6436..000000000 --- a/scons/code_style_python.py +++ /dev/null @@ -1,32 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Provides utility functions for checking and enforcing code style definitions. -""" -from code_style.python.isort import ISort -from code_style.python.pylint import PyLint -from code_style.python.yapf import Yapf -from modules_old import BUILD_MODULE, DOC_MODULE, PYTHON_MODULE - - -def check_python_code_style(**_): - """ - Checks if the Python source files adhere to the code style definitions. If this is not the case, an error is raised. - """ - for module in [BUILD_MODULE, PYTHON_MODULE]: - directory = module.root_dir - print('Checking Python code style in directory "' + directory + '"...') - ISort(directory).run() - Yapf(directory).run() - PyLint(directory).run() - - -def enforce_python_code_style(**_): - """ - Enforces the Python source files to adhere to the code style definitions. - """ - for module in [BUILD_MODULE, PYTHON_MODULE, DOC_MODULE]: - directory = module.root_dir - print('Formatting Python code in directory "' + directory + '"...') - ISort(directory, enforce_changes=True).run() - Yapf(directory, enforce_changes=True).run() diff --git a/scons/code_style_yaml.py b/scons/code_style_yaml.py deleted file mode 100644 index 7ddf7cc90..000000000 --- a/scons/code_style_yaml.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Provides utility functions for checking and enforcing code style definitions. -""" -from code_style.yaml.yamlfix import YamlFix - -YAML_DIRS = [('.', False), ('.github', True)] - - -def check_yaml_code_style(**_): - """ - Checks if the YAML files adhere to the code style definitions. If this is not the case, an error is raised. - """ - for directory, recursive in YAML_DIRS: - print('Checking YAML files in the directory "' + directory + '"...') - YamlFix(directory, recursive=recursive).run() - - -def enforce_yaml_code_style(**_): - """ - Enforces the YAML files to adhere to the code style definitions. - """ - for directory, recursive in YAML_DIRS: - print('Formatting YAML files in the directory "' + directory + '"...') - YamlFix(directory, recursive=recursive, enforce_changes=True).run() diff --git a/scons/modules/__init__.py b/scons/modules/__init__.py new file mode 100644 index 000000000..41da666b1 --- /dev/null +++ b/scons/modules/__init__.py @@ -0,0 +1,28 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Defines modules to be dealt with by the build system. +""" +from os import path + +from code_style.modules import CodeModule +from util.files import FileSearch +from util.languages import Language + +MODULES = [ + CodeModule(Language.YAML, '.', + FileSearch().set_recursive(False).set_hidden(True)), + CodeModule(Language.YAML, '.github'), + CodeModule(Language.MARKDOWN, '.', + FileSearch().set_recursive(False)), + CodeModule(Language.MARKDOWN, 'doc'), + CodeModule(Language.MARKDOWN, 'python'), + CodeModule(Language.PYTHON, 'scons'), + CodeModule(Language.PYTHON, 'doc'), + CodeModule(Language.PYTHON, 'python', + FileSearch().set_recursive(True).exclude_subdirectories_by_name('build')), + CodeModule(Language.CYTHON, 'python', + FileSearch().set_recursive(True).exclude_subdirectories_by_name('build')), + CodeModule(Language.CPP, 'cpp', + FileSearch().set_recursive(True).exclude_subdirectories_by_name('build')) +] diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 95f940379..68e344481 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -7,10 +7,7 @@ from os import path -from code_style_cpp import check_cpp_code_style, enforce_cpp_code_style -from code_style_md import check_md_code_style, enforce_md_code_style -from code_style_python import check_python_code_style, enforce_python_code_style -from code_style_yaml import check_yaml_code_style, enforce_yaml_code_style +from cpp import compile_cpp, install_cpp, setup_cpp from cython import compile_cython, install_cython, setup_cython from dependencies import check_dependency_versions, install_runtime_dependencies from documentation import apidoc_cpp, apidoc_cpp_tocfile, apidoc_python, apidoc_python_tocfile, doc @@ -24,8 +21,6 @@ from util.reflection import import_source_file from util.targets import Target, TargetRegistry -from cpp import compile_cpp, install_cpp, setup_cpp - from SCons.Script import COMMAND_LINE_TARGETS @@ -39,16 +34,6 @@ def __print_if_clean(environment, message: str): # Define target names... -TARGET_NAME_TEST_FORMAT = 'test_format' -TARGET_NAME_TEST_FORMAT_PYTHON = TARGET_NAME_TEST_FORMAT + '_python' -TARGET_NAME_TEST_FORMAT_CPP = TARGET_NAME_TEST_FORMAT + '_cpp' -TARGET_NAME_TEST_FORMAT_MD = TARGET_NAME_TEST_FORMAT + '_md' -TARGET_NAME_TEST_FORMAT_YAML = TARGET_NAME_TEST_FORMAT + '_yaml' -TARGET_NAME_FORMAT = 'format' -TARGET_NAME_FORMAT_PYTHON = TARGET_NAME_FORMAT + '_python' -TARGET_NAME_FORMAT_CPP = TARGET_NAME_FORMAT + '_cpp' -TARGET_NAME_FORMAT_MD = TARGET_NAME_FORMAT + '_md' -TARGET_NAME_FORMAT_YAML = TARGET_NAME_FORMAT + '_yaml' TARGET_NAME_DEPENDENCIES_CHECK = 'check_dependencies' TARGET_NAME_GITHUB_ACTIONS_CHECK = 'check_github_actions' TARGET_NAME_GITHUB_ACTIONS_UPDATE = 'update_github_actions' @@ -70,13 +55,11 @@ def __print_if_clean(environment, message: str): TARGET_NAME_DOC = 'doc' VALID_TARGETS = { - TARGET_NAME_TEST_FORMAT, TARGET_NAME_TEST_FORMAT_PYTHON, TARGET_NAME_TEST_FORMAT_CPP, TARGET_NAME_TEST_FORMAT_MD, - TARGET_NAME_TEST_FORMAT_YAML, TARGET_NAME_FORMAT, TARGET_NAME_FORMAT_PYTHON, TARGET_NAME_FORMAT_CPP, - TARGET_NAME_FORMAT_MD, TARGET_NAME_FORMAT_YAML, TARGET_NAME_DEPENDENCIES_CHECK, TARGET_NAME_GITHUB_ACTIONS_CHECK, - TARGET_NAME_GITHUB_ACTIONS_UPDATE, TARGET_NAME_VENV, TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP, - TARGET_NAME_COMPILE_CYTHON, TARGET_NAME_INSTALL, TARGET_NAME_INSTALL_CPP, TARGET_NAME_INSTALL_CYTHON, - TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS, TARGET_NAME_TESTS, TARGET_NAME_TESTS_CPP, - TARGET_NAME_TESTS_PYTHON, TARGET_NAME_APIDOC, TARGET_NAME_APIDOC_CPP, TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC + TARGET_NAME_DEPENDENCIES_CHECK, TARGET_NAME_GITHUB_ACTIONS_CHECK, TARGET_NAME_GITHUB_ACTIONS_UPDATE, + TARGET_NAME_VENV, TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP, TARGET_NAME_COMPILE_CYTHON, TARGET_NAME_INSTALL, + TARGET_NAME_INSTALL_CPP, TARGET_NAME_INSTALL_CYTHON, TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS, + TARGET_NAME_TESTS, TARGET_NAME_TESTS_CPP, TARGET_NAME_TESTS_PYTHON, TARGET_NAME_APIDOC, TARGET_NAME_APIDOC_CPP, + TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC } DEFAULT_TARGET = TARGET_NAME_INSTALL_WHEELS @@ -117,23 +100,6 @@ def __print_if_clean(environment, message: str): # Create temporary file ".sconsign.dblite" in the build directory... env.SConsignFile(name=path.relpath(path.join(BUILD_MODULE.build_dir, '.sconsign'), BUILD_MODULE.root_dir)) -# Define targets for checking code style definitions... -target_test_format_python = __create_phony_target(env, TARGET_NAME_TEST_FORMAT_PYTHON, action=check_python_code_style) -target_test_format_cpp = __create_phony_target(env, TARGET_NAME_TEST_FORMAT_CPP, action=check_cpp_code_style) -target_test_format_md = __create_phony_target(env, TARGET_NAME_TEST_FORMAT_MD, action=check_md_code_style) -target_test_format_yaml = __create_phony_target(env, TARGET_NAME_TEST_FORMAT_YAML, action=check_yaml_code_style) -target_test_format = __create_phony_target(env, TARGET_NAME_TEST_FORMAT) -env.Depends(target_test_format, - [target_test_format_python, target_test_format_cpp, target_test_format_md, target_test_format_yaml]) - -# Define targets for enforcing code style definitions... -target_format_python = __create_phony_target(env, TARGET_NAME_FORMAT_PYTHON, action=enforce_python_code_style) -target_format_cpp = __create_phony_target(env, TARGET_NAME_FORMAT_CPP, action=enforce_cpp_code_style) -target_format_md = __create_phony_target(env, TARGET_NAME_FORMAT_MD, action=enforce_md_code_style) -target_format_yaml = __create_phony_target(env, TARGET_NAME_FORMAT_YAML, action=enforce_yaml_code_style) -target_format = __create_phony_target(env, TARGET_NAME_FORMAT) -env.Depends(target_format, [target_format_python, target_format_cpp, target_format_md, target_format_yaml]) - # Define target for checking dependency versions... __create_phony_target(env, TARGET_NAME_DEPENDENCIES_CHECK, action=check_dependency_versions) diff --git a/scons/util/requirements.txt b/scons/util/requirements.txt index 46cc120e2..7440aaa37 100644 --- a/scons/util/requirements.txt +++ b/scons/util/requirements.txt @@ -1,19 +1,11 @@ build >= 1.2, < 1.3 -cpplint >= 2.0, < 2.1 -clang-format >= 19.1, < 19.2 cython >= 3.0, < 3.1 -isort >= 5.13, < 5.14 -mdformat >= 0.7, < 0.8 -mdformat-myst >= 0.2, < 0.3 meson >= 1.6, < 1.7 ninja >= 1.11, < 1.12 pygithub >= 2.5, < 2.6 -pylint >= 3.3, < 3.4 pyyaml >= 6.0, < 6.1 setuptools scons >= 4.8, < 4.9 tabulate >= 0.9, < 0.10 unittest-xml-reporting >= 3.2, < 3.3 wheel >= 0.45, < 0.46 -yamlfix >= 1.17, < 1.18 -yapf >= 0.43, < 0.44 From c7666ad7156e60b605a3e25d8508bdb0f3570016 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Tue, 26 Nov 2024 21:24:40 +0100 Subject: [PATCH 040/114] Rename file dependencies.py to python_dependencies.py. --- scons/{dependencies.py => python_dependencies.py} | 0 scons/sconstruct.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename scons/{dependencies.py => python_dependencies.py} (100%) diff --git a/scons/dependencies.py b/scons/python_dependencies.py similarity index 100% rename from scons/dependencies.py rename to scons/python_dependencies.py diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 68e344481..ed44fd564 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -9,11 +9,11 @@ from cpp import compile_cpp, install_cpp, setup_cpp from cython import compile_cython, install_cython, setup_cython -from dependencies import check_dependency_versions, install_runtime_dependencies from documentation import apidoc_cpp, apidoc_cpp_tocfile, apidoc_python, apidoc_python_tocfile, doc from github_actions import check_github_actions, update_github_actions from modules_old import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE from packaging import build_python_wheel, install_python_wheels +from python_dependencies import check_dependency_versions, install_runtime_dependencies from testing import tests_cpp, tests_python from util.files import DirectorySearch from util.format import format_iterable From ee7c4c5f323e4661a5ced2877679c895f3e4b098 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Tue, 26 Nov 2024 21:25:03 +0100 Subject: [PATCH 041/114] Add class Table. --- scons/dependencies/__init__.py | 0 scons/util/table.py | 45 ++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 scons/dependencies/__init__.py create mode 100644 scons/util/table.py diff --git a/scons/dependencies/__init__.py b/scons/dependencies/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scons/util/table.py b/scons/util/table.py new file mode 100644 index 000000000..ee3113841 --- /dev/null +++ b/scons/util/table.py @@ -0,0 +1,45 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes for creating tables. +""" +from util.pip import Pip +from util.units import BuildUnit + + +class Table: + """ + A table with optional headers. + """ + + def __init__(self, build_unit: BuildUnit, *headers: str): + """ + :param build_unit: The build unit, the table is created for + :param headers: The headers of the table + """ + self.build_unit = build_unit + self.headers = list(headers) if headers else None + self.rows = [] + + def add_row(self, *entries: str): + """ + Adds a new row to the end of the table. + + :param entries: The entries of the row to be added + """ + self.rows.append(list(entries)) + + def sort_rows(self, column_index: int, *additional_column_indices: int): + """ + Sorts the rows in the table. + + :param column_index: The index of the column to sort by + :param additional_column_indices: Additional indices of columns to sort by + """ + self.rows.sort(key=lambda row: ([row[i] for i in [column_index] + list(additional_column_indices)])) + + def __str__(self) -> str: + Pip(self.build_unit).install_packages('tabulate') + # pylint: disable=import-outside-toplevel + from tabulate import tabulate + return tabulate(self.rows, headers=self.headers) From 7f5cade491460fa9bfd29a1b4502ee14a071f6b2 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 27 Nov 2024 00:33:38 +0100 Subject: [PATCH 042/114] Add class TextFile. --- scons/util/io.py | 33 +++++++++++++++++++++++++++++++++ scons/util/pip.py | 22 ++++++++-------------- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/scons/util/io.py b/scons/util/io.py index d39fe400d..612f7fd4c 100644 --- a/scons/util/io.py +++ b/scons/util/io.py @@ -3,6 +3,8 @@ Provides utility functions for reading and writing files. """ +from functools import cached_property +from typing import List ENCODING_UTF8 = 'utf-8' @@ -23,3 +25,34 @@ def write_file(file: str): :param file: The file to be opened """ return open(file, mode='w', encoding=ENCODING_UTF8) + + +class TextFile: + """ + Allows to read and write the content of a text file. + """ + + def __init__(self, file: str): + """ + :param file: The path to the text file + """ + self.file = file + + @cached_property + def lines(self) -> List[str]: + """ + The lines in the text file. + """ + with read_file(self.file) as file: + return file.readlines() + + def write_lines(self, lines: List[str]): + """ + Overwrites all lines in the text file. + + :param lines: The lines to be written + """ + with write_file(self.file) as file: + file.writelines(lines) + + del self.lines diff --git a/scons/util/pip.py b/scons/util/pip.py index 8737d9c95..36e993b99 100644 --- a/scons/util/pip.py +++ b/scons/util/pip.py @@ -5,11 +5,10 @@ """ from abc import ABC from dataclasses import dataclass -from functools import cached_property from typing import Dict, Optional, Set from util.cmd import Command as Cmd -from util.io import read_file +from util.io import TextFile from util.units import BuildUnit @@ -75,24 +74,20 @@ def __hash__(self) -> int: return hash(self.package) -@dataclass -class RequirementsFile: +class RequirementsFile(TextFile): """ Represents a specific requirements.txt file. - - Attributes: - requirements_file: The path to the requirements file """ - requirements_file: str - @cached_property + @property def requirements_by_package(self) -> Dict[Package, Requirement]: """ A dictionary that contains all requirements in the requirements file by their package. """ - with read_file(self.requirements_file) as file: - requirements = [Requirement.parse(line) for line in file.readlines() if line.strip('\n').strip()] - return {requirement.package: requirement for requirement in requirements} + return { + requirement.package: requirement + for requirement in [Requirement.parse(line) for line in self.lines if line.strip('\n').strip()] + } @property def requirements(self) -> Set[Requirement]: @@ -118,8 +113,7 @@ def lookup(self, *packages: Package, accept_missing: bool = False) -> Set[Requir if requirement: requirements.add(requirement) elif not accept_missing: - raise RuntimeError('Package "' + str(package) + '" not found in requirements file "' - + self.requirements_file + '"') + raise RuntimeError('Package "' + str(package) + '" not found in requirements file "' + self.file + '"') return requirements From 87d5a8105fe93902fb9e24a23cd446cb5c0ede91 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Tue, 26 Nov 2024 21:33:23 +0100 Subject: [PATCH 043/114] Add class YamlFile. --- scons/dependencies/github/__init__.py | 0 scons/dependencies/github/pyyaml.py | 40 ++++++++++++++++++++++ scons/dependencies/github/requirements.txt | 1 + 3 files changed, 41 insertions(+) create mode 100644 scons/dependencies/github/__init__.py create mode 100644 scons/dependencies/github/pyyaml.py create mode 100644 scons/dependencies/github/requirements.txt diff --git a/scons/dependencies/github/__init__.py b/scons/dependencies/github/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scons/dependencies/github/pyyaml.py b/scons/dependencies/github/pyyaml.py new file mode 100644 index 000000000..7513d363b --- /dev/null +++ b/scons/dependencies/github/pyyaml.py @@ -0,0 +1,40 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes for reading the contents of YAML files via "pyyaml". +""" +from functools import cached_property +from typing import Dict, List + +from util.io import TextFile, read_file +from util.pip import Pip +from util.units import BuildUnit + + +class YamlFile(TextFile): + """ + A YAML file. + """ + + def __init__(self, build_unit: BuildUnit, file: str): + """ + :param build_unit: The build unit from which the YAML file is read + :param file: The path to the YAML file + """ + super().__init__(file) + self.build_unit = build_unit + + @cached_property + def yaml_dict(self) -> Dict: + """ + A dictionary that stores the content of the YAML file. + """ + Pip(self.build_unit).install_packages('pyyaml') + # pylint: disable=import-outside-toplevel + import yaml + with read_file(self.file) as file: + return yaml.load(file.read(), Loader=yaml.CLoader) + + def write_lines(self, lines: List[str]): + super().write_lines(lines) + del self.yaml_dict diff --git a/scons/dependencies/github/requirements.txt b/scons/dependencies/github/requirements.txt new file mode 100644 index 000000000..7f5b0b3e2 --- /dev/null +++ b/scons/dependencies/github/requirements.txt @@ -0,0 +1 @@ +pyyaml >= 6.0, < 6.1 From df7362bd8ffed3cb8be09fda2a4adce86c6852de Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Tue, 26 Nov 2024 22:09:43 +0100 Subject: [PATCH 044/114] Add class GithubApi. --- scons/dependencies/github/pygithub.py | 75 ++++++++++++++++++++++ scons/dependencies/github/requirements.txt | 1 + 2 files changed, 76 insertions(+) create mode 100644 scons/dependencies/github/pygithub.py diff --git a/scons/dependencies/github/pygithub.py b/scons/dependencies/github/pygithub.py new file mode 100644 index 000000000..781ef0065 --- /dev/null +++ b/scons/dependencies/github/pygithub.py @@ -0,0 +1,75 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes for accessing the GitHub API via "pygithub". +""" +from typing import Optional + +from util.pip import Pip +from util.units import BuildUnit + + +class GithubApi: + """ + Allows to access the GitHub API. + """ + + class Repository: + """ + Allows to query information about a single GitHub repository. + """ + + def __init__(self, repository_name: str, authentication): + """ + :param repository_name: The name of the repository + :param authentication: The authentication to be used for accessing the repository or None, if no + authentication should be used + """ + self.repository_name = repository_name + self.authentication = authentication + + def get_latest_release_tag(self) -> Optional[str]: + """ + Returns the tag of the repository's latest release, if any. + + :return: The tag of the latest release or None, if no release is available + """ + # pylint: disable=import-outside-toplevel + from github import Github, UnknownObjectException + + with Github(auth=self.authentication) as client: + try: + repository = client.get_repo(self.repository_name) + latest_release = repository.get_latest_release() + return latest_release.tag_name + except UnknownObjectException as error: + raise RuntimeError('Failed to query latest release of GitHub repository "' + self.repository_name + + '"') from error + + def __init__(self, build_unit: BuildUnit): + """ + :param build_unit: The build unit to access the GitHub API from + """ + Pip(build_unit).install_packages('pygithub') + self.authentication = None + + def set_token(self, token: Optional[str]) -> 'GithubApi': + """ + Sets a token to be used for authentication. + + :param token: The token to be set or None, if no token should be used + :return: The `GithubApi` itself + """ + # pylint: disable=import-outside-toplevel + from github import Auth + self.authentication = Auth.Token(token) if token else None + return self + + def open_repository(self, repository_name: str) -> Repository: + """ + Specifies the name of a GitHub repository about which information should be queried. + + :param repository_name: The name of the repository, e.g., "mrapp-ke/MLRL-Boomer" + :return: A `GithubApi.Repository` + """ + return GithubApi.Repository(repository_name, self.authentication) diff --git a/scons/dependencies/github/requirements.txt b/scons/dependencies/github/requirements.txt index 7f5b0b3e2..27a52b775 100644 --- a/scons/dependencies/github/requirements.txt +++ b/scons/dependencies/github/requirements.txt @@ -1 +1,2 @@ +pygithub >= 2.5, < 2.6 pyyaml >= 6.0, < 6.1 From 4b7ef4f0096b735980ec8c78f6e4adc8fa97c5d6 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Tue, 26 Nov 2024 22:27:57 +0100 Subject: [PATCH 045/114] Dynamically register targets for checking and updating GitHub Actions. --- scons/dependencies/github/__init__.py | 13 + scons/dependencies/github/actions.py | 343 ++++++++++++++++++++ scons/dependencies/github/requirements.txt | 1 + scons/dependencies/github/targets.py | 56 ++++ scons/github_actions.py | 353 --------------------- scons/sconstruct.py | 16 +- scons/util/requirements.txt | 2 - 7 files changed, 417 insertions(+), 367 deletions(-) create mode 100644 scons/dependencies/github/actions.py create mode 100644 scons/dependencies/github/targets.py delete mode 100644 scons/github_actions.py diff --git a/scons/dependencies/github/__init__.py b/scons/dependencies/github/__init__.py index e69de29bb..b6fe86d02 100644 --- a/scons/dependencies/github/__init__.py +++ b/scons/dependencies/github/__init__.py @@ -0,0 +1,13 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Defines targets for updating the project's GitHub Actions. +""" +from dependencies.github.targets import CheckGithubActions, UpdateGithubActions +from util.targets import PhonyTarget, TargetBuilder +from util.units import BuildUnit + +TARGETS = TargetBuilder(BuildUnit.by_name('dependencies', 'github')) \ + .add_phony_target('check_github_actions').set_runnables(CheckGithubActions()) \ + .add_phony_target('update_github_actions').set_runnables(UpdateGithubActions()) \ + .build() diff --git a/scons/dependencies/github/actions.py b/scons/dependencies/github/actions.py new file mode 100644 index 000000000..f78e9264c --- /dev/null +++ b/scons/dependencies/github/actions.py @@ -0,0 +1,343 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for checking the project's GitHub workflows for outdated Actions. +""" +import sys + +from dataclasses import dataclass, replace +from functools import cached_property, reduce +from os import environ, path +from typing import Dict, List, Optional, Set + +from dependencies.github.pygithub import GithubApi +from dependencies.github.pyyaml import YamlFile +from util.env import get_env +from util.files import FileSearch +from util.languages import Language +from util.units import BuildUnit + + +@dataclass +class ActionVersion: + """ + The version of a GitHub Action. + + Attributes: + version: The full version string + """ + version: str + + SEPARATOR = '.' + + @staticmethod + def from_version_numbers(*version_numbers: int) -> 'ActionVersion': + """ + Creates and returns the version of a GitHub Action from one or several version numbers. + + :param version_numbers: The version numbers + :return: The version that has been created + """ + return ActionVersion(ActionVersion.SEPARATOR.join([str(version_number) for version_number in version_numbers])) + + @property + def version_numbers(self) -> List[int]: + """ + A list that stores the individual version numbers, the full version consists of. + """ + return [int(version_number) for version_number in str(self).split(self.SEPARATOR)] + + def __str__(self) -> str: + return self.version.lstrip('v') + + def __lt__(self, other: 'ActionVersion') -> bool: + first_version_numbers = self.version_numbers + second_version_numbers = other.version_numbers + + for i in range(min(len(first_version_numbers), len(second_version_numbers))): + first_version_number = first_version_numbers[i] + second_version_number = second_version_numbers[i] + + if first_version_number > second_version_number: + return False + if first_version_number < second_version_number: + return True + + return False + + +@dataclass +class Action: + """ + A GitHub Action. + + Attributes: + name: The name of the Action + version: The version of the Action + """ + name: str + version: ActionVersion + + SEPARATOR = '@' + + @staticmethod + def from_uses_clause(uses_clause: str) -> 'Action': + """ + Creates and returns a GitHub Action from the uses-clause of a workflow. + + :param uses_clause: The uses-clause + :return: The GitHub Action that has been created + """ + parts = uses_clause.split(Action.SEPARATOR) + + if len(parts) != 2: + raise ValueError('Uses-clause must contain the symbol + "' + Action.SEPARATOR + '", but got "' + uses_clause + + '"') + + return Action(name=parts[0], version=ActionVersion(parts[1])) + + @property + def repository(self) -> str: + """ + The name of the repository, where the GitHub Action is hosted. + """ + repository = self.name + separator = '/' + parts = repository.split(separator) + return separator.join(parts[:2]) if len(parts) > 2 else repository + + def __str__(self) -> str: + return self.name + self.SEPARATOR + str(self.version) + + def __eq__(self, other: 'Action') -> bool: + return str(self) == str(other) + + def __hash__(self) -> int: + return hash(str(self)) + + +class Workflow(YamlFile): + """ + A GitHub workflow. + """ + + TAG_USES = 'uses' + + @cached_property + def uses_clauses(self) -> List[str]: + """ + A list that contains all uses-clauses in the workflow. + """ + uses_clauses = [] + + for job in self.yaml_dict.get('jobs', {}).values(): + for step in job.get('steps', []): + uses_clause = step.get(self.TAG_USES, None) + + if uses_clause: + uses_clauses.append(uses_clause) + + return uses_clauses + + @cached_property + def actions(self) -> Set[Action]: + """ + A set that contains all GitHub Actions used in the workflow. + """ + actions = set() + + for uses_clause in self.uses_clauses: + try: + actions.add(Action.from_uses_clause(uses_clause)) + except ValueError as error: + print('Failed to parse uses-clause in workflow "' + self.file + '": ' + str(error)) + sys.exit(-1) + + return actions + + def update_actions(self, *updated_actions: Action): + """ + Updates given Actions in the workflow definition file. + + :param updated_actions: The actions to be updated + """ + updated_actions_by_name = reduce(lambda aggr, x: dict(aggr, **{x.name: x}), updated_actions, {}) + uses_prefix = self.TAG_USES + ':' + updated_lines = [] + + for line in self.lines: + updated_lines.append(line) + line_stripped = line.strip() + + if line_stripped.startswith(uses_prefix): + uses_clause = line_stripped[len(uses_prefix):].strip() + action = Action.from_uses_clause(uses_clause) + updated_action = updated_actions_by_name.get(action.name) + + if updated_action: + updated_lines[-1] = line.replace(str(action.version), str(updated_action.version)) + + self.write_lines(updated_lines) + + def write_lines(self, lines: List[str]): + super().write_lines(lines) + del self.uses_clauses + del self.actions + + def __eq__(self, other: 'Workflow') -> bool: + return self.file == other.file + + def __hash__(self) -> int: + return hash(self.file) + + +class WorkflowUpdater: + """ + Allows checking the versions of GitHub Actions used in multiple workflows and updating outdated ones. + """ + + ENV_GITHUB_TOKEN = 'GITHUB_TOKEN' + + @dataclass + class OutdatedAction: + """ + An outdated GitHub Action. + + Attributes: + action: The outdated Action + latest_version: The latest version of the Action + """ + action: Action + latest_version: ActionVersion + + def __str__(self) -> str: + return str(self.action) + + def __eq__(self, other: 'WorkflowUpdater.OutdatedAction') -> bool: + return self.action == other.action + + def __hash__(self) -> int: + return hash(self.action) + + @dataclass + class UpdatedAction: + """ + A GitHub Action that has been updated. + + Attributes: + previous: The previous Action + updated: The updated Action + """ + previous: 'WorkflowUpdater.OutdatedAction' + updated: Action + + def __str__(self) -> str: + return str(self.updated) + + def __eq__(self, other: 'WorkflowUpdater.UpdatedAction') -> bool: + return self.updated == other.updated + + def __hash__(self) -> int: + return hash(self.updated) + + def __get_github_token(self) -> Optional[str]: + github_token = get_env(environ, self.ENV_GITHUB_TOKEN) + + if not github_token: + print('No GitHub API token is set. You can specify it via the environment variable ' + self.ENV_GITHUB_TOKEN + + '.') + + return github_token + + def __query_latest_action_version(self, action: Action) -> ActionVersion: + repository_name = action.repository + + try: + latest_tag = GithubApi(self.build_unit) \ + .set_token(self.__get_github_token()) \ + .open_repository(repository_name) \ + .get_latest_release_tag() + + if not latest_tag: + raise RuntimeError('No releases available') + + return ActionVersion(latest_tag) + except RuntimeError as error: + print('Unable to determine latest version of action "' + str(action) + '" hosted in repository "' + + repository_name + '": ' + str(error)) + sys.exit(-1) + + def __get_latest_action_version(self, action: Action) -> ActionVersion: + latest_version = self.version_cache.get(action.name) + + if not latest_version: + print('Checking version of GitHub Action "' + action.name + '"...') + latest_version = self.__query_latest_action_version(action) + self.version_cache[action.name] = latest_version + + return latest_version + + def __init__(self, build_unit: BuildUnit, workflow_directory: str = path.join('.github', 'workflows')): + """ + :param build_unit: The build unit from which workflow definition files should be read + :param workflow_directory: The path to the directory where the workflow definition files are located + """ + self.build_unit = build_unit + self.workflow_directory = workflow_directory + self.version_cache = {} + + @cached_property + def workflows(self) -> Set[Workflow]: + """ + All GitHub workflows that are defined in the directory where workflow definition files are located. + """ + workflows = set() + + for workflow_file in FileSearch().set_languages(Language.YAML).list(self.workflow_directory): + print('Searching for GitHub Actions in workflow "' + workflow_file + '"...') + workflows.add(Workflow(self.build_unit, workflow_file)) + + return workflows + + def find_outdated_workflows(self) -> Dict[Workflow, Set[OutdatedAction]]: + """ + Finds and returns all workflows with outdated GitHub actions. + + :return: A dictionary that contains for each workflow a set of outdated Actions + """ + outdated_workflows = {} + + for workflow in self.workflows: + for action in workflow.actions: + latest_version = self.__get_latest_action_version(action) + + if action.version < latest_version: + outdated_actions = outdated_workflows.setdefault(workflow, set()) + outdated_actions.add(WorkflowUpdater.OutdatedAction(action, latest_version)) + + return outdated_workflows + + def update_outdated_workflows(self) -> Dict[Workflow, Set[UpdatedAction]]: + """ + Updates all workflows with outdated GitHub Actions. + + :return: A dictionary that contains for each workflow a set of updated Actions + """ + updated_workflows = {} + + for workflow, outdated_actions in self.find_outdated_workflows().items(): + updated_actions = set() + + for outdated_action in outdated_actions: + previous_version = outdated_action.action.version + previous_version_numbers = previous_version.version_numbers + latest_version_numbers = outdated_action.latest_version.version_numbers + max_version_numbers = min(len(previous_version_numbers), len(latest_version_numbers)) + updated_version = ActionVersion.from_version_numbers(*latest_version_numbers[:max_version_numbers]) + updated_actions = updated_workflows.setdefault(workflow, updated_actions) + updated_action = replace(outdated_action.action, version=updated_version) + updated_actions.add(WorkflowUpdater.UpdatedAction(previous=outdated_action, updated=updated_action)) + + workflow.update_actions(*[updated_action.updated for updated_action in updated_actions]) + + return updated_workflows diff --git a/scons/dependencies/github/requirements.txt b/scons/dependencies/github/requirements.txt index 27a52b775..d06d96dac 100644 --- a/scons/dependencies/github/requirements.txt +++ b/scons/dependencies/github/requirements.txt @@ -1,2 +1,3 @@ pygithub >= 2.5, < 2.6 pyyaml >= 6.0, < 6.1 +tabulate >= 0.9, < 0.10 diff --git a/scons/dependencies/github/targets.py b/scons/dependencies/github/targets.py new file mode 100644 index 000000000..ef39d0bb2 --- /dev/null +++ b/scons/dependencies/github/targets.py @@ -0,0 +1,56 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Implements targets for updating the project's GitHub Actions. +""" +from dependencies.github.actions import WorkflowUpdater +from util.modules import ModuleRegistry +from util.table import Table +from util.targets import PhonyTarget +from util.units import BuildUnit + + +class CheckGithubActions(PhonyTarget.Runnable): + """ + Prints all outdated Actions used in the project's GitHub workflows. + """ + + def run(self, build_unit: BuildUnit, _: ModuleRegistry): + outdated_workflows = WorkflowUpdater(build_unit).find_outdated_workflows() + + if outdated_workflows: + table = Table(build_unit, 'Workflow', 'Action', 'Current version', 'Latest version') + + for workflow, outdated_actions in outdated_workflows.items(): + for outdated_action in outdated_actions: + table.add_row(workflow.file, str(outdated_action.action.name), str(outdated_action.action.version), + str(outdated_action.latest_version)) + + table.sort_rows(0, 1) + print('The following GitHub Actions are outdated:\n') + print(str(table)) + else: + print('All GitHub Actions are up-to-date!') + + +class UpdateGithubActions(PhonyTarget.Runnable): + """ + Updates and prints all outdated Actions used in the project's GitHub workflows. + """ + + def run(self, build_unit: BuildUnit, _: ModuleRegistry): + updated_workflows = WorkflowUpdater(build_unit).update_outdated_workflows() + + if updated_workflows: + table = Table(build_unit, 'Workflow', 'Action', 'Previous version', 'Updated version') + + for workflow, updated_actions in updated_workflows.items(): + for updated_action in updated_actions: + table.add_row(workflow.file, updated_action.updated.name, + str(updated_action.previous.action.version), str(updated_action.updated.version)) + + table.sort_rows(0, 1) + print('The following GitHub Actions have been updated:\n') + print(str(table)) + else: + print('No GitHub Actions have been updated.') diff --git a/scons/github_actions.py b/scons/github_actions.py deleted file mode 100644 index 6f0abebb9..000000000 --- a/scons/github_actions.py +++ /dev/null @@ -1,353 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Provides utility functions for checking the project's GitHub workflows for outdated Actions. -""" -import sys - -from dataclasses import dataclass, field -from functools import reduce -from glob import glob -from os import environ, path -from typing import List, Optional, Set - -from util.env import get_env -from util.io import read_file, write_file -from util.pip import Pip - -ENV_GITHUB_TOKEN = 'GITHUB_TOKEN' - - -@dataclass -class ActionVersion: - """ - The version of a GitHub Action. - - Attributes: - version: The full version string - """ - version: str - - SEPARATOR = '.' - - @staticmethod - def from_version_numbers(*version_numbers: int) -> 'ActionVersion': - """ - Creates and returns the version of a GitHub Action from one or several version numbers. - - :param version_numbers: The version numbers - :return: The version that has been created - """ - return ActionVersion(ActionVersion.SEPARATOR.join([str(version_number) for version_number in version_numbers])) - - @property - def version_numbers(self) -> List[int]: - """ - A list that stores the individual version numbers, the full version consists of. - """ - return [int(version_number) for version_number in str(self).split(self.SEPARATOR)] - - def __str__(self) -> str: - return self.version.lstrip('v') - - def __lt__(self, other: 'ActionVersion') -> bool: - first_version_numbers = self.version_numbers - second_version_numbers = other.version_numbers - - for i in range(min(len(first_version_numbers), len(second_version_numbers))): - first_version_number = first_version_numbers[i] - second_version_number = second_version_numbers[i] - - if first_version_number > second_version_number: - return False - if first_version_number < second_version_number: - return True - - return False - - -@dataclass -class Action: - """ - A GitHub Action. - - Attributes: - name: The name of the Action - version: The version of the Action - latest_version: The latest version of the Action, if known - """ - name: str - version: ActionVersion - latest_version: Optional[ActionVersion] = None - - SEPARATOR = '@' - - @staticmethod - def from_uses_clause(uses_clause: str) -> 'Action': - """ - Creates and returns a GitHub Action from the uses-clause of a workflow. - - :param uses_clause: The uses-clause - :return: The GitHub Action that has been created - """ - parts = uses_clause.split(Action.SEPARATOR) - - if len(parts) != 2: - raise ValueError('Uses-clause must contain the symbol + "' + Action.SEPARATOR + '", but got "' + uses_clause - + '"') - - return Action(name=parts[0], version=ActionVersion(parts[1])) - - @property - def repository(self) -> str: - """ - The name of the repository, where the GitHub Action is hosted. - """ - repository = self.name - separator = '/' - parts = repository.split(separator) - return separator.join(parts[:2]) if len(parts) > 2 else repository - - @property - def is_outdated(self) -> bool: - """ - True, if the GitHub Action is known to be outdated, False otherwise. - """ - return self.latest_version and self.version < self.latest_version - - def __str__(self) -> str: - return self.name + self.SEPARATOR + str(self.version) - - def __eq__(self, other: 'Action') -> bool: - return str(self) == str(other) - - def __hash__(self): - return hash(str(self)) - - -@dataclass -class Workflow: - """ - A GitHub workflow. - - Attributes: - workflow_file: The path of the workflow definition file - yaml_dict: A dictionary that stores the YAML structure of the workflow definition file - actions: A set that stores all Actions in the workflow - """ - workflow_file: str - yaml_dict: dict - actions: Set[Action] = field(default_factory=set) - - TAG_USES = 'uses' - - @property - def uses_clauses(self) -> List[str]: - """ - A list that contains all uses-clauses in the workflow. - """ - uses_clauses = [] - - for job in self.yaml_dict.get('jobs', {}).values(): - for step in job.get('steps', []): - uses_clause = step.get(self.TAG_USES, None) - - if uses_clause: - uses_clauses.append(uses_clause) - - return uses_clauses - - @property - def outdated_actions(self) -> Set[Action]: - """ - A set that stores all Actions in the workflow that are known to be outdated. - """ - return {action for action in self.actions if action.is_outdated} - - def __eq__(self, other: 'Workflow') -> bool: - return self.workflow_file == other.workflow_file - - def __hash__(self): - return hash(self.workflow_file) - - -def __read_workflow(workflow_file: str) -> Workflow: - Pip().install_packages('pyyaml') - # pylint: disable=import-outside-toplevel - import yaml - with read_file(workflow_file) as file: - yaml_dict = yaml.load(file.read(), Loader=yaml.CLoader) - return Workflow(workflow_file=workflow_file, yaml_dict=yaml_dict) - - -def __read_workflow_lines(workflow_file: str) -> List[str]: - with read_file(workflow_file) as file: - return file.readlines() - - -def __write_workflow_lines(workflow_file: str, lines: List[str]): - with write_file(workflow_file) as file: - file.writelines(lines) - - -def __update_workflow(workflow_file: str, *updated_actions: Action): - updated_actions_by_name = reduce(lambda aggr, x: dict(aggr, **{x.name: x}), updated_actions, {}) - lines = __read_workflow_lines(workflow_file) - uses_prefix = Workflow.TAG_USES + ':' - updated_lines = [] - - for line in lines: - updated_lines.append(line) - line_stripped = line.strip() - - if line_stripped.startswith(uses_prefix): - uses_clause = line_stripped[len(uses_prefix):].strip() - action = Action.from_uses_clause(uses_clause) - updated_action = updated_actions_by_name.get(action.name) - - if updated_action: - updated_lines[-1] = line.replace(str(action.version), str(updated_action.version)) - - __write_workflow_lines(workflow_file, updated_lines) - - -def __parse_workflow(workflow_file: str) -> Workflow: - print('Searching for GitHub Actions in workflow "' + workflow_file + '"...') - workflow = __read_workflow(workflow_file) - - for uses_clause in workflow.uses_clauses: - try: - workflow.actions.add(Action.from_uses_clause(uses_clause)) - except ValueError as error: - print('Failed to parse uses-clause in workflow "' + workflow_file + '": ' + str(error)) - sys.exit(-1) - - return workflow - - -def __parse_workflows(*workflow_files: str) -> Set[Workflow]: - return {__parse_workflow(workflow_file) for workflow_file in workflow_files} - - -def __query_latest_action_version(action: Action, github_token: Optional[str] = None) -> Optional[ActionVersion]: - repository_name = action.repository - Pip().install_packages('pygithub') - # pylint: disable=import-outside-toplevel - from github import Auth, Github, UnknownObjectException - - try: - github_auth = Auth.Token(github_token) if github_token else None - github_client = Github(auth=github_auth) - github_repository = github_client.get_repo(repository_name) - latest_release = github_repository.get_latest_release() - latest_tag = latest_release.tag_name - return ActionVersion(latest_tag) - except UnknownObjectException as error: - print('Query to GitHub API failed for action "' + str(action) + '" hosted in repository "' + repository_name - + '": ' + str(error)) - sys.exit(-1) - - -def __get_github_token() -> Optional[str]: - github_token = get_env(environ, ENV_GITHUB_TOKEN) - - if not github_token: - print('No GitHub API token is set. You can specify it via the environment variable ' + ENV_GITHUB_TOKEN + '.') - - return github_token - - -def __determine_latest_action_versions(*workflows: Workflow) -> Set[Workflow]: - github_token = __get_github_token() - version_cache = {} - - for workflow in workflows: - for action in workflow.actions: - latest_version = version_cache.get(action.name) - - if not latest_version: - print('Checking version of GitHub Action "' + action.name + '"...') - latest_version = __query_latest_action_version(action, github_token=github_token) - version_cache[action.name] = latest_version - - action.latest_version = latest_version - - return set(workflows) - - -def __parse_all_workflows() -> Set[Workflow]: - workflow_directory = path.join('.github', 'workflows') - workflow_files = glob(path.join(workflow_directory, '*.y*ml')) - return __determine_latest_action_versions(*__parse_workflows(*workflow_files)) - - -def __print_table(header: List[str], rows: List[List[str]]): - Pip().install_packages('tabulate') - # pylint: disable=import-outside-toplevel - from tabulate import tabulate - print(tabulate(rows, headers=header)) - - -def __print_outdated_actions(*workflows: Workflow): - rows = [] - - for workflow in workflows: - for action in workflow.outdated_actions: - rows.append([workflow.workflow_file, str(action.name), str(action.version), str(action.latest_version)]) - - if rows: - rows.sort(key=lambda row: (row[0], row[1])) - header = ['Workflow', 'Action', 'Current version', 'Latest version'] - print('The following GitHub Actions are outdated:\n') - __print_table(header=header, rows=rows) - else: - print('All GitHub Actions are up-to-date!') - - -def __update_outdated_actions(*workflows: Workflow) -> Set[Workflow]: - rows = [] - - for workflow in workflows: - outdated_actions = workflow.outdated_actions - - if outdated_actions: - workflow_file = workflow.workflow_file - updated_actions = set() - - for action in outdated_actions: - previous_version = action.version - previous_version_numbers = previous_version.version_numbers - latest_version_numbers = action.latest_version.version_numbers - max_version_numbers = min(len(previous_version_numbers), len(latest_version_numbers)) - updated_version = ActionVersion.from_version_numbers(*latest_version_numbers[:max_version_numbers]) - rows.append([workflow_file, action.name, str(previous_version), str(updated_version)]) - action.version = updated_version - updated_actions.add(action) - - __update_workflow(workflow_file, *updated_actions) - - if rows: - rows.sort(key=lambda row: (row[0], row[1])) - header = ['Workflow', 'Action', 'Previous version', 'Updated version'] - print('The following GitHub Actions have been updated:\n') - __print_table(header=header, rows=rows) - else: - print('No GitHub Actions have been updated.') - - return set(workflows) - - -def check_github_actions(**_): - """ - Checks the project's GitHub workflows for outdated Actions. - """ - workflows = __parse_all_workflows() - __print_outdated_actions(*workflows) - - -def update_github_actions(**_): - """ - Updates the versions of outdated GitHub Actions in the project's workflows. - """ - workflows = __parse_all_workflows() - __update_outdated_actions(*workflows) diff --git a/scons/sconstruct.py b/scons/sconstruct.py index ed44fd564..cc3bf25f8 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -10,7 +10,6 @@ from cpp import compile_cpp, install_cpp, setup_cpp from cython import compile_cython, install_cython, setup_cython from documentation import apidoc_cpp, apidoc_cpp_tocfile, apidoc_python, apidoc_python_tocfile, doc -from github_actions import check_github_actions, update_github_actions from modules_old import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE from packaging import build_python_wheel, install_python_wheels from python_dependencies import check_dependency_versions, install_runtime_dependencies @@ -35,8 +34,6 @@ def __print_if_clean(environment, message: str): # Define target names... TARGET_NAME_DEPENDENCIES_CHECK = 'check_dependencies' -TARGET_NAME_GITHUB_ACTIONS_CHECK = 'check_github_actions' -TARGET_NAME_GITHUB_ACTIONS_UPDATE = 'update_github_actions' TARGET_NAME_VENV = 'venv' TARGET_NAME_COMPILE = 'compile' TARGET_NAME_COMPILE_CPP = TARGET_NAME_COMPILE + '_cpp' @@ -55,11 +52,10 @@ def __print_if_clean(environment, message: str): TARGET_NAME_DOC = 'doc' VALID_TARGETS = { - TARGET_NAME_DEPENDENCIES_CHECK, TARGET_NAME_GITHUB_ACTIONS_CHECK, TARGET_NAME_GITHUB_ACTIONS_UPDATE, - TARGET_NAME_VENV, TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP, TARGET_NAME_COMPILE_CYTHON, TARGET_NAME_INSTALL, - TARGET_NAME_INSTALL_CPP, TARGET_NAME_INSTALL_CYTHON, TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS, - TARGET_NAME_TESTS, TARGET_NAME_TESTS_CPP, TARGET_NAME_TESTS_PYTHON, TARGET_NAME_APIDOC, TARGET_NAME_APIDOC_CPP, - TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC + TARGET_NAME_DEPENDENCIES_CHECK, TARGET_NAME_VENV, TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP, + TARGET_NAME_COMPILE_CYTHON, TARGET_NAME_INSTALL, TARGET_NAME_INSTALL_CPP, TARGET_NAME_INSTALL_CYTHON, + TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS, TARGET_NAME_TESTS, TARGET_NAME_TESTS_CPP, + TARGET_NAME_TESTS_PYTHON, TARGET_NAME_APIDOC, TARGET_NAME_APIDOC_CPP, TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC } DEFAULT_TARGET = TARGET_NAME_INSTALL_WHEELS @@ -103,10 +99,6 @@ def __print_if_clean(environment, message: str): # Define target for checking dependency versions... __create_phony_target(env, TARGET_NAME_DEPENDENCIES_CHECK, action=check_dependency_versions) -# Define target for checking and updating the versions of GitHub Actions... -__create_phony_target(env, TARGET_NAME_GITHUB_ACTIONS_CHECK, action=check_github_actions) -__create_phony_target(env, TARGET_NAME_GITHUB_ACTIONS_UPDATE, action=update_github_actions) - # Define target for installing runtime dependencies... target_venv = __create_phony_target(env, TARGET_NAME_VENV, action=install_runtime_dependencies) diff --git a/scons/util/requirements.txt b/scons/util/requirements.txt index 7440aaa37..8d3d821d5 100644 --- a/scons/util/requirements.txt +++ b/scons/util/requirements.txt @@ -2,8 +2,6 @@ build >= 1.2, < 1.3 cython >= 3.0, < 3.1 meson >= 1.6, < 1.7 ninja >= 1.11, < 1.12 -pygithub >= 2.5, < 2.6 -pyyaml >= 6.0, < 6.1 setuptools scons >= 4.8, < 4.9 tabulate >= 0.9, < 0.10 From dfe0362773c7ad67f8cc366adf3751d847263897 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 27 Nov 2024 22:32:33 +0100 Subject: [PATCH 046/114] Remove function "by_name" from class BuildUnit. --- scons/code_style/__init__.py | 2 +- scons/code_style/cpp/__init__.py | 2 +- scons/code_style/markdown/__init__.py | 2 +- scons/code_style/python/__init__.py | 2 +- scons/code_style/yaml/__init__.py | 2 +- scons/dependencies/github/__init__.py | 2 +- scons/python_dependencies.py | 3 ++- scons/util/pip.py | 2 +- scons/util/run.py | 2 +- scons/util/units.py | 26 +++++++++----------------- scons/versioning/__init__.py | 3 ++- 11 files changed, 21 insertions(+), 27 deletions(-) diff --git a/scons/code_style/__init__.py b/scons/code_style/__init__.py index 2b4b8df00..b12b23f14 100644 --- a/scons/code_style/__init__.py +++ b/scons/code_style/__init__.py @@ -10,7 +10,7 @@ from util.targets import TargetBuilder from util.units import BuildUnit -TARGETS = TargetBuilder(BuildUnit.by_name('code_style')) \ +TARGETS = TargetBuilder(BuildUnit('code_style')) \ .add_phony_target('format') \ .depends_on(FORMAT_PYTHON, FORMAT_CPP, FORMAT_MARKDOWN, FORMAT_YAML) \ .nop() \ diff --git a/scons/code_style/cpp/__init__.py b/scons/code_style/cpp/__init__.py index 6f7900299..af53d0039 100644 --- a/scons/code_style/cpp/__init__.py +++ b/scons/code_style/cpp/__init__.py @@ -11,7 +11,7 @@ TEST_FORMAT_CPP = 'test_format_cpp' -TARGETS = TargetBuilder(BuildUnit.by_name('code_style', 'cpp')) \ +TARGETS = TargetBuilder(BuildUnit('code_style', 'cpp')) \ .add_phony_target(FORMAT_CPP).set_runnables(EnforceCppCodeStyle()) \ .add_phony_target(TEST_FORMAT_CPP).set_runnables(CheckCppCodeStyle()) \ .build() diff --git a/scons/code_style/markdown/__init__.py b/scons/code_style/markdown/__init__.py index e5d546e13..6de932b35 100644 --- a/scons/code_style/markdown/__init__.py +++ b/scons/code_style/markdown/__init__.py @@ -11,7 +11,7 @@ TEST_FORMAT_MARKDOWN = 'test_format_md' -TARGETS = TargetBuilder(BuildUnit.by_name('code_style', 'markdown')) \ +TARGETS = TargetBuilder(BuildUnit('code_style', 'markdown')) \ .add_phony_target(FORMAT_MARKDOWN).set_runnables(EnforceMarkdownCodeStyle()) \ .add_phony_target(TEST_FORMAT_MARKDOWN).set_runnables(CheckMarkdownCodeStyle()) \ .build() diff --git a/scons/code_style/python/__init__.py b/scons/code_style/python/__init__.py index b30122718..67811183e 100644 --- a/scons/code_style/python/__init__.py +++ b/scons/code_style/python/__init__.py @@ -12,7 +12,7 @@ TEST_FORMAT_PYTHON = 'test_format_python' -TARGETS = TargetBuilder(BuildUnit.by_name('code_style', 'python')) \ +TARGETS = TargetBuilder(BuildUnit('code_style', 'python')) \ .add_phony_target(FORMAT_PYTHON).set_runnables(EnforcePythonCodeStyle(), EnforceCythonCodeStyle()) \ .add_phony_target(TEST_FORMAT_PYTHON).set_runnables(CheckPythonCodeStyle(), CheckCythonCodeStyle()) \ .build() diff --git a/scons/code_style/yaml/__init__.py b/scons/code_style/yaml/__init__.py index 508bc96c2..6160e5ed6 100644 --- a/scons/code_style/yaml/__init__.py +++ b/scons/code_style/yaml/__init__.py @@ -11,7 +11,7 @@ TEST_FORMAT_YAML = 'test_format_yaml' -TARGETS = TargetBuilder(BuildUnit.by_name('code_style', 'yaml')) \ +TARGETS = TargetBuilder(BuildUnit('code_style', 'yaml')) \ .add_phony_target(FORMAT_YAML).set_runnables(EnforceYamlCodeStyle()) \ .add_phony_target(TEST_FORMAT_YAML).set_runnables(CheckYamlCodeStyle()) \ .build() diff --git a/scons/dependencies/github/__init__.py b/scons/dependencies/github/__init__.py index b6fe86d02..f625be702 100644 --- a/scons/dependencies/github/__init__.py +++ b/scons/dependencies/github/__init__.py @@ -7,7 +7,7 @@ from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit -TARGETS = TargetBuilder(BuildUnit.by_name('dependencies', 'github')) \ +TARGETS = TargetBuilder(BuildUnit('dependencies', 'github')) \ .add_phony_target('check_github_actions').set_runnables(CheckGithubActions()) \ .add_phony_target('update_github_actions').set_runnables(UpdateGithubActions()) \ .build() diff --git a/scons/python_dependencies.py b/scons/python_dependencies.py index 96c132953..d41e9d219 100644 --- a/scons/python_dependencies.py +++ b/scons/python_dependencies.py @@ -10,6 +10,7 @@ from modules_old import ALL_MODULES, CPP_MODULE, PYTHON_MODULE, Module from util.pip import Package, Pip, RequirementsFile +from util.units import BuildUnit @dataclass @@ -55,7 +56,7 @@ def __install_module_dependencies(module: Module, *dependencies: str): def __print_table(header: List[str], rows: List[List[str]]): - Pip().install_packages('tabulate') + Pip(BuildUnit('util')).install_packages('tabulate') # pylint: disable=import-outside-toplevel from tabulate import tabulate print(tabulate(rows, headers=header)) diff --git a/scons/util/pip.py b/scons/util/pip.py index 36e993b99..1d128caf4 100644 --- a/scons/util/pip.py +++ b/scons/util/pip.py @@ -184,7 +184,7 @@ def __install_requirement(requirement: Requirement, dry_run: bool = False): except RuntimeError: Pip.__install_requirement(requirement) - def __init__(self, build_unit: BuildUnit = BuildUnit()): + def __init__(self, build_unit: BuildUnit = BuildUnit('util')): """ :param build_unit: The build unit for which packages should be installed """ diff --git a/scons/util/run.py b/scons/util/run.py index d1be1c827..6c3a6fef3 100644 --- a/scons/util/run.py +++ b/scons/util/run.py @@ -20,7 +20,7 @@ class RunOptions(Command.RunOptions): Allows to customize options for running an external program. """ - def __init__(self, build_unit: BuildUnit = BuildUnit()): + def __init__(self, build_unit: BuildUnit = BuildUnit('util')): """ :param build_unit: The build unit from which the program should be run """ diff --git a/scons/util/units.py b/scons/util/units.py index df82f8fee..13f10bab9 100644 --- a/scons/util/units.py +++ b/scons/util/units.py @@ -3,31 +3,23 @@ Provides classes that provide information about independent units of the build system. """ -from dataclasses import dataclass from os import path -@dataclass class BuildUnit: """ An independent unit of the build system that may come with its own built-time dependencies. - - Attributes: - root_directory: The path to the root directory of this unit - requirements_file: The path to the requirements file that specifies the dependencies required by this unit """ - def __init__(self, root_directory: str = path.join('scons', 'util')): - self.root_directory = root_directory - self.requirements_file: str = path.join(root_directory, 'requirements.txt') - - @staticmethod - def by_name(unit_name: str, *subdirectories: str) -> 'BuildUnit': + def __init__(self, *subdirectories: str): """ - Creates and returns a `BuildUnit` with a specific name. + :param subdirectories: The subdirectories within the build system that lead to the root directory of this unit + """ + self.root_directory = path.join('scons', *subdirectories) - :param unit_name: The name of the build unit - :param subdirectories: Optional subdirectories - :return: The `BuildUnit` that has been created + @property + def requirements_file(self) -> str: + """ + The path to the requirements file that specifies the build-time dependencies of this unit. """ - return BuildUnit(path.join('scons', unit_name, *subdirectories)) + return path.join(self.root_directory, 'requirements.txt') diff --git a/scons/versioning/__init__.py b/scons/versioning/__init__.py index 22abce0c2..d524dc4b4 100644 --- a/scons/versioning/__init__.py +++ b/scons/versioning/__init__.py @@ -2,12 +2,13 @@ Defines build targets for updating the project's version and changelog. """ from util.targets import PhonyTarget, TargetBuilder +from util.units import BuildUnit from versioning.changelog import print_latest_changelog, update_changelog_bugfix, update_changelog_feature, \ update_changelog_main, validate_changelog_bugfix, validate_changelog_feature, validate_changelog_main from versioning.versioning import apply_development_version, increment_development_version, increment_major_version, \ increment_minor_version, increment_patch_version, print_current_version, reset_development_version -TARGETS = TargetBuilder() \ +TARGETS = TargetBuilder(BuildUnit('util')) \ .add_phony_target('increment_development_version').set_functions(increment_development_version) \ .add_phony_target('reset_development_version').set_functions(reset_development_version) \ .add_phony_target('apply_development_version').set_functions(apply_development_version) \ From 8036d097c783f0328b6ad722e93d5c364c23e5cb Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 27 Nov 2024 23:39:27 +0100 Subject: [PATCH 047/114] Add function "for_build_unit" to class Pip. --- build | 2 +- build.bat | 2 +- scons/dependencies/github/pygithub.py | 2 +- scons/dependencies/github/pyyaml.py | 2 +- scons/python_dependencies.py | 4 ++-- scons/util/pip.py | 16 +++++++++++++--- scons/util/run.py | 2 +- scons/util/table.py | 2 +- 8 files changed, 21 insertions(+), 11 deletions(-) diff --git a/build b/build index bf7fbff16..f5c4091ff 100755 --- a/build +++ b/build @@ -22,7 +22,7 @@ fi if [ -d "$VENV_DIR" ]; then . $VENV_DIR/bin/activate - python3 -c "import sys; sys.path.append('$SCONS_DIR'); from util.pip import Pip; Pip().install_packages('scons')" + python3 -c "import sys; sys.path.append('$SCONS_DIR'); from util.pip import Pip; Pip.for_build_unit().install_packages('scons')" scons --silent --file $SCONS_DIR/sconstruct.py $@ deactivate fi diff --git a/build.bat b/build.bat index 1b006dc3a..d5f247836 100644 --- a/build.bat +++ b/build.bat @@ -20,7 +20,7 @@ if not exist "%VENV_DIR%" ( if exist "%VENV_DIR%" ( call %VENV_DIR%\Scripts\activate || exit - .\%VENV_DIR%\Scripts\python -c "import sys;sys.path.append('%SCONS_DIR%');from util.pip import Pip;Pip().install_packages('scons')" || exit + .\%VENV_DIR%\Scripts\python -c "import sys;sys.path.append('%SCONS_DIR%');from util.pip import Pip;Pip.for_build_unit().install_packages('scons')" || exit .\%VENV_DIR%\Scripts\python -m SCons --silent --file %SCONS_DIR%\sconstruct.py %* || exit call deactivate || exit ) diff --git a/scons/dependencies/github/pygithub.py b/scons/dependencies/github/pygithub.py index 781ef0065..7eba4170f 100644 --- a/scons/dependencies/github/pygithub.py +++ b/scons/dependencies/github/pygithub.py @@ -50,7 +50,7 @@ def __init__(self, build_unit: BuildUnit): """ :param build_unit: The build unit to access the GitHub API from """ - Pip(build_unit).install_packages('pygithub') + Pip.for_build_unit(build_unit).install_packages('pygithub') self.authentication = None def set_token(self, token: Optional[str]) -> 'GithubApi': diff --git a/scons/dependencies/github/pyyaml.py b/scons/dependencies/github/pyyaml.py index 7513d363b..8cdb03906 100644 --- a/scons/dependencies/github/pyyaml.py +++ b/scons/dependencies/github/pyyaml.py @@ -29,7 +29,7 @@ def yaml_dict(self) -> Dict: """ A dictionary that stores the content of the YAML file. """ - Pip(self.build_unit).install_packages('pyyaml') + Pip.for_build_unit(self.build_unit).install_packages('pyyaml') # pylint: disable=import-outside-toplevel import yaml with read_file(self.file) as file: diff --git a/scons/python_dependencies.py b/scons/python_dependencies.py index d41e9d219..be158427b 100644 --- a/scons/python_dependencies.py +++ b/scons/python_dependencies.py @@ -52,11 +52,11 @@ def __install_module_dependencies(module: Module, *dependencies: str): requirements_file = module.requirements_file if path.isfile(requirements_file): - Pip(module).install_packages(*dependencies) + Pip.for_build_unit(module).install_packages(*dependencies) def __print_table(header: List[str], rows: List[List[str]]): - Pip(BuildUnit('util')).install_packages('tabulate') + Pip.for_build_unit(BuildUnit('util')).install_packages('tabulate') # pylint: disable=import-outside-toplevel from tabulate import tabulate print(tabulate(rows, headers=header)) diff --git a/scons/util/pip.py b/scons/util/pip.py index 1d128caf4..452c7800d 100644 --- a/scons/util/pip.py +++ b/scons/util/pip.py @@ -184,11 +184,21 @@ def __install_requirement(requirement: Requirement, dry_run: bool = False): except RuntimeError: Pip.__install_requirement(requirement) - def __init__(self, build_unit: BuildUnit = BuildUnit('util')): + def __init__(self, requirements_file: str): """ - :param build_unit: The build unit for which packages should be installed + :param requirements_file: The requirements file that specifies the version of the packages to be installed """ - self.requirements_file = RequirementsFile(build_unit.requirements_file) + self.requirements_file = RequirementsFile(requirements_file) + + @staticmethod + def for_build_unit(build_unit: BuildUnit = BuildUnit('util')): + """ + Creates and returns a new `Pip` instance for installing packages for a specific build unit. + + :param build_unit: The build unit for which packages should be installed + :return: The `Pip` instance that has been created + """ + return Pip(build_unit.requirements_file) def install_packages(self, *package_names: str, accept_missing: bool = False): """ diff --git a/scons/util/run.py b/scons/util/run.py index 6c3a6fef3..bdf10f18a 100644 --- a/scons/util/run.py +++ b/scons/util/run.py @@ -36,7 +36,7 @@ def run(self, command: Command, capture_output: bool) -> CompletedProcess: dependencies.append(command.command) dependencies.extend(self.dependencies) - Pip(self.build_unit).install_packages(*dependencies) + Pip.for_build_unit(self.build_unit).install_packages(*dependencies) return super().run(command, capture_output) def __init__(self, program: str, *arguments: str): diff --git a/scons/util/table.py b/scons/util/table.py index ee3113841..daafedb47 100644 --- a/scons/util/table.py +++ b/scons/util/table.py @@ -39,7 +39,7 @@ def sort_rows(self, column_index: int, *additional_column_indices: int): self.rows.sort(key=lambda row: ([row[i] for i in [column_index] + list(additional_column_indices)])) def __str__(self) -> str: - Pip(self.build_unit).install_packages('tabulate') + Pip.for_build_unit(self.build_unit).install_packages('tabulate') # pylint: disable=import-outside-toplevel from tabulate import tabulate return tabulate(self.rows, headers=self.headers) From 026f3aefa808e95d5e5cac0292189a0fd478c305 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 27 Nov 2024 23:43:22 +0100 Subject: [PATCH 048/114] Add function "install_all_packages" to class Pip. --- scons/util/pip.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scons/util/pip.py b/scons/util/pip.py index 452c7800d..ff826810b 100644 --- a/scons/util/pip.py +++ b/scons/util/pip.py @@ -202,7 +202,7 @@ def for_build_unit(build_unit: BuildUnit = BuildUnit('util')): def install_packages(self, *package_names: str, accept_missing: bool = False): """ - Installs one or several dependencies. + Installs one or several dependencies in the requirements file. :param package_names: The names of the packages that should be installed :param accept_missing: False, if an error should be raised if a package is not listed in the requirements file, @@ -213,3 +213,12 @@ def install_packages(self, *package_names: str, accept_missing: bool = False): for requirement in requirements: self.__install_requirement(requirement, dry_run=True) + + def install_all_packages(self): + """ + Installs all dependencies in the requirements file. + """ + requirements = self.requirements_file.requirements + + for requirement in requirements: + self.__install_requirement(requirement, dry_run=True) From c1fb35301510bb6982bbb1953a485aa23c944104 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 28 Nov 2024 00:30:49 +0100 Subject: [PATCH 049/114] Add function "filter_by_name" to class FileSearch. --- scons/code_style/modules.py | 2 +- scons/dependencies/github/actions.py | 2 +- scons/sconstruct.py | 27 +++++++++++---------------- scons/util/files.py | 16 +++++++++++++--- 4 files changed, 26 insertions(+), 21 deletions(-) diff --git a/scons/code_style/modules.py b/scons/code_style/modules.py index cf545255e..1852c7bf1 100644 --- a/scons/code_style/modules.py +++ b/scons/code_style/modules.py @@ -49,4 +49,4 @@ def find_source_files(self) -> List[str]: :return: A list that contains the paths of the source files that have been found """ - return self.file_search.set_languages(self.language).list(self.root_directory) + return self.file_search.filter_by_language(self.language).list(self.root_directory) diff --git a/scons/dependencies/github/actions.py b/scons/dependencies/github/actions.py index f78e9264c..ac6d114ea 100644 --- a/scons/dependencies/github/actions.py +++ b/scons/dependencies/github/actions.py @@ -293,7 +293,7 @@ def workflows(self) -> Set[Workflow]: """ workflows = set() - for workflow_file in FileSearch().set_languages(Language.YAML).list(self.workflow_directory): + for workflow_file in FileSearch().filter_by_language(Language.YAML).list(self.workflow_directory): print('Searching for GitHub Actions in workflow "' + workflow_file + '"...') workflows.add(Workflow(self.build_unit, workflow_file)) diff --git a/scons/sconstruct.py b/scons/sconstruct.py index cc3bf25f8..4dbe2fead 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -14,7 +14,7 @@ from packaging import build_python_wheel, install_python_wheels from python_dependencies import check_dependency_versions, install_runtime_dependencies from testing import tests_cpp, tests_python -from util.files import DirectorySearch +from util.files import FileSearch from util.format import format_iterable from util.modules import Module, ModuleRegistry from util.reflection import import_source_file @@ -61,28 +61,23 @@ def __print_if_clean(environment, message: str): DEFAULT_TARGET = TARGET_NAME_INSTALL_WHEELS # Register modules... +init_files = FileSearch().set_recursive(True).filter_by_name('__init__.py').list(BUILD_MODULE.root_dir) module_registry = ModuleRegistry() -for subdirectory in DirectorySearch().set_recursive(True).list(BUILD_MODULE.root_dir): - init_file = path.join(subdirectory, '__init__.py') - - if path.isfile(init_file): - for module in getattr(import_source_file(init_file), 'MODULES', []): - if isinstance(module, Module): - module_registry.register(module) +for init_file in init_files: + for module in getattr(import_source_file(init_file), 'MODULES', []): + if isinstance(module, Module): + module_registry.register(module) # Register build targets... target_registry = TargetRegistry(module_registry) env = target_registry.environment -for subdirectory in DirectorySearch().set_recursive(True).list(BUILD_MODULE.root_dir): - init_file = path.join(subdirectory, '__init__.py') - - if path.isfile(init_file): - for target in getattr(import_source_file(init_file), 'TARGETS', []): - if isinstance(target, Target): - target_registry.add_target(target) - VALID_TARGETS.add(target.name) +for init_file in init_files: + for target in getattr(import_source_file(init_file), 'TARGETS', []): + if isinstance(target, Target): + target_registry.add_target(target) + VALID_TARGETS.add(target.name) target_registry.register() diff --git a/scons/util/files.py b/scons/util/files.py index 01c2fb219..f2f24da39 100644 --- a/scons/util/files.py +++ b/scons/util/files.py @@ -130,7 +130,17 @@ def set_hidden(self, hidden: bool) -> 'FileSearch': self.hidden = hidden return self - def set_suffixes(self, *suffixes: str) -> 'FileSearch': + def filter_by_name(self, *names: str) -> 'FileSearch': + """ + Sets the names of the files that should be included. + + :param names: The names of the files that should be included (including their suffix) + :return: The `FileSearch` itself + """ + self.file_patterns = {name for name in names} + return self + + def filter_by_suffix(self, *suffixes: str) -> 'FileSearch': """ Sets the suffixes of the files that should be included. @@ -140,14 +150,14 @@ def set_suffixes(self, *suffixes: str) -> 'FileSearch': self.file_patterns = {'*.' + suffix for suffix in suffixes} return self - def set_languages(self, *languages: Language) -> 'FileSearch': + def filter_by_language(self, *languages: Language) -> 'FileSearch': """ Sets the suffixes of the files that should be included. :param languages: The languages of the files that should be included :return: The `FileSearch` itself """ - return self.set_suffixes(*reduce(lambda aggr, language: aggr | language.value, languages, set())) + return self.filter_by_suffix(*reduce(lambda aggr, language: aggr | language.value, languages, set())) def list(self, *directories: str) -> List[str]: """ From 88f88cf03cd6865b7ef8628deb6fe70b5751a094 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 27 Nov 2024 23:16:26 +0100 Subject: [PATCH 050/114] Add class PythonDependencyModule. --- scons/dependencies/python/__init__.py | 0 scons/dependencies/python/modules.py | 57 +++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 scons/dependencies/python/__init__.py create mode 100644 scons/dependencies/python/modules.py diff --git a/scons/dependencies/python/__init__.py b/scons/dependencies/python/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scons/dependencies/python/modules.py b/scons/dependencies/python/modules.py new file mode 100644 index 000000000..6b50100a3 --- /dev/null +++ b/scons/dependencies/python/modules.py @@ -0,0 +1,57 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that provide access to Python requirements files that belong to individual modules. +""" +from enum import Enum, auto +from typing import List + +from util.files import FileSearch +from util.modules import Module + + +class DependencyType(Enum): + """ + The type of the Python dependencies. + """ + BUILD_TIME = auto() + RUNTIME = auto() + + +class PythonDependencyModule(Module): + """ + A module that contains source code that comes with Python dependencies. + """ + + class Filter(Module.Filter): + """ + A filter that matches modules that contain source that comes with Python dependencies. + """ + + def __init__(self, *dependency_types: DependencyType): + """ + :param dependency_types: The type of the Python dependencies of the modules to be matched or None, if no + restrictions should be imposed on the types of dependencies + """ + self.dependency_types = set(dependency_types) + + def matches(self, module: Module) -> bool: + return isinstance(module, PythonDependencyModule) and (not self.dependency_types + or module.dependency_type in self.dependency_types) + + def __init__(self, dependency_type: DependencyType, root_directory: str, file_search: FileSearch = FileSearch()): + """ + :param dependency_type: The type of the Python dependencies + :param root_directory: The path to the module's root directory + :param file_search: The `FileSearch` that should be used to search for requirements files + """ + self.dependency_type = dependency_type + self.root_directory = root_directory + + def find_requirements_files(self) -> List[str]: + """ + Finds and returns all requirements files that belong to the module. + + :return: A list that contains the paths of the requirements files that have been found + """ + return self.file_search.filter_by_name('requirements.txt').list(self.root_directory) From e237e7d6766c7b81bf2ba43b3b707cdf0d9a8d75 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 27 Nov 2024 23:19:57 +0100 Subject: [PATCH 051/114] Dynamically register targets and modules for installing Python runtime dependencies. --- scons/dependencies/python/__init__.py | 14 ++++++++++++++ scons/dependencies/python/modules.py | 1 + scons/dependencies/python/targets.py | 21 +++++++++++++++++++++ scons/modules/__init__.py | 10 +++++++--- scons/python_dependencies.py | 10 +--------- scons/sconstruct.py | 16 ++++++---------- 6 files changed, 50 insertions(+), 22 deletions(-) create mode 100644 scons/dependencies/python/targets.py diff --git a/scons/dependencies/python/__init__.py b/scons/dependencies/python/__init__.py index e69de29bb..ee7a02dc1 100644 --- a/scons/dependencies/python/__init__.py +++ b/scons/dependencies/python/__init__.py @@ -0,0 +1,14 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Defines targets for updating the Python runtime dependencies that are required by the project's source code. +""" +from dependencies.python.targets import InstallRuntimeDependencies +from util.targets import PhonyTarget, TargetBuilder +from util.units import BuildUnit + +VENV = 'venv' + +TARGETS = TargetBuilder(BuildUnit('dependencies', 'python')) \ + .add_phony_target(VENV).set_runnables(InstallRuntimeDependencies()) \ + .build() diff --git a/scons/dependencies/python/modules.py b/scons/dependencies/python/modules.py index 6b50100a3..b56cb4b62 100644 --- a/scons/dependencies/python/modules.py +++ b/scons/dependencies/python/modules.py @@ -47,6 +47,7 @@ def __init__(self, dependency_type: DependencyType, root_directory: str, file_se """ self.dependency_type = dependency_type self.root_directory = root_directory + self.file_search = file_search def find_requirements_files(self) -> List[str]: """ diff --git a/scons/dependencies/python/targets.py b/scons/dependencies/python/targets.py new file mode 100644 index 000000000..2fa7a9759 --- /dev/null +++ b/scons/dependencies/python/targets.py @@ -0,0 +1,21 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Implements targets for installing runtime requirements that are required by the project's source code. +""" +from dependencies.python.modules import DependencyType, PythonDependencyModule +from util.modules import ModuleRegistry +from util.pip import Pip +from util.targets import PhonyTarget +from util.units import BuildUnit + + +class InstallRuntimeDependencies(PhonyTarget.Runnable): + """ + Installs all runtime dependencies that are required by the project's source code. + """ + + def run(self, _: BuildUnit, modules: ModuleRegistry): + for module in modules.lookup(PythonDependencyModule.Filter(DependencyType.RUNTIME)): + for requirements_file in module.find_requirements_files(): + Pip(requirements_file).install_all_packages() diff --git a/scons/modules/__init__.py b/scons/modules/__init__.py index 41da666b1..6da3e6424 100644 --- a/scons/modules/__init__.py +++ b/scons/modules/__init__.py @@ -3,9 +3,8 @@ Defines modules to be dealt with by the build system. """ -from os import path - from code_style.modules import CodeModule +from dependencies.python.modules import DependencyType, PythonDependencyModule from util.files import FileSearch from util.languages import Language @@ -24,5 +23,10 @@ CodeModule(Language.CYTHON, 'python', FileSearch().set_recursive(True).exclude_subdirectories_by_name('build')), CodeModule(Language.CPP, 'cpp', - FileSearch().set_recursive(True).exclude_subdirectories_by_name('build')) + FileSearch().set_recursive(True).exclude_subdirectories_by_name('build')), + PythonDependencyModule(DependencyType.BUILD_TIME, 'scons', + FileSearch().set_recursive(True)), + PythonDependencyModule(DependencyType.BUILD_TIME, 'doc', + FileSearch().set_recursive(True)), + PythonDependencyModule(DependencyType.RUNTIME, 'python') ] diff --git a/scons/python_dependencies.py b/scons/python_dependencies.py index be158427b..bf4424e6f 100644 --- a/scons/python_dependencies.py +++ b/scons/python_dependencies.py @@ -8,7 +8,7 @@ from os import path from typing import List -from modules_old import ALL_MODULES, CPP_MODULE, PYTHON_MODULE, Module +from modules_old import ALL_MODULES, Module from util.pip import Package, Pip, RequirementsFile from util.units import BuildUnit @@ -62,14 +62,6 @@ def __print_table(header: List[str], rows: List[List[str]]): print(tabulate(rows, headers=header)) -def install_runtime_dependencies(**_): - """ - Installs all runtime dependencies that are required by the Python and C++ module. - """ - __install_module_dependencies(PYTHON_MODULE) - __install_module_dependencies(CPP_MODULE) - - def check_dependency_versions(**_): """ Installs all dependencies used by the project and checks for outdated dependencies. diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 4dbe2fead..a1c38670b 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -12,7 +12,7 @@ from documentation import apidoc_cpp, apidoc_cpp_tocfile, apidoc_python, apidoc_python_tocfile, doc from modules_old import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE from packaging import build_python_wheel, install_python_wheels -from python_dependencies import check_dependency_versions, install_runtime_dependencies +from python_dependencies import check_dependency_versions from testing import tests_cpp, tests_python from util.files import FileSearch from util.format import format_iterable @@ -34,7 +34,6 @@ def __print_if_clean(environment, message: str): # Define target names... TARGET_NAME_DEPENDENCIES_CHECK = 'check_dependencies' -TARGET_NAME_VENV = 'venv' TARGET_NAME_COMPILE = 'compile' TARGET_NAME_COMPILE_CPP = TARGET_NAME_COMPILE + '_cpp' TARGET_NAME_COMPILE_CYTHON = TARGET_NAME_COMPILE + '_cython' @@ -52,10 +51,10 @@ def __print_if_clean(environment, message: str): TARGET_NAME_DOC = 'doc' VALID_TARGETS = { - TARGET_NAME_DEPENDENCIES_CHECK, TARGET_NAME_VENV, TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP, - TARGET_NAME_COMPILE_CYTHON, TARGET_NAME_INSTALL, TARGET_NAME_INSTALL_CPP, TARGET_NAME_INSTALL_CYTHON, - TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS, TARGET_NAME_TESTS, TARGET_NAME_TESTS_CPP, - TARGET_NAME_TESTS_PYTHON, TARGET_NAME_APIDOC, TARGET_NAME_APIDOC_CPP, TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC + TARGET_NAME_DEPENDENCIES_CHECK, TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP, TARGET_NAME_COMPILE_CYTHON, + TARGET_NAME_INSTALL, TARGET_NAME_INSTALL_CPP, TARGET_NAME_INSTALL_CYTHON, TARGET_NAME_BUILD_WHEELS, + TARGET_NAME_INSTALL_WHEELS, TARGET_NAME_TESTS, TARGET_NAME_TESTS_CPP, TARGET_NAME_TESTS_PYTHON, TARGET_NAME_APIDOC, + TARGET_NAME_APIDOC_CPP, TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC } DEFAULT_TARGET = TARGET_NAME_INSTALL_WHEELS @@ -94,13 +93,10 @@ def __print_if_clean(environment, message: str): # Define target for checking dependency versions... __create_phony_target(env, TARGET_NAME_DEPENDENCIES_CHECK, action=check_dependency_versions) -# Define target for installing runtime dependencies... -target_venv = __create_phony_target(env, TARGET_NAME_VENV, action=install_runtime_dependencies) - # Define targets for compiling the C++ and Cython code... env.Command(CPP_MODULE.build_dir, None, action=setup_cpp) target_compile_cpp = __create_phony_target(env, TARGET_NAME_COMPILE_CPP, action=compile_cpp) -env.Depends(target_compile_cpp, [target_venv, CPP_MODULE.build_dir]) +env.Depends(target_compile_cpp, ['target_venv', CPP_MODULE.build_dir]) env.Command(PYTHON_MODULE.build_dir, None, action=setup_cython) target_compile_cython = __create_phony_target(env, TARGET_NAME_COMPILE_CYTHON, action=compile_cython) From b75dc1d0fef0a7c6c808ab9b8b7326839f7cae50 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 28 Nov 2024 00:31:16 +0100 Subject: [PATCH 052/114] Move __init__.py file. --- scons/{modules => }/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scons/{modules => }/__init__.py (100%) diff --git a/scons/modules/__init__.py b/scons/__init__.py similarity index 100% rename from scons/modules/__init__.py rename to scons/__init__.py From 5c4bde0c89ec79de1e4a6bf471a4db5da59db39f Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 28 Nov 2024 01:28:05 +0100 Subject: [PATCH 053/114] Add function "list_outdated_dependencies" to class Pip. --- scons/util/io.py | 3 ++ scons/util/pip.py | 84 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/scons/util/io.py b/scons/util/io.py index 612f7fd4c..5dc52f8ae 100644 --- a/scons/util/io.py +++ b/scons/util/io.py @@ -56,3 +56,6 @@ def write_lines(self, lines: List[str]): file.writelines(lines) del self.lines + + def __str__(self) -> str: + return self.file diff --git a/scons/util/pip.py b/scons/util/pip.py index ff826810b..d9da6e676 100644 --- a/scons/util/pip.py +++ b/scons/util/pip.py @@ -74,6 +74,25 @@ def __hash__(self) -> int: return hash(self.package) +@dataclass +class Dependency: + """ + Provides information about a dependency. + + Attributes: + installed: The version of the dependency that is currently installed + latest: The latest version of the dependency + """ + installed: Requirement + latest: Requirement + + def __eq__(self, other: 'Dependency') -> bool: + return self.installed == other.installed + + def __hash__(self) -> int: + return hash(self.installed) + + class RequirementsFile(TextFile): """ Represents a specific requirements.txt file. @@ -96,7 +115,7 @@ def requirements(self) -> Set[Requirement]: """ return set(self.requirements_by_package.values()) - def lookup(self, *packages: Package, accept_missing: bool = False) -> Set[Requirement]: + def lookup_requirements(self, *packages: Package, accept_missing: bool = False) -> Set[Requirement]: """ Looks up the requirements for given packages in the requirements file. @@ -117,6 +136,18 @@ def lookup(self, *packages: Package, accept_missing: bool = False) -> Set[Requir return requirements + def lookup_requirement(self, package: Package, accept_missing: bool = False) -> Optional[Requirement]: + """ + Looks up the requirement for a given package in the requirements file. + + :param package: The package that should be looked up + :param accept_missing: False, if an error should be raised if the package is not listed in the requirements + file, True, if it should simply be ignored + :return: The requirement for the given package + """ + requirements = self.lookup_requirements(package, accept_missing=accept_missing) + return requirements.pop() if requirements else None + class Pip: """ @@ -148,6 +179,18 @@ def __init__(self, requirement: Requirement, dry_run: bool = False): super().__init__('install', str(requirement), '--upgrade', '--upgrade-strategy', 'eager', '--prefer-binary') self.add_conditional_arguments(dry_run, '--dry-run') + class ListCommand(Command): + """ + Allows to list information about installed packages via the command `pip list`. + """ + + def __init__(self, outdated: bool = False): + """ + :param outdated: True, if only outdated packages should be listed, False otherwise + """ + super().__init__('list') + self.add_conditional_arguments(outdated, '--outdated') + @staticmethod def __would_install_requirement(requirement: Requirement, stdout: str) -> bool: prefix = 'Would install' @@ -209,7 +252,7 @@ def install_packages(self, *package_names: str, accept_missing: bool = False): True, if it should simply be ignored """ packages = [Package(package_name) for package_name in package_names] - requirements = self.requirements_file.lookup(*packages, accept_missing=accept_missing) + requirements = self.requirements_file.lookup_requirements(*packages, accept_missing=accept_missing) for requirement in requirements: self.__install_requirement(requirement, dry_run=True) @@ -218,7 +261,44 @@ def install_all_packages(self): """ Installs all dependencies in the requirements file. """ + print('Installing all dependencies in requirements file "' + str(self.requirements_file) + '"...') requirements = self.requirements_file.requirements for requirement in requirements: self.__install_requirement(requirement, dry_run=True) + + def list_outdated_dependencies(self) -> Set[Dependency]: + self.install_all_packages() + + print('Checking for outdated dependencies in requirements file "' + str(self.requirements_file) + '"...') + stdout = Pip.ListCommand(outdated=True).print_command(False).capture_output() + stdout_lines = stdout.strip().split('\n') + i = 0 + + for line in stdout_lines: + i += 1 + + if line.startswith('----'): + break + + outdated_dependencies = set() + + for line in stdout_lines[i:]: + parts = line.split() + + if len(parts) < 3: + raise ValueError( + 'Output of command "pip list" is expected to be a table with at least three columns, but got:' + + line) + + package = Package(parts[0]) + requirement = self.requirements_file.lookup_requirement(package, accept_missing=True) + + if requirement and requirement.version: + installed_version = parts[1] + latest_version = parts[2] + outdated_dependencies.add( + Dependency(installed=Requirement(package, version=installed_version), + latest=Requirement(package, version=latest_version))) + + return outdated_dependencies From 42279245dbc42e460d54733db64fee5b25bab506 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 30 Nov 2024 20:13:31 +0100 Subject: [PATCH 054/114] Dynamically register target for checking dependency versions. --- scons/dependencies/python/__init__.py | 3 +- scons/dependencies/python/requirements.txt | 1 + scons/dependencies/python/targets.py | 31 +++++++ scons/python_dependencies.py | 94 ---------------------- scons/sconstruct.py | 13 +-- scons/util/requirements.txt | 1 - 6 files changed, 38 insertions(+), 105 deletions(-) create mode 100644 scons/dependencies/python/requirements.txt delete mode 100644 scons/python_dependencies.py diff --git a/scons/dependencies/python/__init__.py b/scons/dependencies/python/__init__.py index ee7a02dc1..f59979d88 100644 --- a/scons/dependencies/python/__init__.py +++ b/scons/dependencies/python/__init__.py @@ -3,7 +3,7 @@ Defines targets for updating the Python runtime dependencies that are required by the project's source code. """ -from dependencies.python.targets import InstallRuntimeDependencies +from dependencies.python.targets import CheckPythonDependencies, InstallRuntimeDependencies from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -11,4 +11,5 @@ TARGETS = TargetBuilder(BuildUnit('dependencies', 'python')) \ .add_phony_target(VENV).set_runnables(InstallRuntimeDependencies()) \ + .add_phony_target('check_dependencies').set_runnables(CheckPythonDependencies()) \ .build() diff --git a/scons/dependencies/python/requirements.txt b/scons/dependencies/python/requirements.txt new file mode 100644 index 000000000..8cd36735d --- /dev/null +++ b/scons/dependencies/python/requirements.txt @@ -0,0 +1 @@ +tabulate >= 0.9, < 0.10 diff --git a/scons/dependencies/python/targets.py b/scons/dependencies/python/targets.py index 2fa7a9759..f86ba4006 100644 --- a/scons/dependencies/python/targets.py +++ b/scons/dependencies/python/targets.py @@ -6,6 +6,7 @@ from dependencies.python.modules import DependencyType, PythonDependencyModule from util.modules import ModuleRegistry from util.pip import Pip +from util.table import Table from util.targets import PhonyTarget from util.units import BuildUnit @@ -19,3 +20,33 @@ def run(self, _: BuildUnit, modules: ModuleRegistry): for module in modules.lookup(PythonDependencyModule.Filter(DependencyType.RUNTIME)): for requirements_file in module.find_requirements_files(): Pip(requirements_file).install_all_packages() + + +class CheckPythonDependencies(PhonyTarget.Runnable): + """ + Installs all Python dependencies used by the project and checks for outdated ones. + """ + + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + outdated_dependencies_by_requirements_file = {} + + for module in modules.lookup(PythonDependencyModule.Filter()): + for requirements_file in module.find_requirements_files(): + outdated_dependencies = Pip(requirements_file).list_outdated_dependencies() + + if outdated_dependencies: + outdated_dependencies_by_requirements_file[requirements_file] = outdated_dependencies + + if outdated_dependencies_by_requirements_file: + table = Table(build_unit, 'Requirements file', 'Dependency', 'Installed version', 'Latest version') + + for requirements_file, outdated_dependencies in outdated_dependencies_by_requirements_file.items(): + for dependency in outdated_dependencies: + table.add_row(requirements_file, str(dependency.installed.package), dependency.installed.version, + dependency.latest.version) + + table.sort_rows(0, 1) + print('The following dependencies are outdated:\n') + print(str(table)) + else: + print('All dependencies are up-to-date!') diff --git a/scons/python_dependencies.py b/scons/python_dependencies.py deleted file mode 100644 index bf4424e6f..000000000 --- a/scons/python_dependencies.py +++ /dev/null @@ -1,94 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Provides utility functions for dealing with dependencies. -""" - -from dataclasses import dataclass -from os import path -from typing import List - -from modules_old import ALL_MODULES, Module -from util.pip import Package, Pip, RequirementsFile -from util.units import BuildUnit - - -@dataclass -class Dependency: - """ - Provides information about an installed dependency. - - Attributes: - package: The Python package - installed_version: The version of the dependency that is currently installed - latest_version: The latest version of the dependency - """ - package: Package - installed_version: str - latest_version: str - - -def __find_outdated_dependencies() -> List[Dependency]: - stdout = Pip.Command('list', '--outdated').print_command(False).capture_output() - stdout_lines = stdout.strip().split('\n') - i = 0 - - for line in stdout_lines: - i += 1 - - if line.startswith('----'): - break - - outdated_dependencies = [] - - for line in stdout_lines[i:]: - parts = line.split() - outdated_dependencies.append(Dependency(Package(parts[0]), installed_version=parts[1], latest_version=parts[2])) - - return outdated_dependencies - - -def __install_module_dependencies(module: Module, *dependencies: str): - requirements_file = module.requirements_file - - if path.isfile(requirements_file): - Pip.for_build_unit(module).install_packages(*dependencies) - - -def __print_table(header: List[str], rows: List[List[str]]): - Pip.for_build_unit(BuildUnit('util')).install_packages('tabulate') - # pylint: disable=import-outside-toplevel - from tabulate import tabulate - print(tabulate(rows, headers=header)) - - -def check_dependency_versions(**_): - """ - Installs all dependencies used by the project and checks for outdated dependencies. - """ - print('Installing all dependencies...') - for module in ALL_MODULES: - __install_module_dependencies(module) - - print('Checking for outdated dependencies...') - outdated_dependencies = __find_outdated_dependencies() - rows = [] - - for dependency in outdated_dependencies: - for module in ALL_MODULES: - requirements_file = module.requirements_file - - if path.isfile(requirements_file): - requirements = RequirementsFile(requirements_file).lookup(dependency.package, accept_missing=True) - - if requirements and requirements.pop().version: - rows.append([str(dependency.package), dependency.installed_version, dependency.latest_version]) - break - - if rows: - rows.sort(key=lambda row: row[0]) - header = ['Dependency', 'Installed version', 'Latest version'] - print('The following dependencies are outdated:\n') - __print_table(header=header, rows=rows) - else: - print('All dependencies are up-to-date!') diff --git a/scons/sconstruct.py b/scons/sconstruct.py index a1c38670b..2b085e19f 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -12,7 +12,6 @@ from documentation import apidoc_cpp, apidoc_cpp_tocfile, apidoc_python, apidoc_python_tocfile, doc from modules_old import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE from packaging import build_python_wheel, install_python_wheels -from python_dependencies import check_dependency_versions from testing import tests_cpp, tests_python from util.files import FileSearch from util.format import format_iterable @@ -33,7 +32,6 @@ def __print_if_clean(environment, message: str): # Define target names... -TARGET_NAME_DEPENDENCIES_CHECK = 'check_dependencies' TARGET_NAME_COMPILE = 'compile' TARGET_NAME_COMPILE_CPP = TARGET_NAME_COMPILE + '_cpp' TARGET_NAME_COMPILE_CYTHON = TARGET_NAME_COMPILE + '_cython' @@ -51,10 +49,10 @@ def __print_if_clean(environment, message: str): TARGET_NAME_DOC = 'doc' VALID_TARGETS = { - TARGET_NAME_DEPENDENCIES_CHECK, TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP, TARGET_NAME_COMPILE_CYTHON, - TARGET_NAME_INSTALL, TARGET_NAME_INSTALL_CPP, TARGET_NAME_INSTALL_CYTHON, TARGET_NAME_BUILD_WHEELS, - TARGET_NAME_INSTALL_WHEELS, TARGET_NAME_TESTS, TARGET_NAME_TESTS_CPP, TARGET_NAME_TESTS_PYTHON, TARGET_NAME_APIDOC, - TARGET_NAME_APIDOC_CPP, TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC + TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP, TARGET_NAME_COMPILE_CYTHON, TARGET_NAME_INSTALL, + TARGET_NAME_INSTALL_CPP, TARGET_NAME_INSTALL_CYTHON, TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS, + TARGET_NAME_TESTS, TARGET_NAME_TESTS_CPP, TARGET_NAME_TESTS_PYTHON, TARGET_NAME_APIDOC, TARGET_NAME_APIDOC_CPP, + TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC } DEFAULT_TARGET = TARGET_NAME_INSTALL_WHEELS @@ -90,9 +88,6 @@ def __print_if_clean(environment, message: str): # Create temporary file ".sconsign.dblite" in the build directory... env.SConsignFile(name=path.relpath(path.join(BUILD_MODULE.build_dir, '.sconsign'), BUILD_MODULE.root_dir)) -# Define target for checking dependency versions... -__create_phony_target(env, TARGET_NAME_DEPENDENCIES_CHECK, action=check_dependency_versions) - # Define targets for compiling the C++ and Cython code... env.Command(CPP_MODULE.build_dir, None, action=setup_cpp) target_compile_cpp = __create_phony_target(env, TARGET_NAME_COMPILE_CPP, action=compile_cpp) diff --git a/scons/util/requirements.txt b/scons/util/requirements.txt index 8d3d821d5..4ba9aa2ea 100644 --- a/scons/util/requirements.txt +++ b/scons/util/requirements.txt @@ -4,6 +4,5 @@ meson >= 1.6, < 1.7 ninja >= 1.11, < 1.12 setuptools scons >= 4.8, < 4.9 -tabulate >= 0.9, < 0.10 unittest-xml-reporting >= 3.2, < 3.3 wheel >= 0.45, < 0.46 From 372434d72e8bcc85fc4c5b3733e41c451f8d20fa Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 30 Nov 2024 23:37:25 +0100 Subject: [PATCH 055/114] Add class BuildTarget. --- scons/util/targets.py | 181 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 159 insertions(+), 22 deletions(-) diff --git a/scons/util/targets.py b/scons/util/targets.py index 10eb717af..593393fcf 100644 --- a/scons/util/targets.py +++ b/scons/util/targets.py @@ -6,7 +6,9 @@ import sys from abc import ABC, abstractmethod +from functools import reduce from typing import Any, Callable, List, Set +from uuid import uuid4 from util.modules import ModuleRegistry from util.units import BuildUnit @@ -24,7 +26,12 @@ class Builder(ABC): An abstract base class for all builders that allow to configure and create targets. """ - def __init__(self): + def __init__(self, parent_builder: Any): + """ + :param parent_builder: The builder, this builder has been created from + """ + self.parent_builder = parent_builder + self.child_builders = [] self.dependencies = set() def depends_on(self, *target_names: str) -> 'Target.Builder': @@ -37,18 +44,55 @@ def depends_on(self, *target_names: str) -> 'Target.Builder': self.dependencies.update(target_names) return self + def depends_on_build_target(self) -> 'BuildTarget.Builder': + """ + Creates and returns a `BuildTarget.Builder` that allows to configure a build target, this target should + depend on. + + :return: The `BuildTarget.Builder` that has been created + """ + target_name = str(uuid4()) + target_builder = BuildTarget.Builder(self, target_name) + self.dependencies.add(target_name) + self.child_builders.append(target_builder) + return target_builder + + def depends_on_phony_target(self) -> 'PhonyTarget.Builder': + """ + Creates and returns a `PhonyTarget.Builder` that allows to configure a phony target, this target should + depend on. + + :return: The `PhonyTarget.Builder` that has been created + """ + target_name = str(uuid4()) + target_builder = PhonyTarget.Builder(self, target_name) + self.child_builders.append(target_builder) + self.dependencies.add(target_name) + return target_builder + @abstractmethod - def build(self) -> 'Target': + def _build(self, build_unit: BuildUnit) -> 'Target': """ - Creates and returns the target that has been configured via the builder. + Must be implemented by subclasses in order to create the target that has been configured via the builder. - :return: The target that has been created + :param build_unit: The build unit, the target belongs to + :return: The target that has been created + """ + + def build(self, build_unit: BuildUnit) -> List['Target']: """ + Creates and returns all targets that have been configured via the builder. + + :param build_unit: The build unit, the target belongs to + :return: The targets that have been created + """ + return [self._build(build_unit)] + reduce(lambda aggr, builder: aggr + builder.build(build_unit), + self.child_builders, []) def __init__(self, name: str, dependencies: Set[str]): """ :param name: The name of the target - :param dependencies: The name of the targets, this target depends on + :param dependencies: The names of the targets, this target depends on """ self.name = name self.dependencies = dependencies @@ -64,6 +108,87 @@ def register(self, environment: Environment, module_registry: ModuleRegistry) -> """ +class BuildTarget(Target): + """ + A build target, which executes a certain action and produces one or several output files. + """ + + class Runnable(ABC): + """ + An abstract base class for all classes that can be run via a build target. + """ + + @abstractmethod + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + """ + Must be implemented by subclasses in order to run the target. + + :param build_unit: The build unit, the target belongs to + :param modules: A `ModuleRegistry` that can be used by the target for looking up modules + """ + + def get_output_files(self, modules: ModuleRegistry) -> List[str]: + """ + May be overridden by subclasses in order to return the output files produced by the target. + + :param modules: A `ModuleRegistry` that can be used by the target for looking up modules + :return: A list that contains the output files + """ + return [] + + class Builder(Target.Builder): + """ + A builder that allows to configure and create build targets. + """ + + def __init__(self, parent_builder: Any, name: str): + """ + :param parent_builder: The builder, this builder has been created from + :param name: The name of the target + """ + super().__init__(parent_builder) + self.name = name + self.runnables = [] + + def set_runnables(self, *runnables: 'BuildTarget.Runnable') -> Any: + """ + Sets one or several `Runnable` objects to be run by the target. + + :param runnables: The `Runnable` objects to be set + :return: The builder, this builder has been created from + """ + self.runnables = list(runnables) + return self.parent_builder + + def _build(self, build_unit: BuildUnit) -> Target: + return BuildTarget(self.name, self.dependencies, self.runnables, build_unit) + + def __init__(self, name: str, dependencies: Set[str], runnables: List[Runnable], build_unit: BuildUnit): + """ + :param name: The name of the target or None, if the target does not have a name + :param dependencies: The names of the targets, this target depends on + :param runnables: The `BuildTarget.Runnable` to the be run by the target + :param build_unit: The `BuildUnit`, the target belongs to + """ + super().__init__(name, dependencies) + self.runnables = runnables + self.build_unit = build_unit + + def register(self, environment: Environment, module_registry: ModuleRegistry) -> Any: + + def action(): + for runnable in self.runnables: + runnable.run(self.build_unit, module_registry) + + output_files = reduce(lambda aggr, runnable: runnable.get_output_files(module_registry), self.runnables, []) + target = (output_files if len(output_files) > 1 else output_files[0]) if output_files else None + + if target: + return environment.Command(target, None, action=lambda **_: action()) + + return environment.AlwaysBuild(environment.Alias(self.name, None, action=lambda **_: action())) + + class PhonyTarget(Target): """ A phony target, which executes a certain action and does not produce any output files. @@ -76,6 +201,7 @@ class Runnable(ABC): An abstract base class for all classes that can be run via a phony target. """ + @abstractmethod def run(self, build_unit: BuildUnit, modules: ModuleRegistry): """ Must be implemented by subclasses in order to run the target. @@ -89,60 +215,60 @@ class Builder(Target.Builder): A builder that allows to configure and create phony targets. """ - def __init__(self, target_builder: 'TargetBuilder', name: str): + def __init__(self, parent_builder: Any, name: str): """ - :param target_builder: The `TargetBuilder`, this builder has been created from + :param parent_builder: The builder, this builder has been created from :param name: The name of the target """ - super().__init__() - self.target_builder = target_builder + super().__init__(parent_builder) self.name = name self.functions = [] self.runnables = [] - def nop(self) -> 'TargetBuilder': + def nop(self) -> Any: """ Instructs the target to not execute any action. :return: The `TargetBuilder`, this builder has been created from """ - return self.target_builder + return self.parent_builder - def set_functions(self, *functions: 'PhonyTarget.Function') -> 'TargetBuilder': + def set_functions(self, *functions: 'PhonyTarget.Function') -> Any: """ Sets one or several functions to be run by the target. :param functions: The functions to be set - :return: The `TargetBuilder`, this builder has been created from + :return: The builder, this builder has been created from """ self.functions = list(functions) - return self.target_builder + return self.parent_builder - def set_runnables(self, *runnables: 'PhonyTarget.Runnable') -> 'TargetBuilder': + def set_runnables(self, *runnables: 'PhonyTarget.Runnable') -> Any: """ Sets one or several `Runnable` objects to be run by the target. :param runnables: The `Runnable` objects to be set - :return: The `TargetBuilder`, this builder has been created from + :return: The builder, this builder has been created from """ self.runnables = list(runnables) - return self.target_builder + return self.parent_builder - def build(self) -> Target: + def _build(self, build_unit: BuildUnit) -> Target: def action(module_registry: ModuleRegistry): for function in self.functions: function() for runnable in self.runnables: - runnable.run(self.target_builder.build_unit, module_registry) + runnable.run(build_unit, module_registry) return PhonyTarget(self.name, self.dependencies, action) def __init__(self, name: str, dependencies: Set[str], action: Callable[[ModuleRegistry], None]): """ - :param name: The name of the target - :param action: The action to be executed by the target + :param name: The name of the target + :param dependencies: The names of the targets, this target depends on + :param action: The action to be executed by the target """ super().__init__(name, dependencies) self.action = action @@ -164,6 +290,17 @@ def __init__(self, build_unit: BuildUnit = BuildUnit()): self.build_unit = build_unit self.target_builders = [] + def add_build_target(self, name: str) -> BuildTarget.Builder: + """ + Adds a build target. + + :param name: The name of the target + :return: A `BuildTarget.Builder` that allows to configure the target + """ + target_builder = BuildTarget.Builder(self, name) + self.target_builders.append(target_builder) + return target_builder + def add_phony_target(self, name: str) -> PhonyTarget.Builder: """ Adds a phony target. @@ -181,7 +318,7 @@ def build(self) -> List[Target]: :return: A list that stores the targets that have been created """ - return [target_builder.build() for target_builder in self.target_builders] + return reduce(lambda aggr, builder: aggr + builder.build(self.build_unit), self.target_builders, []) class TargetRegistry: From bdabfd98187d7aa40bb73695d9f223ef97c638e9 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sun, 1 Dec 2024 00:54:42 +0100 Subject: [PATCH 056/114] Allow to add custom filters to a FileSearch. --- scons/util/files.py | 81 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 14 deletions(-) diff --git a/scons/util/files.py b/scons/util/files.py index f2f24da39..824b7b093 100644 --- a/scons/util/files.py +++ b/scons/util/files.py @@ -6,7 +6,7 @@ from functools import partial, reduce from glob import glob from os import path -from typing import Callable, List +from typing import Callable, List, Optional, Set from util.languages import Language @@ -84,9 +84,11 @@ class FileSearch: Allows to search for files. """ + Filter = Callable[[str, str], bool] + def __init__(self): self.hidden = False - self.file_patterns = {'*'} + self.filters = [] self.directory_search = DirectorySearch() def set_recursive(self, recursive: bool) -> 'FileSearch': @@ -130,29 +132,76 @@ def set_hidden(self, hidden: bool) -> 'FileSearch': self.hidden = hidden return self + def add_filters(self, *filter_functions: Filter) -> 'FileSearch': + """ + Adds one or several filters that match files to be included. + + :param filter_functions: The filters to be added + :return: The `FileSearch` itself + """ + self.filters.extend(filter_functions) + return self + def filter_by_name(self, *names: str) -> 'FileSearch': """ - Sets the names of the files that should be included. + Adds one or several filters that match files to be included based on their name. :param names: The names of the files that should be included (including their suffix) :return: The `FileSearch` itself """ - self.file_patterns = {name for name in names} - return self + + def filter_file(filtered_names: Set[str], _: str, file_name: str): + return file_name in filtered_names + + return self.add_filters(*[partial(filter_file, name) for name in names]) + + def filter_by_substrings(self, + starts_with: Optional[str] = None, + not_starts_with: Optional[str] = None, + ends_with: Optional[str] = None, + not_ends_with: Optional[str] = None, + contains: Optional[str] = None, + not_contains: Optional[str] = None) -> 'FileSearch': + """ + Adds a filter that matches files based on whether their name contains specific substrings. + + :param starts_with: A substring, names must start with or None, if no restrictions should be imposed + :param not_starts_with: A substring, names must not start with or None, if no restrictions should be imposed + :param ends_with: A substring, names must end with or None, if no restrictions should be imposed + :param not_ends_with: A substring, names must not end with or None, if no restrictions should be imposed + :param contains: A substring, names must contain or None, if no restrictions should be imposed + :param not_contains: A substring, names must not contain or None, if no restrictions should be imposed + :return: The `FileSearch` itself + """ + + def filter_file(start: Optional[str], not_start: Optional[str], end: Optional[str], not_end: Optional[str], + substring: Optional[str], not_substring: Optional[str], _: str, file_name: str): + return (not start or file_name.startswith(start)) \ + and (not not_start or not file_name.startswith(not_start)) \ + and (not end or file_name.endswith(end)) \ + and (not not_end or file_name.endswith(not_end)) \ + and (not substring or file_name.find(substring) >= 0) \ + and (not not_substring or file_name.find(not_substring) < 0) + + return self.add_filters( + partial(filter_file, starts_with, not_starts_with, ends_with, not_ends_with, contains, not_contains)) def filter_by_suffix(self, *suffixes: str) -> 'FileSearch': """ - Sets the suffixes of the files that should be included. + Adds one or several filters that match files to be included based on their suffix. :param suffixes: The suffixes of the files that should be included (without starting dot) :return: The `FileSearch` itself """ - self.file_patterns = {'*.' + suffix for suffix in suffixes} - return self + + def filter_file(filtered_suffixes: List[str], _: str, file_name: str): + return reduce(lambda aggr, suffix: aggr or file_name.endswith(suffix), filtered_suffixes, False) + + return self.add_filters(partial(filter_file, list(suffixes))) def filter_by_language(self, *languages: Language) -> 'FileSearch': """ - Sets the suffixes of the files that should be included. + Adds one or several filters that match files to be included based on the programming language they belong to. :param languages: The languages of the files that should be included :return: The `FileSearch` itself @@ -169,13 +218,17 @@ def list(self, *directories: str) -> List[str]: result = [] subdirectories = self.directory_search.list(*directories) if self.directory_search.recursive else [] + def filter_file(file: str) -> bool: + return path.isfile(file) and (not self.filters or reduce( + lambda aggr, file_filter: aggr or file_filter(path.dirname(file), path.basename(file)), self.filters, + False)) + for directory in list(directories) + subdirectories: - for file_pattern in self.file_patterns: - files = [file for file in glob(path.join(directory, file_pattern)) if path.isfile(file)] + files = [file for file in glob(path.join(directory, '*')) if filter_file(file)] - if self.hidden: - files.extend([file for file in glob(path.join(directory, '.' + file_pattern)) if path.isfile(file)]) + if self.hidden: + files.extend([file for file in glob(path.join(directory, '.*')) if filter_file(file)]) - result.extend(files) + result.extend(files) return result From ee2ce06761b80c5faed19d0cc046a3b849e1968c Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Mon, 2 Dec 2024 13:32:18 +0100 Subject: [PATCH 057/114] Add classes Requirements and RequirementsFiles. --- scons/dependencies/python/targets.py | 32 +++++++-------- scons/util/pip.py | 61 ++++++++++++++++++++-------- 2 files changed, 58 insertions(+), 35 deletions(-) diff --git a/scons/dependencies/python/targets.py b/scons/dependencies/python/targets.py index f86ba4006..1f806af65 100644 --- a/scons/dependencies/python/targets.py +++ b/scons/dependencies/python/targets.py @@ -3,6 +3,8 @@ Implements targets for installing runtime requirements that are required by the project's source code. """ +from functools import reduce + from dependencies.python.modules import DependencyType, PythonDependencyModule from util.modules import ModuleRegistry from util.pip import Pip @@ -17,9 +19,10 @@ class InstallRuntimeDependencies(PhonyTarget.Runnable): """ def run(self, _: BuildUnit, modules: ModuleRegistry): - for module in modules.lookup(PythonDependencyModule.Filter(DependencyType.RUNTIME)): - for requirements_file in module.find_requirements_files(): - Pip(requirements_file).install_all_packages() + dependency_modules = modules.lookup(PythonDependencyModule.Filter(DependencyType.RUNTIME)) + requirements_files = reduce(lambda aggr, module: aggr + module.find_requirements_files(), dependency_modules, + []) + Pip(*requirements_files).install_all_packages() class CheckPythonDependencies(PhonyTarget.Runnable): @@ -28,22 +31,17 @@ class CheckPythonDependencies(PhonyTarget.Runnable): """ def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - outdated_dependencies_by_requirements_file = {} - - for module in modules.lookup(PythonDependencyModule.Filter()): - for requirements_file in module.find_requirements_files(): - outdated_dependencies = Pip(requirements_file).list_outdated_dependencies() - - if outdated_dependencies: - outdated_dependencies_by_requirements_file[requirements_file] = outdated_dependencies + dependency_modules = modules.lookup(PythonDependencyModule.Filter()) + requirements_files = reduce(lambda aggr, module: aggr + module.find_requirements_files(), dependency_modules, + []) + outdated_dependencies = Pip(*requirements_files).list_outdated_dependencies() - if outdated_dependencies_by_requirements_file: - table = Table(build_unit, 'Requirements file', 'Dependency', 'Installed version', 'Latest version') + if outdated_dependencies: + table = Table(build_unit, 'Dependency', 'Installed version', 'Latest version') - for requirements_file, outdated_dependencies in outdated_dependencies_by_requirements_file.items(): - for dependency in outdated_dependencies: - table.add_row(requirements_file, str(dependency.installed.package), dependency.installed.version, - dependency.latest.version) + for outdated_dependency in outdated_dependencies: + table.add_row(str(outdated_dependency.installed.package), outdated_dependency.installed.version, + outdated_dependency.latest.version) table.sort_rows(0, 1) print('The following dependencies are outdated:\n') diff --git a/scons/util/pip.py b/scons/util/pip.py index d9da6e676..6b7f3f4cc 100644 --- a/scons/util/pip.py +++ b/scons/util/pip.py @@ -3,8 +3,9 @@ Provides utility functions for installing Python packages via pip. """ -from abc import ABC +from abc import ABC, abstractmethod from dataclasses import dataclass +from functools import reduce from typing import Dict, Optional, Set from util.cmd import Command as Cmd @@ -93,20 +94,17 @@ def __hash__(self) -> int: return hash(self.installed) -class RequirementsFile(TextFile): +class Requirements(ABC): """ - Represents a specific requirements.txt file. + An abstract base class for all classes that provide access to requirements. """ @property + @abstractmethod def requirements_by_package(self) -> Dict[Package, Requirement]: """ - A dictionary that contains all requirements in the requirements file by their package. + A dictionary that contains all requirements by their package. """ - return { - requirement.package: requirement - for requirement in [Requirement.parse(line) for line in self.lines if line.strip('\n').strip()] - } @property def requirements(self) -> Set[Requirement]: @@ -132,7 +130,7 @@ def lookup_requirements(self, *packages: Package, accept_missing: bool = False) if requirement: requirements.add(requirement) elif not accept_missing: - raise RuntimeError('Package "' + str(package) + '" not found in requirements file "' + self.file + '"') + raise RuntimeError('Requirement for package "' + str(package) + '" not found') return requirements @@ -149,6 +147,33 @@ def lookup_requirement(self, package: Package, accept_missing: bool = False) -> return requirements.pop() if requirements else None +class RequirementsFile(TextFile, Requirements): + """ + Represents a specific requirements.txt file. + """ + + @property + def requirements_by_package(self) -> Dict[Package, Requirement]: + return { + requirement.package: requirement + for requirement in [Requirement.parse(line) for line in self.lines if line.strip('\n').strip()] + } + + +class RequirementsFiles(Requirements): + """ + Represents multiple requirements.txt files. + """ + + def __init__(self, *requirements_files: str): + self.requirements_files = [RequirementsFile(requirements_file) for requirements_file in requirements_files] + + @property + def requirements_by_package(self) -> Dict[Package, Requirement]: + return reduce(lambda aggr, requirements_file: aggr | requirements_file.requirements_by_package, + self.requirements_files, {}) + + class Pip: """ Allows to install Python packages via pip. @@ -227,11 +252,12 @@ def __install_requirement(requirement: Requirement, dry_run: bool = False): except RuntimeError: Pip.__install_requirement(requirement) - def __init__(self, requirements_file: str): + def __init__(self, *requirements_files: str): """ - :param requirements_file: The requirements file that specifies the version of the packages to be installed + :param requirements_files: The paths to the requirements files that specify the versions of the packages to be + installed """ - self.requirements_file = RequirementsFile(requirements_file) + self.requirements = RequirementsFiles(*requirements_files) @staticmethod def for_build_unit(build_unit: BuildUnit = BuildUnit('util')): @@ -252,7 +278,7 @@ def install_packages(self, *package_names: str, accept_missing: bool = False): True, if it should simply be ignored """ packages = [Package(package_name) for package_name in package_names] - requirements = self.requirements_file.lookup_requirements(*packages, accept_missing=accept_missing) + requirements = self.requirements.lookup_requirements(*packages, accept_missing=accept_missing) for requirement in requirements: self.__install_requirement(requirement, dry_run=True) @@ -261,16 +287,15 @@ def install_all_packages(self): """ Installs all dependencies in the requirements file. """ - print('Installing all dependencies in requirements file "' + str(self.requirements_file) + '"...') - requirements = self.requirements_file.requirements + print('Installing all dependencies...') - for requirement in requirements: + for requirement in self.requirements.requirements: self.__install_requirement(requirement, dry_run=True) def list_outdated_dependencies(self) -> Set[Dependency]: self.install_all_packages() - print('Checking for outdated dependencies in requirements file "' + str(self.requirements_file) + '"...') + print('Checking for outdated dependencies...') stdout = Pip.ListCommand(outdated=True).print_command(False).capture_output() stdout_lines = stdout.strip().split('\n') i = 0 @@ -292,7 +317,7 @@ def list_outdated_dependencies(self) -> Set[Dependency]: + line) package = Package(parts[0]) - requirement = self.requirements_file.lookup_requirement(package, accept_missing=True) + requirement = self.requirements.lookup_requirement(package, accept_missing=True) if requirement and requirement.version: installed_version = parts[1] From 2e4101652f0f4129a2894945bccd765d17b76c7d Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Mon, 2 Dec 2024 14:04:33 +0100 Subject: [PATCH 058/114] Add classes VersionFile and DevelopmentVersionFile. --- scons/dependencies/github/actions.py | 4 +- scons/dependencies/github/pyyaml.py | 4 +- scons/util/io.py | 2 +- scons/versioning/versioning.py | 120 +++++++++++++++------------ 4 files changed, 71 insertions(+), 59 deletions(-) diff --git a/scons/dependencies/github/actions.py b/scons/dependencies/github/actions.py index ac6d114ea..8b2715307 100644 --- a/scons/dependencies/github/actions.py +++ b/scons/dependencies/github/actions.py @@ -177,9 +177,9 @@ def update_actions(self, *updated_actions: Action): if updated_action: updated_lines[-1] = line.replace(str(action.version), str(updated_action.version)) - self.write_lines(updated_lines) + self.write_lines(*updated_lines) - def write_lines(self, lines: List[str]): + def write_lines(self, *lines: str): super().write_lines(lines) del self.uses_clauses del self.actions diff --git a/scons/dependencies/github/pyyaml.py b/scons/dependencies/github/pyyaml.py index 8cdb03906..5111d1aa4 100644 --- a/scons/dependencies/github/pyyaml.py +++ b/scons/dependencies/github/pyyaml.py @@ -4,7 +4,7 @@ Provides classes for reading the contents of YAML files via "pyyaml". """ from functools import cached_property -from typing import Dict, List +from typing import Dict from util.io import TextFile, read_file from util.pip import Pip @@ -35,6 +35,6 @@ def yaml_dict(self) -> Dict: with read_file(self.file) as file: return yaml.load(file.read(), Loader=yaml.CLoader) - def write_lines(self, lines: List[str]): + def write_lines(self, *lines: str): super().write_lines(lines) del self.yaml_dict diff --git a/scons/util/io.py b/scons/util/io.py index 5dc52f8ae..f34291a1d 100644 --- a/scons/util/io.py +++ b/scons/util/io.py @@ -46,7 +46,7 @@ def lines(self) -> List[str]: with read_file(self.file) as file: return file.readlines() - def write_lines(self, lines: List[str]): + def write_lines(self, *lines: str): """ Overwrites all lines in the text file. diff --git a/scons/versioning/versioning.py b/scons/versioning/versioning.py index f4f722f2e..e9877b1c5 100644 --- a/scons/versioning/versioning.py +++ b/scons/versioning/versioning.py @@ -3,16 +3,11 @@ Provides actions for updating the project's version. """ -import sys - -from dataclasses import dataclass +from dataclasses import dataclass, replace +from functools import cached_property from typing import Optional -from util.io import read_file, write_file - -VERSION_FILE = '.version' - -DEV_VERSION_FILE = '.version-dev' +from util.io import TextFile @dataclass @@ -76,44 +71,65 @@ def __str__(self) -> str: return version -def __read_version_file(version_file) -> str: - with read_file(version_file) as file: - lines = file.readlines() +class VersionFile(TextFile): + """ + The file that stores the project's version. + """ + + def __init__(self): + super().__init__('.version') + + @cached_property + def version(self) -> Version: + lines = self.lines if len(lines) != 1: - print('File "' + version_file + '" must contain exactly one line') - sys.exit(-1) + raise ValueError('File "' + self.file + '" must contain exactly one line') - return lines[0] + return Version.parse(lines[0]) + def update(self, version: Version): + self.write_lines(str(version)) + print('Updated version to "' + str(version) + '"') -def __write_version_file(version_file, version: str): - with write_file(version_file) as file: - file.write(version) + def write_lines(self, *lines: str): + super().write_lines(lines) + del self.version -def __get_current_development_version() -> int: - current_version = __read_version_file(DEV_VERSION_FILE) - print('Current development version is "' + current_version + '"') - return Version.parse_version_number(current_version) +class DevelopmentVersionFile(TextFile): + """ + The file that stores the project's development version. + """ + + @cached_property + def development_version(self) -> int: + lines = self.lines + + if len(lines) != 1: + raise ValueError('File "' + self.file + '" must contain exactly one line') + + return Version.parse_version_number(lines[0]) + def update(self, development_version: int): + self.write_lines(str(development_version)) + print('Updated development version to "' + str(development_version) + '"') -def __update_development_version(dev: int): - updated_version = str(dev) - print('Updated version to "' + updated_version + '"') - __write_version_file(DEV_VERSION_FILE, updated_version) + def write_lines(self, *lines: str): + super().write_lines(lines) + del self.development_version -def __get_current_version() -> Version: - current_version = __read_version_file(VERSION_FILE) - print('Current version is "' + current_version + '"') - return Version.parse(current_version) +def __get_version_file() -> VersionFile: + version_file = VersionFile() + print('Current version is "' + str(version_file.version) + '"') + return version_file -def __update_version(version: Version): - updated_version = str(version) - print('Updated version to "' + updated_version + '"') - __write_version_file(VERSION_FILE, updated_version) +def __get_development_version_file() -> DevelopmentVersionFile: + version_file = DevelopmentVersionFile() + print('Current development version is "' + str(version_file.development_version) + '"') + return version_file def get_current_version() -> Version: @@ -122,7 +138,7 @@ def get_current_version() -> Version: :return: The project's current version """ - return Version.parse(__read_version_file(VERSION_FILE)) + return VersionFile().version def print_current_version(): @@ -136,53 +152,49 @@ def increment_development_version(): """ Increments the development version. """ - dev = __get_current_development_version() - dev += 1 - __update_development_version(dev) + version_file = __get_development_version_file() + version_file.update(version_file.dev + 1) def reset_development_version(): """ Resets the development version. """ - __get_current_development_version() - __update_development_version(0) + version_file = __get_development_version_file() + version_file.update(0) def apply_development_version(): """ Appends the development version to the current semantic version. """ - version = __get_current_version() - version.dev = __get_current_development_version() - __update_version(version) + version_file = __get_version_file() + development_version = __get_development_version_file().development_version + version_file.update(replace(version_file.version, dev=development_version)) def increment_patch_version(): """ Increments the patch version. """ - version = __get_current_version() - version.patch += 1 - __update_version(version) + version_file = __get_version_file() + version = version_file.version + version_file.update(replace(version, patch=version.patch + 1)) def increment_minor_version(): """ Increments the minor version. """ - version = __get_current_version() - version.minor += 1 - version.patch = 0 - __update_version(version) + version_file = __get_version_file() + version = version_file.version + version_file.update(replace(version, minor=version.minor + 1, patch=0)) def increment_major_version(): """ Increments the major version. """ - version = __get_current_version() - version.major += 1 - version.minor = 0 - version.patch = 0 - __update_version(version) + version_file = __get_version_file() + version = version_file.version + version_file.update(replace(version, major=version.major + 1, minor=0, patch=0)) From 267d9ab8563ff31d875279d6649ae1749a41a056 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Mon, 2 Dec 2024 14:23:28 +0100 Subject: [PATCH 059/114] Add classes ChangesetFile and ChangelogFile. --- scons/util/io.py | 17 +- scons/versioning/changelog.py | 366 ++++++++++++++++++---------------- 2 files changed, 207 insertions(+), 176 deletions(-) diff --git a/scons/util/io.py b/scons/util/io.py index f34291a1d..d710291d6 100644 --- a/scons/util/io.py +++ b/scons/util/io.py @@ -4,6 +4,7 @@ Provides utility functions for reading and writing files. """ from functools import cached_property +from os import path from typing import List ENCODING_UTF8 = 'utf-8' @@ -32,17 +33,22 @@ class TextFile: Allows to read and write the content of a text file. """ - def __init__(self, file: str): + def __init__(self, file: str, accept_missing: bool = False): """ - :param file: The path to the text file + :param file: The path to the text file + :param accept_missing: True, if no errors should be raised if the text file is missing, False otherwise """ self.file = file + self.accept_missing = accept_missing @cached_property def lines(self) -> List[str]: """ The lines in the text file. """ + if self.accept_missing and not path.isfile(self.file): + return [] + with read_file(self.file) as file: return file.readlines() @@ -57,5 +63,12 @@ def write_lines(self, *lines: str): del self.lines + def clear(self): + """ + Clears the text file. + """ + print('Clearing file "' + self.file + '"...') + self.write_lines('') + def __str__(self) -> str: return self.file diff --git a/scons/versioning/changelog.py b/scons/versioning/changelog.py index 976f5e6c3..a24a0bd24 100644 --- a/scons/versioning/changelog.py +++ b/scons/versioning/changelog.py @@ -8,36 +8,22 @@ from dataclasses import dataclass, field from datetime import date from enum import Enum, auto -from os import path +from functools import cached_property from typing import List, Optional -from util.io import read_file, write_file +from util.io import TextFile from versioning.versioning import Version, get_current_version -PREFIX_HEADER = '# ' +CHANGESET_FILE_MAIN = '.changelog-main.md' -PREFIX_SUB_HEADER = '## ' +CHANGESET_FILE_FEATURE = '.changelog-feature.md' -PREFIX_SUB_SUB_HEADER = '### ' - -PREFIX_DASH = '- ' - -PREFIX_ASTERISK = '* ' - -URL_DOCUMENTATION = 'https://mlrl-boomer.readthedocs.io/en/' - -CHANGELOG_FILE_MAIN = '.changelog-main.md' - -CHANGELOG_FILE_FEATURE = '.changelog-feature.md' - -CHANGELOG_FILE_BUGFIX = '.changelog-bugfix.md' - -CHANGELOG_FILE = 'CHANGELOG.md' +CHANGESET_FILE_BUGFIX = '.changelog-bugfix.md' class LineType(Enum): """ - Represents different types of lines that may occur in a changelog. + Represents different types of lines that may occur in a changeset. """ BLANK = auto() HEADER = auto() @@ -52,9 +38,9 @@ def parse(line: str) -> Optional['LineType']: """ if not line or line.isspace(): return LineType.BLANK - if line.startswith(PREFIX_HEADER): + if line.startswith(Line.PREFIX_HEADER): return LineType.HEADER - if line.startswith(PREFIX_DASH) or line.startswith(PREFIX_ASTERISK): + if line.startswith(Line.PREFIX_DASH) or line.startswith(Line.PREFIX_ASTERISK): return LineType.ENUMERATION return None @@ -62,7 +48,7 @@ def parse(line: str) -> Optional['LineType']: @dataclass class Line: """ - A single line in a changelog. + A single line in a changeset. Attributes: line_number: The line number, starting at 1 @@ -75,6 +61,41 @@ class Line: line: str content: str + PREFIX_HEADER = '# ' + + PREFIX_DASH = '- ' + + PREFIX_ASTERISK = '* ' + + @staticmethod + def parse(line: str, line_number: int) -> 'Line': + """ + Parses and returns a single line in a changeset. + + :param line: The line to be parsed + :param line_number: The number of the line to parsed (starting at 1) + :return: The `Line` that has been created + """ + line = line.strip('\n') + line_type = LineType.parse(line) + + if not line_type: + raise ValueError('Line ' + str(line_number) + + ' is invalid: Must be blank, a top-level header (starting with "' + Line.PREFIX_HEADER + + '"), or an enumeration (starting with "' + Line.PREFIX_DASH + '" or "' + + Line.PREFIX_ASTERISK + '"), but is "' + line + '"') + + content = line + + if line_type != LineType.BLANK: + content = line.lstrip(Line.PREFIX_HEADER).lstrip(Line.PREFIX_DASH).lstrip(Line.PREFIX_ASTERISK) + + if not content or content.isspace(): + raise ValueError('Line ' + str(line_number) + ' is is invalid: Content must not be blank, but is "' + + line + '"') + + return Line(line_number=line_number, line_type=line_type, line=line, content=content) + @dataclass class Changeset: @@ -89,14 +110,77 @@ class Changeset: changes: List[str] = field(default_factory=list) def __str__(self) -> str: - changeset = PREFIX_SUB_SUB_HEADER + self.header + '\n\n' + changeset = '### ' + self.header + '\n\n' for content in self.changes: - changeset += PREFIX_DASH + content + '\n' + changeset += Line.PREFIX_DASH + content + '\n' return changeset +class ChangesetFile(TextFile): + """ + A file that stores several changesets. + """ + + def __validate_line(self, current_line: Optional[Line], previous_line: Optional[Line]): + current_line_is_enumeration = current_line and current_line.line_type == LineType.ENUMERATION + + if current_line_is_enumeration and not previous_line: + raise ValueError('File "' + self.file + '" must start with a top-level header (starting with "' + + Line.PREFIX_HEADER + '")') + + current_line_is_header = current_line and current_line.line_type == LineType.HEADER + previous_line_is_header = previous_line and previous_line.line_type == LineType.HEADER + + if (current_line_is_header and previous_line_is_header) or (not current_line and previous_line_is_header): + raise ValueError('Header "' + previous_line.line + '" at line ' + str(previous_line.line_number) + + ' of file "' + self.file + '" is not followed by any content') + + @cached_property + def parsed_lines(self) -> List[Line]: + parsed_lines = [] + + for i, line in enumerate(self.lines): + current_line = Line.parse(line, line_number=i + 1) + + if current_line.line_type != LineType.BLANK: + parsed_lines.append(current_line) + + return parsed_lines + + @cached_property + def changesets(self) -> List[Changeset]: + changesets = [] + + for line in self.parsed_lines: + if line.line_type == LineType.HEADER: + changesets.append(Changeset(header=line.content)) + elif line.line_type == LineType.ENUMERATION: + current_changeset = changesets[-1] + current_changeset.changes.append(line.content) + + return changesets + + def validate(self): + """ + Validates the changelog. + """ + previous_line = None + + for i, current_line in enumerate(self.parsed_lines): + if current_line.line_type != LineType.BLANK: + self.__validate_line(current_line=current_line, previous_line=previous_line) + previous_line = current_line + + self.__validate_line(current_line=None, previous_line=previous_line) + + def write_lines(self, *lines: str): + super().write_lines(lines) + del self.parsed_lines + del self.changesets + + class ReleaseType(Enum): """ Represents the type of a release. @@ -122,6 +206,10 @@ class Release: release_type: ReleaseType changesets: List[Changeset] = field(default_factory=list) + URL_DOCUMENTATION = 'https://mlrl-boomer.readthedocs.io/en/' + + PREFIX_SUB_HEADER = '## ' + @staticmethod def __format_release_month(month: int) -> str: return ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'][month - 1] @@ -143,11 +231,12 @@ def __format_disclaimer(self) -> str: if [changeset for changeset in self.changesets if changeset.header.lower() == 'api changes']: return ('```{warning}\nThis release comes with API changes. For an updated overview of the available ' + 'parameters and command line arguments, please refer to the ' + '[documentation](' - + URL_DOCUMENTATION + str(self.version) + ').\n```\n\n') + + self.URL_DOCUMENTATION + str(self.version) + ').\n```\n\n') return '' def __str__(self) -> str: - release = PREFIX_SUB_HEADER + 'Version ' + str(self.version) + ' (' + self.__format_release_date() + ')\n\n' + release = self.PREFIX_SUB_HEADER + 'Version ' + str( + self.version) + ' (' + self.__format_release_date() + ')\n\n' release += 'A ' + self.release_type.value + ' release that comes with the following changes.\n\n' release += self.__format_disclaimer() @@ -157,100 +246,78 @@ def __str__(self) -> str: return release -def __read_lines(changelog_file: str, skip_if_missing: bool = False) -> List[str]: - if skip_if_missing and not path.isfile(changelog_file): - return [] - - with read_file(changelog_file) as file: - return file.readlines() - - -def __write_lines(changelog_file: str, lines: List[str]): - with write_file(changelog_file) as file: - file.writelines(lines) - - -def __parse_line(changelog_file: str, line_number: int, line: str) -> Line: - line = line.strip('\n') - line_type = LineType.parse(line) - - if not line_type: - print('Line ' + str(line_number) + ' of file "' + changelog_file - + '" is invalid: Must be blank, a top-level header (starting with "' + PREFIX_HEADER - + '"), or an enumeration (starting with "' + PREFIX_DASH + '" or "' + PREFIX_ASTERISK + '"), but is "' - + line + '"') - sys.exit(-1) - - content = line - - if line_type != LineType.BLANK: - content = line.lstrip(PREFIX_HEADER).lstrip(PREFIX_DASH).lstrip(PREFIX_ASTERISK) +class ChangelogFile(TextFile): + """ + The file that stores the project's changelog. + """ - if not content or content.isspace(): - print('Line ' + str(line_number) + ' of file "' + changelog_file - + '" is is invalid: Content must not be blank, but is "' + line + '"') - sys.exit(-1) + def __init__(self): + super().__init__('CHANGELOG.md') - return Line(line_number=line_number, line_type=line_type, line=line, content=content) + def add_release(self, release: Release): + """ + Adds a new release to the project's changelog. + :param release: The release to be added + """ + formatted_release = str(release) + print('Adding new release to changelog file "' + self.file + '":\n\n' + formatted_release) + original_lines = self.lines + modified_lines = [] + offset = 0 -def __validate_line(changelog_file: str, current_line: Optional[Line], previous_line: Optional[Line]): - current_line_is_enumeration = current_line and current_line.line_type == LineType.ENUMERATION + for offset, line in enumerate(original_lines): + if line.startswith(Release.PREFIX_SUB_HEADER): + break - if current_line_is_enumeration and not previous_line: - print('File "' + changelog_file + '" must start with a top-level header (starting with "' + PREFIX_HEADER - + '")') - sys.exit(-1) + modified_lines.append(line) - current_line_is_header = current_line and current_line.line_type == LineType.HEADER - previous_line_is_header = previous_line and previous_line.line_type == LineType.HEADER + modified_lines.append(formatted_release) + modified_lines.extend(original_lines[offset:]) + self.write_lines(*modified_lines) - if (current_line_is_header and previous_line_is_header) or (not current_line and previous_line_is_header): - print('Header "' + previous_line.line + '" at line ' + str(previous_line.line_number) + ' of file "' - + changelog_file + '" is not followed by any content') + @property + def latest(self) -> str: + """ + The latest release in the changelog. + """ + release = '' + lines = self.lines + offset = 0 + + for offset, line in enumerate(lines): + if line.startswith(Release.PREFIX_SUB_HEADER): + break + + for line in lines[offset + 2:]: + if line.startswith(Release.PREFIX_SUB_HEADER): + break + + if line.startswith('```{'): + release += '***' + elif line.startswith('```'): + release = release.rstrip('\n') + release += '***\n' + else: + release += line + + return release.rstrip('\n') + + +def __validate_changeset(changeset_file: str): + try: + print('Validating changeset file "' + changeset_file + '"...') + ChangesetFile(changeset_file, accept_missing=True).validate() + except ValueError as error: + print('Changeset file "' + changeset_file + '" is malformed!\n\n' + str(error)) sys.exit(-1) -def __parse_lines(changelog_file: str, lines: List[str]) -> List[Line]: - previous_line = None - parsed_lines = [] - - for i, line in enumerate(lines): - current_line = __parse_line(changelog_file=changelog_file, line_number=(i + 1), line=line) - - if current_line.line_type != LineType.BLANK: - __validate_line(changelog_file=changelog_file, current_line=current_line, previous_line=previous_line) - previous_line = current_line - parsed_lines.append(current_line) - - __validate_line(changelog_file=changelog_file, current_line=None, previous_line=previous_line) - return parsed_lines - - -def __parse_changesets(changelog_file: str, skip_if_missing: bool = False) -> List[Changeset]: - changesets = [] - lines = __parse_lines(changelog_file, __read_lines(changelog_file, skip_if_missing=skip_if_missing)) - - for line in lines: - if line.line_type == LineType.HEADER: - changesets.append(Changeset(header=line.content)) - elif line.line_type == LineType.ENUMERATION: - current_changeset = changesets[-1] - current_changeset.changes.append(line.content) - - return changesets - - -def __validate_changelog(changelog_file: str): - print('Validating changelog file "' + changelog_file + '"...') - __parse_changesets(changelog_file, skip_if_missing=True) - - -def __merge_changesets(*changelog_files) -> List[Changeset]: +def __merge_changesets(*changeset_files) -> List[Changeset]: changesets_by_header = {} - for changelog_file in changelog_files: - for changeset in __parse_changesets(changelog_file): + for changeset_file in changeset_files: + for changeset in ChangesetFile(changeset_file).changesets: merged_changeset = changesets_by_header.setdefault(changeset.header.lower(), changeset) if merged_changeset != changeset: @@ -259,111 +326,62 @@ def __merge_changesets(*changelog_files) -> List[Changeset]: return list(changesets_by_header.values()) -def __create_release(release_type: ReleaseType, *changelog_files) -> Release: - return Release(version=get_current_version(), - release_date=date.today(), - release_type=release_type, - changesets=__merge_changesets(*changelog_files)) - - -def __add_release_to_changelog(changelog_file: str, new_release: Release): - formatted_release = str(new_release) - print('Adding new release to changelog file "' + changelog_file + '":\n\n' + formatted_release) - original_lines = __read_lines(changelog_file) - modified_lines = [] - offset = 0 - - for offset, line in enumerate(original_lines): - if line.startswith(PREFIX_SUB_HEADER): - break - - modified_lines.append(line) - - modified_lines.append(formatted_release) - modified_lines.extend(original_lines[offset:]) - __write_lines(changelog_file, modified_lines) - - -def __clear_changelogs(*changelog_files): - for changelog_file in changelog_files: - print('Clearing changelog file "' + changelog_file + '"...') - __write_lines(changelog_file, ['']) - - -def __update_changelog(release_type: ReleaseType, *changelog_files): - new_release = __create_release(release_type, *changelog_files) - __add_release_to_changelog(CHANGELOG_FILE, new_release) - __clear_changelogs(*changelog_files) - - -def __get_latest_changelog() -> str: - changelog = '' - lines = __read_lines(CHANGELOG_FILE) - offset = 0 - - for offset, line in enumerate(lines): - if line.startswith(PREFIX_SUB_HEADER): - break - - for line in lines[offset + 2:]: - if line.startswith(PREFIX_SUB_HEADER): - break - - if line.startswith('```{'): - changelog += '***' - elif line.startswith('```'): - changelog = changelog.rstrip('\n') - changelog += '***\n' - else: - changelog += line +def __update_changelog(release_type: ReleaseType, *changeset_files): + merged_changesets = __merge_changesets(*changeset_files) + new_release = Release(version=get_current_version(), + release_date=date.today(), + release_type=release_type, + changesets=merged_changesets) + ChangelogFile().add_release(new_release) - return changelog.rstrip('\n') + for changeset_file in changeset_files: + ChangesetFile(changeset_file).clear() def validate_changelog_bugfix(): """ Validates the changelog file that lists bugfixes. """ - __validate_changelog(CHANGELOG_FILE_BUGFIX) + __validate_changeset(CHANGESET_FILE_BUGFIX) def validate_changelog_feature(): """ Validates the changelog file that lists new features. """ - __validate_changelog(CHANGELOG_FILE_FEATURE) + __validate_changeset(CHANGESET_FILE_FEATURE) def validate_changelog_main(): """ Validates the changelog file that lists major updates. """ - __validate_changelog(CHANGELOG_FILE_MAIN) + __validate_changeset(CHANGESET_FILE_MAIN) def update_changelog_main(): """ Updates the projects changelog when releasing bugfixes. """ - __update_changelog(ReleaseType.MAJOR, CHANGELOG_FILE_MAIN, CHANGELOG_FILE_FEATURE, CHANGELOG_FILE_BUGFIX) + __update_changelog(ReleaseType.MAJOR, CHANGESET_FILE_MAIN, CHANGESET_FILE_FEATURE, CHANGESET_FILE_BUGFIX) def update_changelog_feature(): """ Updates the project's changelog when releasing new features. """ - __update_changelog(ReleaseType.MINOR, CHANGELOG_FILE_FEATURE, CHANGELOG_FILE_BUGFIX) + __update_changelog(ReleaseType.MINOR, CHANGESET_FILE_FEATURE, CHANGESET_FILE_BUGFIX) def update_changelog_bugfix(): """ Updates the project's changelog when releasing major updates. """ - __update_changelog(ReleaseType.PATCH, CHANGELOG_FILE_BUGFIX) + __update_changelog(ReleaseType.PATCH, CHANGESET_FILE_BUGFIX) def print_latest_changelog(): """ Prints the changelog of the latest release. """ - print(__get_latest_changelog()) + print(ChangelogFile().latest) From f53108a12ad58ec5a254a5a5f871bffff8b6629d Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Mon, 2 Dec 2024 17:10:39 +0100 Subject: [PATCH 060/114] Replace property "requirements_file" of class BuildUnit with function "find_requirements_files". --- scons/dependencies/github/requirements.txt | 1 - scons/dependencies/github/targets.py | 2 +- scons/dependencies/python/targets.py | 2 +- .../dependencies/{python => }/requirements.txt | 0 scons/{util => dependencies}/table.py | 0 scons/util/pip.py | 2 +- scons/util/units.py | 17 ++++++++++++++--- 7 files changed, 17 insertions(+), 7 deletions(-) rename scons/dependencies/{python => }/requirements.txt (100%) rename scons/{util => dependencies}/table.py (100%) diff --git a/scons/dependencies/github/requirements.txt b/scons/dependencies/github/requirements.txt index d06d96dac..27a52b775 100644 --- a/scons/dependencies/github/requirements.txt +++ b/scons/dependencies/github/requirements.txt @@ -1,3 +1,2 @@ pygithub >= 2.5, < 2.6 pyyaml >= 6.0, < 6.1 -tabulate >= 0.9, < 0.10 diff --git a/scons/dependencies/github/targets.py b/scons/dependencies/github/targets.py index ef39d0bb2..cf8688b4d 100644 --- a/scons/dependencies/github/targets.py +++ b/scons/dependencies/github/targets.py @@ -4,8 +4,8 @@ Implements targets for updating the project's GitHub Actions. """ from dependencies.github.actions import WorkflowUpdater +from dependencies.table import Table from util.modules import ModuleRegistry -from util.table import Table from util.targets import PhonyTarget from util.units import BuildUnit diff --git a/scons/dependencies/python/targets.py b/scons/dependencies/python/targets.py index 1f806af65..770ba0cf0 100644 --- a/scons/dependencies/python/targets.py +++ b/scons/dependencies/python/targets.py @@ -6,9 +6,9 @@ from functools import reduce from dependencies.python.modules import DependencyType, PythonDependencyModule +from dependencies.table import Table from util.modules import ModuleRegistry from util.pip import Pip -from util.table import Table from util.targets import PhonyTarget from util.units import BuildUnit diff --git a/scons/dependencies/python/requirements.txt b/scons/dependencies/requirements.txt similarity index 100% rename from scons/dependencies/python/requirements.txt rename to scons/dependencies/requirements.txt diff --git a/scons/util/table.py b/scons/dependencies/table.py similarity index 100% rename from scons/util/table.py rename to scons/dependencies/table.py diff --git a/scons/util/pip.py b/scons/util/pip.py index 6b7f3f4cc..1fb234cae 100644 --- a/scons/util/pip.py +++ b/scons/util/pip.py @@ -267,7 +267,7 @@ def for_build_unit(build_unit: BuildUnit = BuildUnit('util')): :param build_unit: The build unit for which packages should be installed :return: The `Pip` instance that has been created """ - return Pip(build_unit.requirements_file) + return Pip(*build_unit.find_requirements_files()) def install_packages(self, *package_names: str, accept_missing: bool = False): """ diff --git a/scons/util/units.py b/scons/util/units.py index 13f10bab9..610606b45 100644 --- a/scons/util/units.py +++ b/scons/util/units.py @@ -4,6 +4,7 @@ Provides classes that provide information about independent units of the build system. """ from os import path +from typing import List class BuildUnit: @@ -17,9 +18,19 @@ def __init__(self, *subdirectories: str): """ self.root_directory = path.join('scons', *subdirectories) - @property - def requirements_file(self) -> str: + def find_requirements_files(self) -> List[str]: """ The path to the requirements file that specifies the build-time dependencies of this unit. """ - return path.join(self.root_directory, 'requirements.txt') + requirements_files = [] + current_directory = self.root_directory + + while path.basename(current_directory) != 'scons': + requirements_file = path.join(current_directory, 'requirements.txt') + + if path.isfile(requirements_file): + requirements_files.append(requirements_file) + + current_directory = path.dirname(current_directory) + + return requirements_files From 6def0e668fdcc513d365702ce17f0268a894feb3 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Mon, 2 Dec 2024 17:32:42 +0100 Subject: [PATCH 061/114] Add property "target_names" to class TargetRegistry. --- scons/sconstruct.py | 2 +- scons/util/targets.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 2b085e19f..2bc3e718a 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -74,11 +74,11 @@ def __print_if_clean(environment, message: str): for target in getattr(import_source_file(init_file), 'TARGETS', []): if isinstance(target, Target): target_registry.add_target(target) - VALID_TARGETS.add(target.name) target_registry.register() # Raise an error if any invalid targets are given... +VALID_TARGETS.update(target_registry.target_names) invalid_targets = [target for target in COMMAND_LINE_TARGETS if target not in VALID_TARGETS] if invalid_targets: diff --git a/scons/util/targets.py b/scons/util/targets.py index 593393fcf..b98652e35 100644 --- a/scons/util/targets.py +++ b/scons/util/targets.py @@ -364,3 +364,10 @@ def register(self): if scons_dependencies: self.environment.Depends(scons_target, scons_dependencies) + + @property + def target_names(self) -> Set[str]: + """ + A set that contains the names of all targets that have previously been added. + """ + return set(self.targets_by_name.keys()) From d5b86d4c97d4e636fdac203896e856a0ce09a9e7 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Mon, 2 Dec 2024 18:30:39 +0100 Subject: [PATCH 062/114] Dynamically register targets for cleaning up output files. --- scons/util/targets.py | 164 ++++++++++++++++++++++++++++++++---------- 1 file changed, 128 insertions(+), 36 deletions(-) diff --git a/scons/util/targets.py b/scons/util/targets.py index b98652e35..b874ccf42 100644 --- a/scons/util/targets.py +++ b/scons/util/targets.py @@ -6,13 +6,15 @@ import sys from abc import ABC, abstractmethod +from dataclasses import dataclass from functools import reduce -from typing import Any, Callable, List, Set +from typing import Any, Callable, Dict, List, Optional, Set from uuid import uuid4 from util.modules import ModuleRegistry from util.units import BuildUnit +from SCons.Script import COMMAND_LINE_TARGETS from SCons.Script.SConscript import SConsEnvironment as Environment @@ -21,6 +23,25 @@ class Target(ABC): An abstract base class for all targets of the build system. """ + @dataclass + class Dependency: + """ + A single dependency of a parent target. + + Attributes: + target_name: The name of the target, the parent target depends on + clean_dependency: True, if the output files of the dependency should also be cleaned when cleaning the + output files of the parent target, False otherwise + """ + target_name: str + clean_dependency: bool + + def __eq__(self, other: 'Target.Dependency') -> bool: + return self.target_name == other.target_name + + def __hash__(self) -> int: + return hash(self.target_name) + class Builder(ABC): """ An abstract base class for all builders that allow to configure and create targets. @@ -34,40 +55,48 @@ def __init__(self, parent_builder: Any): self.child_builders = [] self.dependencies = set() - def depends_on(self, *target_names: str) -> 'Target.Builder': + def depends_on(self, *target_names: str, clean_dependencies: bool = False) -> 'Target.Builder': """ Adds on or several targets, this target should depend on. - :param target_names: The names of the targets, this target should depend on - :return: The `Target.Builder` itself + :param target_names: The names of the targets, this target should depend on + :param clean_dependencies: True, if output files of the dependencies should also be cleaned when cleaning + the output files of this target, False otherwise + :return: The `Target.Builder` itself """ - self.dependencies.update(target_names) + for target_name in target_names: + self.dependencies.add(Target.Dependency(target_name=target_name, clean_dependency=clean_dependencies)) + return self - def depends_on_build_target(self) -> 'BuildTarget.Builder': + def depends_on_build_target(self, clean_dependency: bool = True) -> 'BuildTarget.Builder': """ Creates and returns a `BuildTarget.Builder` that allows to configure a build target, this target should depend on. - :return: The `BuildTarget.Builder` that has been created + :param clean_dependency: True, if output files of the dependency should also be cleaned when cleaning the + output files of this target, False otherwise + :return: The `BuildTarget.Builder` that has been created """ target_name = str(uuid4()) target_builder = BuildTarget.Builder(self, target_name) - self.dependencies.add(target_name) self.child_builders.append(target_builder) + self.depends_on(target_name, clean_dependencies=clean_dependency) return target_builder - def depends_on_phony_target(self) -> 'PhonyTarget.Builder': + def depends_on_phony_target(self, clean_dependency: bool = True) -> 'PhonyTarget.Builder': """ Creates and returns a `PhonyTarget.Builder` that allows to configure a phony target, this target should depend on. - :return: The `PhonyTarget.Builder` that has been created + :param clean_dependency: True, if output files of the dependency should also be cleaned when cleaning the + output files of this target, False otherwise + :return: The `PhonyTarget.Builder` that has been created """ target_name = str(uuid4()) target_builder = PhonyTarget.Builder(self, target_name) self.child_builders.append(target_builder) - self.dependencies.add(target_name) + self.depends_on(target_name, clean_dependencies=clean_dependency) return target_builder @abstractmethod @@ -89,10 +118,10 @@ def build(self, build_unit: BuildUnit) -> List['Target']: return [self._build(build_unit)] + reduce(lambda aggr, builder: aggr + builder.build(build_unit), self.child_builders, []) - def __init__(self, name: str, dependencies: Set[str]): + def __init__(self, name: str, dependencies: Set['Target.Dependency']): """ :param name: The name of the target - :param dependencies: The names of the targets, this target depends on + :param dependencies: The dependencies of the target """ self.name = name self.dependencies = dependencies @@ -100,13 +129,22 @@ def __init__(self, name: str, dependencies: Set[str]): @abstractmethod def register(self, environment: Environment, module_registry: ModuleRegistry) -> Any: """ - Must be implemented by subclasses in order to register the target. + Must be implemented by subclasses in order to register this target. :param environment: The environment, the target should be registered at :param module_registry: The `ModuleRegistry` that can be used by the target for looking up modules :return: The scons target that has been created """ + def get_clean_files(self, module_registry: ModuleRegistry) -> Optional[List[str]]: + """ + May be overridden by subclasses in order to return the files that should be cleaned up for this target. + + :param module_registry: The `ModuleRegistry` that can be used by the target for looking up modules + :return: A list that contains the files to be cleaned or None, if cleaning is not necessary + """ + return None + class BuildTarget(Target): """ @@ -136,6 +174,16 @@ def get_output_files(self, modules: ModuleRegistry) -> List[str]: """ return [] + def get_clean_files(self, modules: ModuleRegistry) -> List[str]: + """ + May be overridden by subclasses in order to return the output files produced by the target that must be + cleaned. + + :param modules: A `ModuleRegistry` that can be used by the target for looking up modules + :return: A list that contains the files to be cleaned + """ + return self.get_output_files(modules) + class Builder(Target.Builder): """ A builder that allows to configure and create build targets. @@ -163,10 +211,11 @@ def set_runnables(self, *runnables: 'BuildTarget.Runnable') -> Any: def _build(self, build_unit: BuildUnit) -> Target: return BuildTarget(self.name, self.dependencies, self.runnables, build_unit) - def __init__(self, name: str, dependencies: Set[str], runnables: List[Runnable], build_unit: BuildUnit): + def __init__(self, name: str, dependencies: Set[Target.Dependency], runnables: List[Runnable], + build_unit: BuildUnit): """ :param name: The name of the target or None, if the target does not have a name - :param dependencies: The names of the targets, this target depends on + :param dependencies: The dependencies of the target :param runnables: The `BuildTarget.Runnable` to the be run by the target :param build_unit: The `BuildUnit`, the target belongs to """ @@ -188,6 +237,9 @@ def action(): return environment.AlwaysBuild(environment.Alias(self.name, None, action=lambda **_: action())) + def get_clean_files(self, module_registry: ModuleRegistry) -> Optional[List[str]]: + return reduce(lambda aggr, runnable: runnable.get_clean_files(module_registry), self.runnables, []) + class PhonyTarget(Target): """ @@ -264,10 +316,10 @@ def action(module_registry: ModuleRegistry): return PhonyTarget(self.name, self.dependencies, action) - def __init__(self, name: str, dependencies: Set[str], action: Callable[[ModuleRegistry], None]): + def __init__(self, name: str, dependencies: Set[Target.Dependency], action: Callable[[ModuleRegistry], None]): """ :param name: The name of the target - :param dependencies: The names of the targets, this target depends on + :param dependencies: The dependencies of the target :param action: The action to be executed by the target """ super().__init__(name, dependencies) @@ -326,6 +378,55 @@ class TargetRegistry: Allows to register targets. """ + def __register_scons_targets(self) -> Dict[str, Any]: + scons_targets_by_name = {} + + for target_name, target in self.targets_by_name.items(): + scons_targets_by_name[target_name] = target.register(self.environment, self.module_registry) + + return scons_targets_by_name + + def __register_scons_dependencies(self, scons_targets_by_name: Dict[str, Any]): + for target_name, target in self.targets_by_name.items(): + scons_target = scons_targets_by_name[target_name] + scons_dependencies = [] + + for dependency in target.dependencies: + try: + scons_dependencies.append(scons_targets_by_name[dependency.target_name]) + except KeyError: + print('Dependency "' + dependency.target_name + '" of target "' + target_name + + '" has not been registered') + sys.exit(-1) + + if scons_dependencies: + self.environment.Depends(scons_target, scons_dependencies) + + def __get_parent_targets(self, *target_names: str) -> Set[str]: + result = set() + parent_targets = reduce(lambda aggr, target_name: aggr | self.parent_targets_by_name.get(target_name, set()), + target_names, set()) + + if parent_targets: + result.update(self.__get_parent_targets(*parent_targets)) + + result.update(parent_targets) + return result + + def __register_scons_clean_targets(self, scons_targets_by_name: Dict[str, Any]): + if self.environment.GetOption('clean'): + for target_name, target in self.targets_by_name.items(): + parent_targets = {target_name} | self.__get_parent_targets(target_name) + + if not COMMAND_LINE_TARGETS or reduce( + lambda aggr, parent_target: aggr or parent_target in COMMAND_LINE_TARGETS, parent_targets, + False): + clean_files = target.get_clean_files(self.module_registry) + + if clean_files: + clean_targets = [scons_targets_by_name[parent_target] for parent_target in parent_targets] + self.environment.Clean(clean_targets, clean_files) + def __init__(self, module_registry: ModuleRegistry): """ :param module_registry: The `ModuleRegistry` that should be used by targets for looking up modules @@ -333,6 +434,7 @@ def __init__(self, module_registry: ModuleRegistry): self.environment = Environment() self.module_registry = module_registry self.targets_by_name = {} + self.parent_targets_by_name = {} def add_target(self, target: Target): """ @@ -342,28 +444,18 @@ def add_target(self, target: Target): """ self.targets_by_name[target.name] = target + for dependency in target.dependencies: + if dependency.clean_dependency: + parent_targets = self.parent_targets_by_name.setdefault(dependency.target_name, set()) + parent_targets.add(target.name) + def register(self): """ Registers all targets that have previously been added. """ - scons_targets_by_name = {} - - for target_name, target in self.targets_by_name.items(): - scons_targets_by_name[target_name] = target.register(self.environment, self.module_registry) - - for target_name, target in self.targets_by_name.items(): - scons_target = scons_targets_by_name[target_name] - scons_dependencies = [] - - for dependency in target.dependencies: - try: - scons_dependencies.append(scons_targets_by_name[dependency]) - except KeyError: - print('Dependency "' + dependency + '" of target "' + target_name + '" has not been registered') - sys.exit(-1) - - if scons_dependencies: - self.environment.Depends(scons_target, scons_dependencies) + scons_targets_by_name = self.__register_scons_targets() + self.__register_scons_dependencies(scons_targets_by_name) + self.__register_scons_clean_targets(scons_targets_by_name) @property def target_names(self) -> Set[str]: From 6190c7d3e4c8e59266b5e620a2b07e3c6f0c697c Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Mon, 2 Dec 2024 23:02:14 +0100 Subject: [PATCH 063/114] Remove log statement from class Pip. --- scons/dependencies/python/targets.py | 6 +++++- scons/util/pip.py | 5 ----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/scons/dependencies/python/targets.py b/scons/dependencies/python/targets.py index 770ba0cf0..0586e3642 100644 --- a/scons/dependencies/python/targets.py +++ b/scons/dependencies/python/targets.py @@ -34,7 +34,11 @@ def run(self, build_unit: BuildUnit, modules: ModuleRegistry): dependency_modules = modules.lookup(PythonDependencyModule.Filter()) requirements_files = reduce(lambda aggr, module: aggr + module.find_requirements_files(), dependency_modules, []) - outdated_dependencies = Pip(*requirements_files).list_outdated_dependencies() + pip = Pip(*requirements_files) + print('Installing all dependencies...') + pip.install_all_packages() + print('Checking for outdated dependencies...') + outdated_dependencies = pip.list_outdated_dependencies() if outdated_dependencies: table = Table(build_unit, 'Dependency', 'Installed version', 'Latest version') diff --git a/scons/util/pip.py b/scons/util/pip.py index 1fb234cae..67bc7382b 100644 --- a/scons/util/pip.py +++ b/scons/util/pip.py @@ -287,15 +287,10 @@ def install_all_packages(self): """ Installs all dependencies in the requirements file. """ - print('Installing all dependencies...') - for requirement in self.requirements.requirements: self.__install_requirement(requirement, dry_run=True) def list_outdated_dependencies(self) -> Set[Dependency]: - self.install_all_packages() - - print('Checking for outdated dependencies...') stdout = Pip.ListCommand(outdated=True).print_command(False).capture_output() stdout_lines = stdout.strip().split('\n') i = 0 From 769f1262a5821c970fdcf0ba023e60f2ad5421a2 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Tue, 3 Dec 2024 00:00:50 +0100 Subject: [PATCH 064/114] Allow to exclude subdirectories by substrings when using the classes DirectorySearch and FileSearch. --- scons/util/files.py | 67 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 63 insertions(+), 4 deletions(-) diff --git a/scons/util/files.py b/scons/util/files.py index 824b7b093..95314452f 100644 --- a/scons/util/files.py +++ b/scons/util/files.py @@ -34,7 +34,7 @@ def set_recursive(self, recursive: bool) -> 'DirectorySearch': def exclude(self, *excludes: Filter) -> 'DirectorySearch': """ - Sets one or several filters that should be used for excluding subdirectories. + Adds one or several filters that should be used for excluding subdirectories. :param excludes: The filters to be set :return: The `DirectorySearch` itself @@ -44,7 +44,7 @@ def exclude(self, *excludes: Filter) -> 'DirectorySearch': def exclude_by_name(self, *names: str) -> 'DirectorySearch': """ - Sets one or several filters that should be used for excluding subdirectories by their names. + Adds one or several filters that should be used for excluding subdirectories by their names. :param names: The names of the subdirectories to be excluded :return: The `DirectorySearch` itself @@ -55,6 +55,38 @@ def filter_directory(excluded_name: str, _: str, directory_name: str): return self.exclude(*[partial(filter_directory, name) for name in names]) + def exclude_by_substrings(self, + starts_with: Optional[str] = None, + not_starts_with: Optional[str] = None, + ends_with: Optional[str] = None, + not_ends_with: Optional[str] = None, + contains: Optional[str] = None, + not_contains: Optional[str] = None) -> 'DirectorySearch': + """ + Adds a filter that that should be used for excluding subdirectories based on whether their name contains + specific substrings. + + :param starts_with: A substring, names must start with or None, if no restrictions should be imposed + :param not_starts_with: A substring, names must not start with or None, if no restrictions should be imposed + :param ends_with: A substring, names must end with or None, if no restrictions should be imposed + :param not_ends_with: A substring, names must not end with or None, if no restrictions should be imposed + :param contains: A substring, names must contain or None, if no restrictions should be imposed + :param not_contains: A substring, names must not contain or None, if no restrictions should be imposed + :return: The `DirectorySearch` itself + """ + + def filter_directory(start: Optional[str], not_start: Optional[str], end: Optional[str], not_end: Optional[str], + substring: Optional[str], not_substring: Optional[str], _: str, file_name: str): + return (not start or file_name.startswith(start)) \ + and (not not_start or not file_name.startswith(not_start)) \ + and (not end or file_name.endswith(end)) \ + and (not not_end or file_name.endswith(not_end)) \ + and (not substring or file_name.find(substring) >= 0) \ + and (not not_substring or file_name.find(not_substring) < 0) + + return self.exclude( + partial(filter_directory, starts_with, not_starts_with, ends_with, not_ends_with, contains, not_contains)) + def list(self, *directories: str) -> List[str]: """ Lists all subdirectories that can be found in given directories. @@ -103,7 +135,7 @@ def set_recursive(self, recursive: bool) -> 'FileSearch': def exclude_subdirectories(self, *excludes: DirectorySearch.Filter) -> 'FileSearch': """ - Sets one or several filters that should be used for excluding subdirectories. Does only have an effect if the + Adds one or several filters that should be used for excluding subdirectories. Does only have an effect if the search is recursive. :param excludes: The filters to be set @@ -114,7 +146,7 @@ def exclude_subdirectories(self, *excludes: DirectorySearch.Filter) -> 'FileSear def exclude_subdirectories_by_name(self, *names: str) -> 'FileSearch': """ - Sets one or several filters that should be used for excluding subdirectories by their names. Does only have an + Adds one or several filters that should be used for excluding subdirectories by their names. Does only have an effect if the search is recursive. :param names: The names of the subdirectories to be excluded @@ -123,6 +155,33 @@ def exclude_subdirectories_by_name(self, *names: str) -> 'FileSearch': self.directory_search.exclude_by_name(*names) return self + def exclude_subdirectories_by_substrings(self, + starts_with: Optional[str] = None, + not_starts_with: Optional[str] = None, + ends_with: Optional[str] = None, + not_ends_with: Optional[str] = None, + contains: Optional[str] = None, + not_contains: Optional[str] = None) -> 'FileSearch': + """ + Adds a filter that should be used for excluding subdirectories based on whether their name contains specific + substrings. + + :param starts_with: A substring, names must start with or None, if no restrictions should be imposed + :param not_starts_with: A substring, names must not start with or None, if no restrictions should be imposed + :param ends_with: A substring, names must end with or None, if no restrictions should be imposed + :param not_ends_with: A substring, names must not end with or None, if no restrictions should be imposed + :param contains: A substring, names must contain or None, if no restrictions should be imposed + :param not_contains: A substring, names must not contain or None, if no restrictions should be imposed + :return: The `FileSearch` itself + """ + self.directory_search.exclude_by_substrings(starts_with=starts_with, + not_starts_with=not_starts_with, + ends_with=ends_with, + not_ends_with=not_ends_with, + contains=contains, + not_contains=not_contains) + return self + def set_hidden(self, hidden: bool) -> 'FileSearch': """ Sets whether hidden files should be included or not. From b1a2647cd107b4544acc103812ef5b4d94e58f6e Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Tue, 3 Dec 2024 00:05:54 +0100 Subject: [PATCH 065/114] Dynamically register targets and modules for compiling code. --- scons/__init__.py | 104 +++++++++++++++++----- scons/compilation/__init__.py | 18 ++++ scons/compilation/cpp/__init__.py | 27 ++++++ scons/compilation/cpp/targets.py | 70 +++++++++++++++ scons/compilation/cython/__init__.py | 27 ++++++ scons/compilation/cython/requirements.txt | 1 + scons/compilation/cython/targets.py | 69 ++++++++++++++ scons/compilation/meson.py | 41 +++++---- scons/compilation/modules.py | 32 ++++++- scons/compilation/requirements.txt | 2 + scons/cpp.py | 42 --------- scons/cython.py | 39 -------- scons/sconstruct.py | 72 ++------------- scons/util/requirements.txt | 3 - 14 files changed, 358 insertions(+), 189 deletions(-) create mode 100644 scons/compilation/cpp/__init__.py create mode 100644 scons/compilation/cpp/targets.py create mode 100644 scons/compilation/cython/__init__.py create mode 100644 scons/compilation/cython/requirements.txt create mode 100644 scons/compilation/cython/targets.py create mode 100644 scons/compilation/requirements.txt delete mode 100644 scons/cpp.py delete mode 100644 scons/cython.py diff --git a/scons/__init__.py b/scons/__init__.py index 6da3e6424..43bb5677b 100644 --- a/scons/__init__.py +++ b/scons/__init__.py @@ -4,29 +4,93 @@ Defines modules to be dealt with by the build system. """ from code_style.modules import CodeModule +from compilation.modules import CompilationModule from dependencies.python.modules import DependencyType, PythonDependencyModule from util.files import FileSearch from util.languages import Language MODULES = [ - CodeModule(Language.YAML, '.', - FileSearch().set_recursive(False).set_hidden(True)), - CodeModule(Language.YAML, '.github'), - CodeModule(Language.MARKDOWN, '.', - FileSearch().set_recursive(False)), - CodeModule(Language.MARKDOWN, 'doc'), - CodeModule(Language.MARKDOWN, 'python'), - CodeModule(Language.PYTHON, 'scons'), - CodeModule(Language.PYTHON, 'doc'), - CodeModule(Language.PYTHON, 'python', - FileSearch().set_recursive(True).exclude_subdirectories_by_name('build')), - CodeModule(Language.CYTHON, 'python', - FileSearch().set_recursive(True).exclude_subdirectories_by_name('build')), - CodeModule(Language.CPP, 'cpp', - FileSearch().set_recursive(True).exclude_subdirectories_by_name('build')), - PythonDependencyModule(DependencyType.BUILD_TIME, 'scons', - FileSearch().set_recursive(True)), - PythonDependencyModule(DependencyType.BUILD_TIME, 'doc', - FileSearch().set_recursive(True)), - PythonDependencyModule(DependencyType.RUNTIME, 'python') + CodeModule( + language=Language.YAML, + root_directory='.', + file_search=FileSearch().set_recursive(False).set_hidden(True), + ), + CodeModule( + language=Language.YAML, + root_directory='.github', + ), + CodeModule( + language=Language.MARKDOWN, + root_directory='.', + file_search=FileSearch().set_recursive(False), + ), + CodeModule( + language=Language.MARKDOWN, + root_directory='doc', + ), + CodeModule( + language=Language.MARKDOWN, + root_directory='python', + ), + CodeModule( + language=Language.PYTHON, + root_directory='scons', + ), + CodeModule( + language=Language.PYTHON, + root_directory='doc', + ), + CodeModule( + language=Language.PYTHON, + root_directory='python', + file_search=FileSearch() \ + .set_recursive(True) \ + .exclude_subdirectories_by_name('build', 'dist', '__pycache__') \ + .exclude_subdirectories_by_substrings(ends_with='.egg.info'), + ), + CodeModule( + language=Language.CYTHON, + root_directory='python', + file_search=FileSearch() \ + .set_recursive(True) \ + .exclude_subdirectories_by_name('build', 'dist', '__pycache__') \ + .exclude_subdirectories_by_substrings(ends_with='.egg-info'), + ), + CodeModule( + language=Language.CPP, + root_directory='cpp', + file_search=FileSearch().set_recursive(True).exclude_subdirectories_by_name('build'), + ), + PythonDependencyModule( + dependency_type=DependencyType.BUILD_TIME, + root_directory='scons', + file_search=FileSearch().set_recursive(True), + ), + PythonDependencyModule( + dependency_type=DependencyType.BUILD_TIME, + root_directory='doc', + file_search=FileSearch().set_recursive(True), + ), + PythonDependencyModule( + dependency_type=DependencyType.RUNTIME, + root_directory='python', + ), + CompilationModule( + language=Language.CPP, + root_directory='cpp', + install_directory='python', + file_search=FileSearch() \ + .filter_by_substrings(starts_with='lib', contains='.so') \ + .filter_by_substrings(ends_with='.dylib') \ + .filter_by_substrings(starts_with='mlrl', ends_with='.lib') \ + .filter_by_substrings(ends_with='.dll'), + ), + CompilationModule( + language=Language.CYTHON, + root_directory='python', + file_search=FileSearch() \ + .filter_by_substrings(not_starts_with='lib', ends_with='.so') \ + .filter_by_substrings(ends_with='.pyd') \ + .filter_by_substrings(not_starts_with='mlrl', ends_with='.lib'), + ) ] diff --git a/scons/compilation/__init__.py b/scons/compilation/__init__.py index e69de29bb..cd1f3a401 100644 --- a/scons/compilation/__init__.py +++ b/scons/compilation/__init__.py @@ -0,0 +1,18 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Defines targets for compiling code. +""" +from compilation.cpp import COMPILE_CPP, INSTALL_CPP +from compilation.cython import COMPILE_CYTHON, INSTALL_CYTHON +from util.targets import TargetBuilder +from util.units import BuildUnit + +TARGETS = TargetBuilder(BuildUnit('compilation')) \ + .add_phony_target('compile') \ + .depends_on(COMPILE_CPP, COMPILE_CYTHON, clean_dependencies=True) \ + .nop() \ + .add_phony_target('install') \ + .depends_on(INSTALL_CPP, INSTALL_CYTHON, clean_dependencies=True) \ + .nop() \ + .build() diff --git a/scons/compilation/cpp/__init__.py b/scons/compilation/cpp/__init__.py new file mode 100644 index 000000000..25f15c028 --- /dev/null +++ b/scons/compilation/cpp/__init__.py @@ -0,0 +1,27 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Defines targets for compiling C++ code. +""" +from compilation.cpp.targets import CompileCpp, InstallCpp, SetupCpp +from dependencies.python import VENV +from util.targets import PhonyTarget, TargetBuilder +from util.units import BuildUnit + +SETUP_CPP = 'setup_cpp' + +COMPILE_CPP = 'compile_cpp' + +INSTALL_CPP = 'install_cpp' + +TARGETS = TargetBuilder(BuildUnit('compilation', 'cpp')) \ + .add_build_target(SETUP_CPP) \ + .depends_on(VENV) \ + .set_runnables(SetupCpp()) \ + .add_phony_target(COMPILE_CPP) \ + .depends_on(SETUP_CPP) \ + .set_runnables(CompileCpp()) \ + .add_build_target(INSTALL_CPP) \ + .depends_on(COMPILE_CPP) \ + .set_runnables(InstallCpp()) \ + .build() diff --git a/scons/compilation/cpp/targets.py b/scons/compilation/cpp/targets.py new file mode 100644 index 000000000..59a1be11d --- /dev/null +++ b/scons/compilation/cpp/targets.py @@ -0,0 +1,70 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Implements targets for compiling C++ code. +""" +from functools import reduce +from typing import List + +from compilation.build_options import BuildOptions, EnvBuildOption +from compilation.meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup +from compilation.modules import CompilationModule +from util.languages import Language +from util.modules import ModuleRegistry +from util.targets import BuildTarget, PhonyTarget +from util.units import BuildUnit + +MODULE_FILTER = CompilationModule.Filter(Language.CPP) + +BUILD_OPTIONS = BuildOptions() \ + .add(EnvBuildOption(name='subprojects')) \ + .add(EnvBuildOption(name='test_support', subpackage='common')) \ + .add(EnvBuildOption(name='multi_threading_support', subpackage='common')) \ + .add(EnvBuildOption(name='gpu_support', subpackage='common')) + + +class SetupCpp(BuildTarget.Runnable): + """ + Sets up the build system for compiling C++ code. + """ + + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + for module in modules.lookup(MODULE_FILTER): + MesonSetup(build_unit, module, build_options=BUILD_OPTIONS).run() + + def get_output_files(self, modules: ModuleRegistry) -> List[str]: + return [module.build_directory for module in modules.lookup(MODULE_FILTER)] + + def get_clean_files(self, modules: ModuleRegistry) -> List[str]: + print('Removing C++ build files...') + return super().get_clean_files(modules) + + +class CompileCpp(PhonyTarget.Runnable): + """ + Compiles C++ code. + """ + + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + print('Compiling C++ code...') + + for module in modules.lookup(MODULE_FILTER): + MesonConfigure(build_unit, module, BUILD_OPTIONS).run() + MesonCompile(build_unit, module).run() + + +class InstallCpp(BuildTarget.Runnable): + """ + Installs shared libraries into the source tree. + """ + + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + print('Installing shared libraries into source tree...') + + for module in modules.lookup(MODULE_FILTER): + MesonInstall(build_unit, module).run() + + def get_clean_files(self, modules: ModuleRegistry) -> List[str]: + print('Removing shared libraries from source tree...') + compilation_modules = modules.lookup(MODULE_FILTER) + return reduce(lambda aggr, module: aggr + module.find_installed_files(), compilation_modules, []) diff --git a/scons/compilation/cython/__init__.py b/scons/compilation/cython/__init__.py new file mode 100644 index 000000000..be4d69e31 --- /dev/null +++ b/scons/compilation/cython/__init__.py @@ -0,0 +1,27 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Defines targets for compiling C++ code. +""" +from compilation.cpp import COMPILE_CPP +from compilation.cython.targets import CompileCython, InstallCython, SetupCython +from util.targets import PhonyTarget, TargetBuilder +from util.units import BuildUnit + +SETUP_CYTHON = 'setup_cython' + +COMPILE_CYTHON = 'compile_cython' + +INSTALL_CYTHON = 'install_cython' + +TARGETS = TargetBuilder(BuildUnit('compilation', 'cython')) \ + .add_build_target(SETUP_CYTHON) \ + .depends_on(COMPILE_CPP) \ + .set_runnables(SetupCython()) \ + .add_phony_target(COMPILE_CYTHON) \ + .depends_on(SETUP_CYTHON) \ + .set_runnables(CompileCython()) \ + .add_build_target(INSTALL_CYTHON) \ + .depends_on(COMPILE_CYTHON) \ + .set_runnables(InstallCython()) \ + .build() diff --git a/scons/compilation/cython/requirements.txt b/scons/compilation/cython/requirements.txt new file mode 100644 index 000000000..14997b329 --- /dev/null +++ b/scons/compilation/cython/requirements.txt @@ -0,0 +1 @@ +cython >= 3.0, < 3.1 diff --git a/scons/compilation/cython/targets.py b/scons/compilation/cython/targets.py new file mode 100644 index 000000000..281631643 --- /dev/null +++ b/scons/compilation/cython/targets.py @@ -0,0 +1,69 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Implements targets for compiling Cython code. +""" +from functools import reduce +from typing import List + +from compilation.build_options import BuildOptions, EnvBuildOption +from compilation.meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup +from compilation.modules import CompilationModule +from util.languages import Language +from util.modules import ModuleRegistry +from util.targets import BuildTarget, PhonyTarget +from util.units import BuildUnit + +MODULE_FILTER = CompilationModule.Filter(Language.CYTHON) + +BUILD_OPTIONS = BuildOptions() \ + .add(EnvBuildOption(name='subprojects')) + + +class SetupCython(BuildTarget.Runnable): + """ + Sets up the build system for compiling the Cython code. + """ + + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + for module in modules.lookup(MODULE_FILTER): + MesonSetup(build_unit, module) \ + .add_dependencies('cython') \ + .run() + + def get_output_files(self, modules: ModuleRegistry) -> List[str]: + return [module.build_directory for module in modules.lookup(MODULE_FILTER)] + + def get_clean_files(self, modules: ModuleRegistry) -> List[str]: + print('Removing Cython build files...') + return super().get_clean_files(modules) + + +class CompileCython(PhonyTarget.Runnable): + """ + Compiles the Cython code. + """ + + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + print('Compiling Cython code...') + + for module in modules.lookup(MODULE_FILTER): + MesonConfigure(build_unit, module, build_options=BUILD_OPTIONS) + MesonCompile(build_unit, module).run() + + +class InstallCython(BuildTarget.Runnable): + """ + Installs extension modules into the source tree. + """ + + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + print('Installing extension modules into source tree...') + + for module in modules.lookup(MODULE_FILTER): + MesonInstall(build_unit, module).run() + + def get_clean_files(self, modules: ModuleRegistry) -> List[str]: + print('Removing extension modules from source tree...') + compilation_modules = modules.lookup(MODULE_FILTER) + return reduce(lambda aggr, module: aggr + module.find_installed_files(), compilation_modules, []) diff --git a/scons/compilation/meson.py b/scons/compilation/meson.py index 5764b7086..82e4e62af 100644 --- a/scons/compilation/meson.py +++ b/scons/compilation/meson.py @@ -7,7 +7,9 @@ from typing import List from compilation.build_options import BuildOptions +from compilation.modules import CompilationModule from util.run import Program +from util.units import BuildUnit def build_options_as_meson_arguments(build_options: BuildOptions) -> List[str]: @@ -32,13 +34,15 @@ class Meson(Program, ABC): An abstract base class for all classes that allow to run the external program "meson". """ - def __init__(self, meson_command: str, *arguments: str): + def __init__(self, build_unit: BuildUnit, meson_command: str, *arguments: str): """ + :param build_unit: The build unit from which the program should be run :param program: The meson command to be run :param arguments: Optional arguments to be passed to meson """ super().__init__('meson', meson_command, *arguments) self.print_arguments(True) + self.set_build_unit(build_unit) class MesonSetup(Meson): @@ -46,13 +50,15 @@ class MesonSetup(Meson): Allows to run the external program "meson setup". """ - def __init__(self, build_directory: str, source_directory: str, build_options: BuildOptions = BuildOptions()): + def __init__(self, build_unit: BuildUnit, module: CompilationModule, build_options: BuildOptions = BuildOptions()): """ - :param build_directory: The path to the build directory - :param source_directory: The path to the source directory - :param build_options: The build options to be used + :param build_unit: The build unit from which the program should be run + :param module: The module, the program should be applied to + :param build_options: The build options to be used """ - super().__init__('setup', *build_options_as_meson_arguments(build_options), build_directory, source_directory) + super().__init__(build_unit, 'setup', *build_options_as_meson_arguments(build_options), module.build_directory, + module.root_directory) + self.add_dependencies('ninja') class MesonConfigure(Meson): @@ -60,12 +66,14 @@ class MesonConfigure(Meson): Allows to run the external program "meson configure". """ - def __init__(self, build_directory: str, build_options: BuildOptions = BuildOptions()): + def __init__(self, build_unit: BuildUnit, module: CompilationModule, build_options: BuildOptions = BuildOptions()): """ - :param build_directory: The path to the build directory + :param build_unit: The build unit from which the program should be run + :param module: The module, the program should be applied to :param build_options: The build options to be used """ - super().__init__('configure', *build_options_as_meson_arguments(build_options), build_directory) + super().__init__(build_unit, 'configure', *build_options_as_meson_arguments(build_options), + module.build_directory) self.build_options = build_options def _should_be_skipped(self) -> bool: @@ -80,11 +88,12 @@ class MesonCompile(Meson): Allows to run the external program "meson compile". """ - def __init__(self, build_directory: str): + def __init__(self, build_unit: BuildUnit, module: CompilationModule): """ - :param build_directory: The path to the build directory + :param build_unit: The build unit from which the program should be run + :param module: The module, the program should be applied to """ - super().__init__('compile', '-C', build_directory) + super().__init__(build_unit, 'compile', '-C', module.build_directory) class MesonInstall(Meson): @@ -92,8 +101,10 @@ class MesonInstall(Meson): Allows to run the external program "meson install". """ - def __init__(self, build_directory: str): + def __init__(self, build_unit: BuildUnit, module: CompilationModule): """ - :param build_directory: The path to the build directory + :param build_unit: The build unit from which the program should be run + :param module: The module, the program should be applied to """ - super().__init__('install', '--no-rebuild', '--only-changed', '-C', build_directory) + super().__init__(build_unit, 'install', '--no-rebuild', '--only-changed', '-C', module.build_directory) + self.install_program(False) diff --git a/scons/compilation/modules.py b/scons/compilation/modules.py index 1590059ed..0e5adc5af 100644 --- a/scons/compilation/modules.py +++ b/scons/compilation/modules.py @@ -4,7 +4,9 @@ Provides classes that provide access to directories and files that belong to individual modules. """ from os import path +from typing import List, Optional +from util.files import FileSearch from util.languages import Language from util.modules import Module @@ -29,13 +31,23 @@ def __init__(self, *languages: Language): def matches(self, module: Module) -> bool: return isinstance(module, CompilationModule) and (not self.languages or module.language in self.languages) - def __init__(self, language: Language, root_directory: str): + def __init__(self, + language: Language, + root_directory: str, + install_directory: Optional[str] = None, + file_search: Optional[FileSearch] = None): """ - :param language: The programming language of the source code that belongs to the module - :param root_directory: The path to the module's root directory + :param language: The programming language of the source code that belongs to the module + :param root_directory: The path to the module's root directory + :param install_directory: The path to the directory into which files are installed or None, if the files are + installed into the root directory + :param file_search: The `FileSearch` that should be used to search for installed files or None, if the + module does never contain any installed files """ self.language = language self.root_directory = root_directory + self.install_directory = install_directory if install_directory else root_directory + self.file_search = file_search @property def build_directory(self) -> str: @@ -43,3 +55,17 @@ def build_directory(self) -> str: The path to the directory, where build files should be stored. """ return path.join(self.root_directory, 'build') + + def find_installed_files(self) -> List[str]: + """ + Finds and returns all installed files that belong to the module. + + :return: A list that contains the paths of the requirements files that have been found + """ + if self.file_search: + return self.file_search \ + .set_recursive(True) \ + .exclude_subdirectories_by_name(path.basename(self.build_directory)) \ + .list(self.install_directory) + + return [] diff --git a/scons/compilation/requirements.txt b/scons/compilation/requirements.txt new file mode 100644 index 000000000..8611b61e2 --- /dev/null +++ b/scons/compilation/requirements.txt @@ -0,0 +1,2 @@ +meson >= 1.6, < 1.7 +ninja >= 1.11, < 1.12 diff --git a/scons/cpp.py b/scons/cpp.py deleted file mode 100644 index ee74f4aee..000000000 --- a/scons/cpp.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Provides utility functions for compiling C++ code. -""" -from compilation.build_options import BuildOptions, EnvBuildOption -from compilation.meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup -from modules_old import CPP_MODULE - -BUILD_OPTIONS = BuildOptions() \ - .add(EnvBuildOption(name='subprojects')) \ - .add(EnvBuildOption(name='test_support', subpackage='common')) \ - .add(EnvBuildOption(name='multi_threading_support', subpackage='common')) \ - .add(EnvBuildOption(name='gpu_support', subpackage='common')) - - -def setup_cpp(**_): - """ - Sets up the build system for compiling the C++ code. - """ - MesonSetup(build_directory=CPP_MODULE.build_dir, - source_directory=CPP_MODULE.root_dir, - build_options=BUILD_OPTIONS) \ - .add_dependencies('ninja') \ - .run() - - -def compile_cpp(**_): - """ - Compiles the C++ code. - """ - MesonConfigure(CPP_MODULE.build_dir, BUILD_OPTIONS).run() - print('Compiling C++ code...') - MesonCompile(CPP_MODULE.build_dir).run() - - -def install_cpp(**_): - """ - Installs shared libraries into the source tree. - """ - print('Installing shared libraries into source tree...') - MesonInstall(CPP_MODULE.build_dir).run() diff --git a/scons/cython.py b/scons/cython.py deleted file mode 100644 index 80afba425..000000000 --- a/scons/cython.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Provides utility functions for compiling Cython code. -""" -from compilation.build_options import BuildOptions, EnvBuildOption -from compilation.meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup -from modules_old import PYTHON_MODULE - -BUILD_OPTIONS = BuildOptions() \ - .add(EnvBuildOption(name='subprojects')) - - -def setup_cython(**_): - """ - Sets up the build system for compiling the Cython code. - """ - MesonSetup(build_directory=PYTHON_MODULE.build_dir, - source_directory=PYTHON_MODULE.root_dir, - build_options=BUILD_OPTIONS) \ - .add_dependencies('cython') \ - .run() - - -def compile_cython(**_): - """ - Compiles the Cython code. - """ - MesonConfigure(PYTHON_MODULE.build_dir, BUILD_OPTIONS) - print('Compiling Cython code...') - MesonCompile(PYTHON_MODULE.build_dir).run() - - -def install_cython(**_): - """ - Installs extension modules into the source tree. - """ - print('Installing extension modules into source tree...') - MesonInstall(PYTHON_MODULE.build_dir).run() diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 2bc3e718a..d4a07f647 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -7,8 +7,6 @@ from os import path -from cpp import compile_cpp, install_cpp, setup_cpp -from cython import compile_cython, install_cython, setup_cython from documentation import apidoc_cpp, apidoc_cpp_tocfile, apidoc_python, apidoc_python_tocfile, doc from modules_old import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE from packaging import build_python_wheel, install_python_wheels @@ -32,12 +30,6 @@ def __print_if_clean(environment, message: str): # Define target names... -TARGET_NAME_COMPILE = 'compile' -TARGET_NAME_COMPILE_CPP = TARGET_NAME_COMPILE + '_cpp' -TARGET_NAME_COMPILE_CYTHON = TARGET_NAME_COMPILE + '_cython' -TARGET_NAME_INSTALL = 'install' -TARGET_NAME_INSTALL_CPP = TARGET_NAME_INSTALL + '_cpp' -TARGET_NAME_INSTALL_CYTHON = TARGET_NAME_INSTALL + '_cython' TARGET_NAME_BUILD_WHEELS = 'build_wheels' TARGET_NAME_INSTALL_WHEELS = 'install_wheels' TARGET_NAME_TESTS = 'tests' @@ -49,10 +41,8 @@ def __print_if_clean(environment, message: str): TARGET_NAME_DOC = 'doc' VALID_TARGETS = { - TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP, TARGET_NAME_COMPILE_CYTHON, TARGET_NAME_INSTALL, - TARGET_NAME_INSTALL_CPP, TARGET_NAME_INSTALL_CYTHON, TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS, - TARGET_NAME_TESTS, TARGET_NAME_TESTS_CPP, TARGET_NAME_TESTS_PYTHON, TARGET_NAME_APIDOC, TARGET_NAME_APIDOC_CPP, - TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC + TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS, TARGET_NAME_TESTS, TARGET_NAME_TESTS_CPP, + TARGET_NAME_TESTS_PYTHON, TARGET_NAME_APIDOC, TARGET_NAME_APIDOC_CPP, TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC } DEFAULT_TARGET = TARGET_NAME_INSTALL_WHEELS @@ -88,58 +78,6 @@ def __print_if_clean(environment, message: str): # Create temporary file ".sconsign.dblite" in the build directory... env.SConsignFile(name=path.relpath(path.join(BUILD_MODULE.build_dir, '.sconsign'), BUILD_MODULE.root_dir)) -# Define targets for compiling the C++ and Cython code... -env.Command(CPP_MODULE.build_dir, None, action=setup_cpp) -target_compile_cpp = __create_phony_target(env, TARGET_NAME_COMPILE_CPP, action=compile_cpp) -env.Depends(target_compile_cpp, ['target_venv', CPP_MODULE.build_dir]) - -env.Command(PYTHON_MODULE.build_dir, None, action=setup_cython) -target_compile_cython = __create_phony_target(env, TARGET_NAME_COMPILE_CYTHON, action=compile_cython) -env.Depends(target_compile_cython, [target_compile_cpp, PYTHON_MODULE.build_dir]) - -target_compile = __create_phony_target(env, TARGET_NAME_COMPILE) -env.Depends(target_compile, [target_compile_cpp, target_compile_cython]) - -# Define targets for cleaning up C++ and Cython build directories... -if not COMMAND_LINE_TARGETS \ - or TARGET_NAME_COMPILE_CPP in COMMAND_LINE_TARGETS \ - or TARGET_NAME_COMPILE in COMMAND_LINE_TARGETS: - __print_if_clean(env, 'Removing C++ build files...') - env.Clean([target_compile_cpp, DEFAULT_TARGET], CPP_MODULE.build_dir) - -if not COMMAND_LINE_TARGETS \ - or TARGET_NAME_COMPILE_CYTHON in COMMAND_LINE_TARGETS \ - or TARGET_NAME_COMPILE in COMMAND_LINE_TARGETS: - __print_if_clean(env, 'Removing Cython build files...') - env.Clean([target_compile_cython, DEFAULT_TARGET], PYTHON_MODULE.build_dir) - -# Define targets for installing shared libraries and extension modules into the source tree... -target_install_cpp = __create_phony_target(env, TARGET_NAME_INSTALL_CPP, action=install_cpp) -env.Depends(target_install_cpp, target_compile_cpp) - -target_install_cython = __create_phony_target(env, TARGET_NAME_INSTALL_CYTHON, action=install_cython) -env.Depends(target_install_cython, target_compile_cython) - -target_install = env.Alias(TARGET_NAME_INSTALL, None, None) -env.Depends(target_install, [target_install_cpp, target_install_cython]) - -# Define targets for removing shared libraries and extension modules from the source tree... -if not COMMAND_LINE_TARGETS \ - or TARGET_NAME_INSTALL_CPP in COMMAND_LINE_TARGETS \ - or TARGET_NAME_INSTALL in COMMAND_LINE_TARGETS: - __print_if_clean(env, 'Removing shared libraries from source tree...') - - for subproject in PYTHON_MODULE.find_subprojects(return_all=True): - env.Clean([target_install_cpp, DEFAULT_TARGET], subproject.find_shared_libraries()) - -if not COMMAND_LINE_TARGETS \ - or TARGET_NAME_INSTALL_CYTHON in COMMAND_LINE_TARGETS \ - or TARGET_NAME_INSTALL in COMMAND_LINE_TARGETS: - __print_if_clean(env, 'Removing extension modules from source tree...') - - for subproject in PYTHON_MODULE.find_subprojects(return_all=True): - env.Clean([target_install_cython, DEFAULT_TARGET], subproject.find_extension_modules()) - # Define targets for building and installing Python wheels... commands_build_wheels = [] commands_install_wheels = [] @@ -156,10 +94,10 @@ def __print_if_clean(environment, message: str): commands_install_wheels.append(command_install_wheels) target_build_wheels = env.Alias(TARGET_NAME_BUILD_WHEELS, None, None) -env.Depends(target_build_wheels, [target_install] + commands_build_wheels) +env.Depends(target_build_wheels, ['target_install'] + commands_build_wheels) target_install_wheels = env.Alias(TARGET_NAME_INSTALL_WHEELS, None, None) -env.Depends(target_install_wheels, [target_install] + commands_install_wheels) +env.Depends(target_install_wheels, ['target_install'] + commands_install_wheels) # Define target for cleaning up Python wheels and associated build directories... if not COMMAND_LINE_TARGETS or TARGET_NAME_BUILD_WHEELS in COMMAND_LINE_TARGETS: @@ -170,7 +108,7 @@ def __print_if_clean(environment, message: str): # Define targets for running automated tests... target_tests_cpp = __create_phony_target(env, TARGET_NAME_TESTS_CPP, action=tests_cpp) -env.Depends(target_tests_cpp, target_compile_cpp) +env.Depends(target_tests_cpp, 'target_compile_cpp') target_tests_python = __create_phony_target(env, TARGET_NAME_TESTS_PYTHON, action=tests_python) env.Depends(target_tests_python, target_install_wheels) diff --git a/scons/util/requirements.txt b/scons/util/requirements.txt index 4ba9aa2ea..2e8319af6 100644 --- a/scons/util/requirements.txt +++ b/scons/util/requirements.txt @@ -1,7 +1,4 @@ build >= 1.2, < 1.3 -cython >= 3.0, < 3.1 -meson >= 1.6, < 1.7 -ninja >= 1.11, < 1.12 setuptools scons >= 4.8, < 4.9 unittest-xml-reporting >= 3.2, < 3.3 From 913b9dc15dd8b5ea83d82ecb5a43e254299145a3 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Tue, 3 Dec 2024 00:57:15 +0100 Subject: [PATCH 066/114] Dynamically register targets and modules for testing C++ code. --- scons/__init__.py | 4 +++ scons/sconstruct.py | 12 +++------ scons/testing/__init__.py | 0 scons/testing/cpp/__init__.py | 15 +++++++++++ scons/testing/cpp/meson.py | 23 ++++++++++++++++ scons/testing/cpp/modules.py | 36 +++++++++++++++++++++++++ scons/testing/cpp/targets.py | 20 ++++++++++++++ scons/testing/modules.py | 23 ++++++++++++++++ scons/{testing.py => testing_python.py} | 13 ++------- 9 files changed, 127 insertions(+), 19 deletions(-) create mode 100644 scons/testing/__init__.py create mode 100644 scons/testing/cpp/__init__.py create mode 100644 scons/testing/cpp/meson.py create mode 100644 scons/testing/cpp/modules.py create mode 100644 scons/testing/cpp/targets.py create mode 100644 scons/testing/modules.py rename scons/{testing.py => testing_python.py} (77%) diff --git a/scons/__init__.py b/scons/__init__.py index 43bb5677b..575c46b2b 100644 --- a/scons/__init__.py +++ b/scons/__init__.py @@ -6,6 +6,7 @@ from code_style.modules import CodeModule from compilation.modules import CompilationModule from dependencies.python.modules import DependencyType, PythonDependencyModule +from testing.cpp.modules import CppTestModule from util.files import FileSearch from util.languages import Language @@ -92,5 +93,8 @@ .filter_by_substrings(not_starts_with='lib', ends_with='.so') \ .filter_by_substrings(ends_with='.pyd') \ .filter_by_substrings(not_starts_with='mlrl', ends_with='.lib'), + ), + CppTestModule( + root_directory='cpp', ) ] diff --git a/scons/sconstruct.py b/scons/sconstruct.py index d4a07f647..b1419d71c 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -10,7 +10,7 @@ from documentation import apidoc_cpp, apidoc_cpp_tocfile, apidoc_python, apidoc_python_tocfile, doc from modules_old import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE from packaging import build_python_wheel, install_python_wheels -from testing import tests_cpp, tests_python +from testing_python import tests_python from util.files import FileSearch from util.format import format_iterable from util.modules import Module, ModuleRegistry @@ -33,7 +33,6 @@ def __print_if_clean(environment, message: str): TARGET_NAME_BUILD_WHEELS = 'build_wheels' TARGET_NAME_INSTALL_WHEELS = 'install_wheels' TARGET_NAME_TESTS = 'tests' -TARGET_NAME_TESTS_CPP = TARGET_NAME_TESTS + '_cpp' TARGET_NAME_TESTS_PYTHON = TARGET_NAME_TESTS + '_python' TARGET_NAME_APIDOC = 'apidoc' TARGET_NAME_APIDOC_CPP = TARGET_NAME_APIDOC + '_cpp' @@ -41,8 +40,8 @@ def __print_if_clean(environment, message: str): TARGET_NAME_DOC = 'doc' VALID_TARGETS = { - TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS, TARGET_NAME_TESTS, TARGET_NAME_TESTS_CPP, - TARGET_NAME_TESTS_PYTHON, TARGET_NAME_APIDOC, TARGET_NAME_APIDOC_CPP, TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC + TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS, TARGET_NAME_TESTS, TARGET_NAME_TESTS_PYTHON, + TARGET_NAME_APIDOC, TARGET_NAME_APIDOC_CPP, TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC } DEFAULT_TARGET = TARGET_NAME_INSTALL_WHEELS @@ -107,14 +106,11 @@ def __print_if_clean(environment, message: str): env.Clean([target_build_wheels, DEFAULT_TARGET], subproject.build_dirs) # Define targets for running automated tests... -target_tests_cpp = __create_phony_target(env, TARGET_NAME_TESTS_CPP, action=tests_cpp) -env.Depends(target_tests_cpp, 'target_compile_cpp') - target_tests_python = __create_phony_target(env, TARGET_NAME_TESTS_PYTHON, action=tests_python) env.Depends(target_tests_python, target_install_wheels) target_tests = __create_phony_target(env, TARGET_NAME_TESTS) -env.Depends(target_tests, [target_tests_cpp, target_tests_python]) +env.Depends(target_tests, ['target_tests_cpp', target_tests_python]) # Define targets for generating the documentation... commands_apidoc_cpp = [] diff --git a/scons/testing/__init__.py b/scons/testing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scons/testing/cpp/__init__.py b/scons/testing/cpp/__init__.py new file mode 100644 index 000000000..7509e615d --- /dev/null +++ b/scons/testing/cpp/__init__.py @@ -0,0 +1,15 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Defines targets for testing C++ code. +""" +from compilation.cpp import COMPILE_CPP +from testing.cpp.targets import TestCpp +from util.targets import PhonyTarget, TargetBuilder +from util.units import BuildUnit + +TARGETS = TargetBuilder(BuildUnit('testing', 'cpp')) \ + .add_phony_target('tests_cpp') \ + .depends_on(COMPILE_CPP) \ + .set_runnables(TestCpp()) \ + .build() diff --git a/scons/testing/cpp/meson.py b/scons/testing/cpp/meson.py new file mode 100644 index 000000000..74103db7a --- /dev/null +++ b/scons/testing/cpp/meson.py @@ -0,0 +1,23 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that allow to run automated tests via the external program "meson". +""" +from compilation.meson import Meson +from testing.cpp.modules import CppTestModule +from util.units import BuildUnit + + +class MesonTest(Meson): + """ + Allows to run the external program "meson test". + """ + + def __init__(self, build_unit: BuildUnit, module: CppTestModule): + """ + :param build_unit: The build unit from which the program should be run + :param module: The module, the program should be applied to + """ + super().__init__(build_unit, 'test', '-C', module.build_directory, '--verbose') + self.add_conditional_arguments(module.fail_fast, '--maxfail', '1') + self.install_program(False) diff --git a/scons/testing/cpp/modules.py b/scons/testing/cpp/modules.py new file mode 100644 index 000000000..6eec3f7c8 --- /dev/null +++ b/scons/testing/cpp/modules.py @@ -0,0 +1,36 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that provide access to automated tests for C++ code that belong to individual modules. +""" +from os import path + +from testing.modules import TestModule +from util.modules import Module + + +class CppTestModule(TestModule): + """ + A module that contains automated tests for C++ code. + """ + + class Filter(Module.Filter): + """ + A filter that matches modules that contain automated tests for C++ code. + """ + + def matches(self, module: Module) -> bool: + return isinstance(module, CppTestModule) + + def __init__(self, root_directory: str): + """ + :param root_directory: The path to the module's root directory + """ + self.root_directory = root_directory + + @property + def build_directory(self) -> str: + """ + The path to the directory, where build files are stored. + """ + return path.join(self.root_directory, 'build') diff --git a/scons/testing/cpp/targets.py b/scons/testing/cpp/targets.py new file mode 100644 index 000000000..13eddba6e --- /dev/null +++ b/scons/testing/cpp/targets.py @@ -0,0 +1,20 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Implements targets for testing C++ code. +""" +from testing.cpp.meson import MesonTest +from testing.cpp.modules import CppTestModule +from util.modules import ModuleRegistry +from util.targets import BuildTarget, PhonyTarget +from util.units import BuildUnit + + +class TestCpp(PhonyTarget.Runnable): + """ + Runs automated tests for C++ code. + """ + + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + for module in modules.lookup(CppTestModule.Filter()): + MesonTest(build_unit, module).run() diff --git a/scons/testing/modules.py b/scons/testing/modules.py new file mode 100644 index 000000000..5563beefd --- /dev/null +++ b/scons/testing/modules.py @@ -0,0 +1,23 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that provide access to automated tests that belong to individual modules. +""" +from abc import ABC +from os import environ + +from util.env import get_env_bool +from util.modules import Module + + +class TestModule(Module, ABC): + """ + An abstract base class for all modules that contain automated tests. + """ + + @property + def fail_fast(self) -> bool: + """ + True, if all tests should be skipped as soon as a single test fails, False otherwise + """ + return get_env_bool(environ, 'FAIL_FAST') diff --git a/scons/testing.py b/scons/testing_python.py similarity index 77% rename from scons/testing.py rename to scons/testing_python.py index d6516d56f..e4a35ec0c 100644 --- a/scons/testing.py +++ b/scons/testing_python.py @@ -5,18 +5,9 @@ """ from os import environ, path -from modules_old import CPP_MODULE, PYTHON_MODULE +from modules_old import PYTHON_MODULE from util.env import get_env_bool -from util.run import Program, PythonModule - - -def tests_cpp(**_): - """ - Runs all automated tests of C++ code. - """ - Program('meson', 'test', '-C', CPP_MODULE.build_dir, '-v') \ - .print_arguments(True) \ - .run() +from util.run import PythonModule def tests_python(**_): From 2464f3467f9b7852306db95ae6dd643b56e69522 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 4 Dec 2024 00:03:38 +0100 Subject: [PATCH 067/114] Allow to add filters to a DirectorySearch. --- scons/util/files.py | 84 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 78 insertions(+), 6 deletions(-) diff --git a/scons/util/files.py b/scons/util/files.py index 95314452f..f802fee73 100644 --- a/scons/util/files.py +++ b/scons/util/files.py @@ -21,6 +21,7 @@ class DirectorySearch: def __init__(self): self.recursive = False self.excludes = [] + self.filters = [] def set_recursive(self, recursive: bool) -> 'DirectorySearch': """ @@ -32,6 +33,29 @@ def set_recursive(self, recursive: bool) -> 'DirectorySearch': self.recursive = recursive return self + def add_filters(self, *filter_functions: Filter) -> 'DirectorySearch': + """ + Adds one or several filters that match subdirectories to be included. + + :param filter_functions: The filters to be added + :return: The `DirectorySearch` itself + """ + self.filters.extend(filter_functions) + return self + + def filter_by_name(self, *names: str) -> 'DirectorySearch': + """ + Adds one or several filters that match subdirectories to be included based on their name. + + :param names: The names of the subdirectories that should be included + :return: The `DirectorySearch` itself + """ + + def filter_directory(filtered_names: Set[str], _: str, directory_name: str): + return directory_name in filtered_names + + return self.add_filters(*[partial(filter_directory, name) for name in names]) + def exclude(self, *excludes: Filter) -> 'DirectorySearch': """ Adds one or several filters that should be used for excluding subdirectories. @@ -97,14 +121,34 @@ def list(self, *directories: str) -> List[str]: result = [] def filter_file(file: str) -> bool: - return path.isdir(file) and not reduce( - lambda aggr, exclude: aggr or exclude(path.dirname(file), path.basename(file)), self.excludes, False) + if path.isdir(file): + parent = path.dirname(file) + file_name = path.basename(file) + + if not reduce(lambda aggr, exclude: aggr or exclude(parent, file_name), self.excludes, False): + return True + + return False + + def filter_subdirectory(subdirectory: str, filters: List[DirectorySearch.Filter]) -> bool: + parent = path.dirname(subdirectory) + directory_name = path.basename(subdirectory) + + if reduce(lambda aggr, dir_filter: aggr or dir_filter(parent, directory_name), filters, False): + return True + + return False for directory in directories: subdirectories = [file for file in glob(path.join(directory, '*')) if filter_file(file)] if self.recursive: - subdirectories.extend(self.list(*subdirectories)) + result.extend(self.list(*subdirectories)) + + if self.filters: + subdirectories = [ + subdirectory for subdirectory in subdirectories if filter_subdirectory(subdirectory, self.filters) + ] result.extend(subdirectories) @@ -133,6 +177,26 @@ def set_recursive(self, recursive: bool) -> 'FileSearch': self.directory_search.set_recursive(recursive) return self + def add_subdirectory_filters(self, *filter_functions: DirectorySearch.Filter) -> 'FileSearch': + """ + Adds one or several filters that match subdirectories to be included. + + :param filter_functions: The filters to be added + :return: The `FileSearch` itself + """ + self.directory_search.add_filters(*filter_functions) + return self + + def filter_subdirectories_by_name(self, *names: str) -> 'FileSearch': + """ + Adds one or several filters that match subdirectories to be included based on their name. + + :param names: The names of the subdirectories that should be included + :return: The `FileSearch` itself + """ + self.directory_search.filter_by_name(*names) + return self + def exclude_subdirectories(self, *excludes: DirectorySearch.Filter) -> 'FileSearch': """ Adds one or several filters that should be used for excluding subdirectories. Does only have an effect if the @@ -278,9 +342,17 @@ def list(self, *directories: str) -> List[str]: subdirectories = self.directory_search.list(*directories) if self.directory_search.recursive else [] def filter_file(file: str) -> bool: - return path.isfile(file) and (not self.filters or reduce( - lambda aggr, file_filter: aggr or file_filter(path.dirname(file), path.basename(file)), self.filters, - False)) + if path.isfile(file): + if not self.filters: + return True + + parent = path.dirname(file) + file_name = path.basename(file) + + if reduce(lambda aggr, file_filter: aggr or file_filter(parent, file_name), self.filters, False): + return True + + return False for directory in list(directories) + subdirectories: files = [file for file in glob(path.join(directory, '*')) if filter_file(file)] From 31f861beb0cedbfae0564863e8e213dfcd034c65 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 4 Dec 2024 00:05:38 +0100 Subject: [PATCH 068/114] Dynamically register targets and modules for testing Python code. --- scons/__init__.py | 4 +++ scons/sconstruct.py | 14 ++------ scons/testing/__init__.py | 15 ++++++++ scons/testing/cpp/__init__.py | 4 ++- scons/testing/python/__init__.py | 16 +++++++++ scons/testing/python/modules.py | 51 +++++++++++++++++++++++++++ scons/testing/python/requirements.txt | 1 + scons/testing/python/targets.py | 20 +++++++++++ scons/testing/python/unittest.py | 36 +++++++++++++++++++ scons/testing_python.py | 31 ---------------- scons/util/requirements.txt | 1 - 11 files changed, 148 insertions(+), 45 deletions(-) create mode 100644 scons/testing/python/__init__.py create mode 100644 scons/testing/python/modules.py create mode 100644 scons/testing/python/requirements.txt create mode 100644 scons/testing/python/targets.py create mode 100644 scons/testing/python/unittest.py delete mode 100644 scons/testing_python.py diff --git a/scons/__init__.py b/scons/__init__.py index 575c46b2b..4909d5609 100644 --- a/scons/__init__.py +++ b/scons/__init__.py @@ -7,6 +7,7 @@ from compilation.modules import CompilationModule from dependencies.python.modules import DependencyType, PythonDependencyModule from testing.cpp.modules import CppTestModule +from testing.python.modules import PythonTestModule from util.files import FileSearch from util.languages import Language @@ -96,5 +97,8 @@ ), CppTestModule( root_directory='cpp', + ), + PythonTestModule( + root_directory='python', ) ] diff --git a/scons/sconstruct.py b/scons/sconstruct.py index b1419d71c..2b66d999b 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -10,7 +10,6 @@ from documentation import apidoc_cpp, apidoc_cpp_tocfile, apidoc_python, apidoc_python_tocfile, doc from modules_old import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE from packaging import build_python_wheel, install_python_wheels -from testing_python import tests_python from util.files import FileSearch from util.format import format_iterable from util.modules import Module, ModuleRegistry @@ -32,16 +31,14 @@ def __print_if_clean(environment, message: str): # Define target names... TARGET_NAME_BUILD_WHEELS = 'build_wheels' TARGET_NAME_INSTALL_WHEELS = 'install_wheels' -TARGET_NAME_TESTS = 'tests' -TARGET_NAME_TESTS_PYTHON = TARGET_NAME_TESTS + '_python' TARGET_NAME_APIDOC = 'apidoc' TARGET_NAME_APIDOC_CPP = TARGET_NAME_APIDOC + '_cpp' TARGET_NAME_APIDOC_PYTHON = TARGET_NAME_APIDOC + '_python' TARGET_NAME_DOC = 'doc' VALID_TARGETS = { - TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS, TARGET_NAME_TESTS, TARGET_NAME_TESTS_PYTHON, - TARGET_NAME_APIDOC, TARGET_NAME_APIDOC_CPP, TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC + TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS, TARGET_NAME_APIDOC, TARGET_NAME_APIDOC_CPP, + TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC } DEFAULT_TARGET = TARGET_NAME_INSTALL_WHEELS @@ -105,13 +102,6 @@ def __print_if_clean(environment, message: str): for subproject in PYTHON_MODULE.find_subprojects(return_all=True): env.Clean([target_build_wheels, DEFAULT_TARGET], subproject.build_dirs) -# Define targets for running automated tests... -target_tests_python = __create_phony_target(env, TARGET_NAME_TESTS_PYTHON, action=tests_python) -env.Depends(target_tests_python, target_install_wheels) - -target_tests = __create_phony_target(env, TARGET_NAME_TESTS) -env.Depends(target_tests, ['target_tests_cpp', target_tests_python]) - # Define targets for generating the documentation... commands_apidoc_cpp = [] commands_apidoc_python = [] diff --git a/scons/testing/__init__.py b/scons/testing/__init__.py index e69de29bb..63e4a000c 100644 --- a/scons/testing/__init__.py +++ b/scons/testing/__init__.py @@ -0,0 +1,15 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Defines targets for testing code. +""" +from testing.cpp import TESTS_CPP +from testing.python import TESTS_PYTHON +from util.targets import TargetBuilder +from util.units import BuildUnit + +TARGETS = TargetBuilder(BuildUnit('testing')) \ + .add_phony_target('tests') \ + .depends_on(TESTS_CPP, TESTS_PYTHON) \ + .nop() \ + .build() diff --git a/scons/testing/cpp/__init__.py b/scons/testing/cpp/__init__.py index 7509e615d..6e3d23dea 100644 --- a/scons/testing/cpp/__init__.py +++ b/scons/testing/cpp/__init__.py @@ -8,8 +8,10 @@ from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit +TESTS_CPP = 'tests_cpp' + TARGETS = TargetBuilder(BuildUnit('testing', 'cpp')) \ - .add_phony_target('tests_cpp') \ + .add_phony_target(TESTS_CPP) \ .depends_on(COMPILE_CPP) \ .set_runnables(TestCpp()) \ .build() diff --git a/scons/testing/python/__init__.py b/scons/testing/python/__init__.py new file mode 100644 index 000000000..fd379679d --- /dev/null +++ b/scons/testing/python/__init__.py @@ -0,0 +1,16 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Defines targets for testing Python code. +""" +from testing.python.targets import TestPython +from util.targets import PhonyTarget, TargetBuilder +from util.units import BuildUnit + +TESTS_PYTHON = 'tests_python' + +# TODO .depends_on(INSTALL_WHEELS) +TARGETS = TargetBuilder(BuildUnit('testing', 'python')) \ + .add_phony_target(TESTS_PYTHON) \ + .set_runnables(TestPython()) \ + .build() diff --git a/scons/testing/python/modules.py b/scons/testing/python/modules.py new file mode 100644 index 000000000..c00878806 --- /dev/null +++ b/scons/testing/python/modules.py @@ -0,0 +1,51 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that provide access to automated tests for Python code that belong to individual modules. +""" +from os import path +from typing import List + +from testing.modules import TestModule +from util.files import DirectorySearch +from util.modules import Module + + +class PythonTestModule(TestModule): + """ + A module that contains automated tests for Python code. + """ + + class Filter(Module.Filter): + """ + A filter that matches modules that contain automated tests for Python code. + """ + + def matches(self, module: Module) -> bool: + return isinstance(module, PythonTestModule) + + def __init__(self, + root_directory: str, + directory_search: DirectorySearch = DirectorySearch().set_recursive(True).exclude_by_name( + 'build').filter_by_name('tests')): + """ + :param root_directory: The path to the module's root directory + :param directory_search: The `DirectorySearch` that should be used for directories containing automated tests + """ + self.root_directory = root_directory + self.directory_search = directory_search + + @property + def test_result_directory(self) -> str: + """ + The path of the directory where tests results should be stored. + """ + return path.join(self.root_directory, 'build', 'test-results') + + def find_test_directories(self) -> List[str]: + """ + Finds and returns all directories that contain automated tests that belong to the module. + + :return: A list that contains the paths of the directories that have been found + """ + return self.directory_search.list(self.root_directory) diff --git a/scons/testing/python/requirements.txt b/scons/testing/python/requirements.txt new file mode 100644 index 000000000..b582722d5 --- /dev/null +++ b/scons/testing/python/requirements.txt @@ -0,0 +1 @@ +unittest-xml-reporting >= 3.2, < 3.3 diff --git a/scons/testing/python/targets.py b/scons/testing/python/targets.py new file mode 100644 index 000000000..9778db28d --- /dev/null +++ b/scons/testing/python/targets.py @@ -0,0 +1,20 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Implements targets for testing Python code. +""" +from testing.python.modules import PythonTestModule +from testing.python.unittest import UnitTest +from util.modules import ModuleRegistry +from util.targets import PhonyTarget +from util.units import BuildUnit + + +class TestPython(PhonyTarget.Runnable): + """ + Runs automated tests for Python code. + """ + + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + for module in modules.lookup(PythonTestModule.Filter()): + UnitTest(build_unit, module).run() diff --git a/scons/testing/python/unittest.py b/scons/testing/python/unittest.py new file mode 100644 index 000000000..69a747309 --- /dev/null +++ b/scons/testing/python/unittest.py @@ -0,0 +1,36 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that allow to run automated tests via the external program "unittest". +""" +from testing.python.modules import PythonTestModule +from util.run import PythonModule +from util.units import BuildUnit + + +class UnitTest: + """ + Allows to run the external program "unittest". + """ + + def __init__(self, build_unit: BuildUnit, module: PythonTestModule): + """ + :param build_unit: The build unit from which the program should be run + :param module: The module, the program should be applied to + """ + self.build_unit = build_unit + self.module = module + + def run(self): + """ + Runs the program. + """ + for test_directory in self.module.find_test_directories(): + PythonModule('xmlrunner', 'discover', '--verbose', '--start-directory', test_directory, '--output', + self.module.test_result_directory) \ + .add_conditional_arguments(self.module.fail_fast, '--failfast') \ + .print_arguments(True) \ + .install_program(False) \ + .add_dependencies('unittest-xml-reporting') \ + .set_build_unit(self.build_unit) \ + .run() diff --git a/scons/testing_python.py b/scons/testing_python.py deleted file mode 100644 index e4a35ec0c..000000000 --- a/scons/testing_python.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Provides utility functions for running automated tests. -""" -from os import environ, path - -from modules_old import PYTHON_MODULE -from util.env import get_env_bool -from util.run import PythonModule - - -def tests_python(**_): - """ - Runs all automated tests of Python code. - """ - output_directory = path.join(PYTHON_MODULE.build_dir, 'test-results') - fail_fast = get_env_bool(environ, 'FAIL_FAST') - - for subproject in PYTHON_MODULE.find_subprojects(): - test_dir = subproject.test_dir - - if path.isdir(test_dir): - print('Running automated tests for subpackage "' + subproject.name + '"...') - PythonModule('xmlrunner', 'discover', '--verbose', '--start-directory', test_dir, '--output', - output_directory) \ - .add_conditional_arguments(fail_fast, '--failfast') \ - .print_arguments(True) \ - .install_program(False) \ - .add_dependencies('unittest-xml-reporting') \ - .run() diff --git a/scons/util/requirements.txt b/scons/util/requirements.txt index 2e8319af6..e2710db13 100644 --- a/scons/util/requirements.txt +++ b/scons/util/requirements.txt @@ -1,5 +1,4 @@ build >= 1.2, < 1.3 setuptools scons >= 4.8, < 4.9 -unittest-xml-reporting >= 3.2, < 3.3 wheel >= 0.45, < 0.46 From a72cc16eb23dc890b6679d37b1b003b4decbce04 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 4 Dec 2024 00:30:00 +0100 Subject: [PATCH 069/114] Move module definitions into appropriate subpackages. --- scons/__init__.py | 104 -------------------------- scons/code_style/cpp/__init__.py | 13 +++- scons/code_style/markdown/__init__.py | 21 +++++- scons/code_style/modules.py | 12 +-- scons/code_style/python/__init__.py | 32 +++++++- scons/code_style/yaml/__init__.py | 17 ++++- scons/compilation/cpp/__init__.py | 18 ++++- scons/compilation/cython/__init__.py | 16 +++- scons/compilation/modules.py | 20 ++--- scons/dependencies/python/__init__.py | 21 +++++- scons/dependencies/python/modules.py | 15 ++-- scons/testing/cpp/__init__.py | 7 +- scons/testing/python/__init__.py | 7 +- scons/testing/python/modules.py | 10 +-- 14 files changed, 173 insertions(+), 140 deletions(-) delete mode 100644 scons/__init__.py diff --git a/scons/__init__.py b/scons/__init__.py deleted file mode 100644 index 4909d5609..000000000 --- a/scons/__init__.py +++ /dev/null @@ -1,104 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Defines modules to be dealt with by the build system. -""" -from code_style.modules import CodeModule -from compilation.modules import CompilationModule -from dependencies.python.modules import DependencyType, PythonDependencyModule -from testing.cpp.modules import CppTestModule -from testing.python.modules import PythonTestModule -from util.files import FileSearch -from util.languages import Language - -MODULES = [ - CodeModule( - language=Language.YAML, - root_directory='.', - file_search=FileSearch().set_recursive(False).set_hidden(True), - ), - CodeModule( - language=Language.YAML, - root_directory='.github', - ), - CodeModule( - language=Language.MARKDOWN, - root_directory='.', - file_search=FileSearch().set_recursive(False), - ), - CodeModule( - language=Language.MARKDOWN, - root_directory='doc', - ), - CodeModule( - language=Language.MARKDOWN, - root_directory='python', - ), - CodeModule( - language=Language.PYTHON, - root_directory='scons', - ), - CodeModule( - language=Language.PYTHON, - root_directory='doc', - ), - CodeModule( - language=Language.PYTHON, - root_directory='python', - file_search=FileSearch() \ - .set_recursive(True) \ - .exclude_subdirectories_by_name('build', 'dist', '__pycache__') \ - .exclude_subdirectories_by_substrings(ends_with='.egg.info'), - ), - CodeModule( - language=Language.CYTHON, - root_directory='python', - file_search=FileSearch() \ - .set_recursive(True) \ - .exclude_subdirectories_by_name('build', 'dist', '__pycache__') \ - .exclude_subdirectories_by_substrings(ends_with='.egg-info'), - ), - CodeModule( - language=Language.CPP, - root_directory='cpp', - file_search=FileSearch().set_recursive(True).exclude_subdirectories_by_name('build'), - ), - PythonDependencyModule( - dependency_type=DependencyType.BUILD_TIME, - root_directory='scons', - file_search=FileSearch().set_recursive(True), - ), - PythonDependencyModule( - dependency_type=DependencyType.BUILD_TIME, - root_directory='doc', - file_search=FileSearch().set_recursive(True), - ), - PythonDependencyModule( - dependency_type=DependencyType.RUNTIME, - root_directory='python', - ), - CompilationModule( - language=Language.CPP, - root_directory='cpp', - install_directory='python', - file_search=FileSearch() \ - .filter_by_substrings(starts_with='lib', contains='.so') \ - .filter_by_substrings(ends_with='.dylib') \ - .filter_by_substrings(starts_with='mlrl', ends_with='.lib') \ - .filter_by_substrings(ends_with='.dll'), - ), - CompilationModule( - language=Language.CYTHON, - root_directory='python', - file_search=FileSearch() \ - .filter_by_substrings(not_starts_with='lib', ends_with='.so') \ - .filter_by_substrings(ends_with='.pyd') \ - .filter_by_substrings(not_starts_with='mlrl', ends_with='.lib'), - ), - CppTestModule( - root_directory='cpp', - ), - PythonTestModule( - root_directory='python', - ) -] diff --git a/scons/code_style/cpp/__init__.py b/scons/code_style/cpp/__init__.py index af53d0039..890a95588 100644 --- a/scons/code_style/cpp/__init__.py +++ b/scons/code_style/cpp/__init__.py @@ -1,9 +1,12 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Defines targets for checking and enforcing code style definitions for C++ files. +Defines targets and modules for checking and enforcing code style definitions for C++ files. """ from code_style.cpp.targets import CheckCppCodeStyle, EnforceCppCodeStyle +from code_style.modules import CodeModule +from util.files import FileSearch +from util.languages import Language from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -15,3 +18,11 @@ .add_phony_target(FORMAT_CPP).set_runnables(EnforceCppCodeStyle()) \ .add_phony_target(TEST_FORMAT_CPP).set_runnables(CheckCppCodeStyle()) \ .build() + +MODULES = [ + CodeModule( + language=Language.CPP, + root_directory='cpp', + source_file_search=FileSearch().set_recursive(True).exclude_subdirectories_by_name('build'), + ), +] diff --git a/scons/code_style/markdown/__init__.py b/scons/code_style/markdown/__init__.py index 6de932b35..e2e496998 100644 --- a/scons/code_style/markdown/__init__.py +++ b/scons/code_style/markdown/__init__.py @@ -1,9 +1,12 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Defines targets for checking and enforcing code style definitions for Markdown files. +Defines targets and modules for checking and enforcing code style definitions for Markdown files. """ from code_style.markdown.targets import CheckMarkdownCodeStyle, EnforceMarkdownCodeStyle +from code_style.modules import CodeModule +from util.files import FileSearch +from util.languages import Language from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -15,3 +18,19 @@ .add_phony_target(FORMAT_MARKDOWN).set_runnables(EnforceMarkdownCodeStyle()) \ .add_phony_target(TEST_FORMAT_MARKDOWN).set_runnables(CheckMarkdownCodeStyle()) \ .build() + +MODULES = [ + CodeModule( + language=Language.MARKDOWN, + root_directory='.', + source_file_search=FileSearch().set_recursive(False), + ), + CodeModule( + language=Language.MARKDOWN, + root_directory='python', + ), + CodeModule( + language=Language.MARKDOWN, + root_directory='doc', + ), +] diff --git a/scons/code_style/modules.py b/scons/code_style/modules.py index 1852c7bf1..ffd5fc1d9 100644 --- a/scons/code_style/modules.py +++ b/scons/code_style/modules.py @@ -33,15 +33,15 @@ def matches(self, module: Module) -> bool: def __init__(self, language: Language, root_directory: str, - file_search: FileSearch = FileSearch().set_recursive(True)): + source_file_search: FileSearch = FileSearch().set_recursive(True)): """ - :param language: The programming language of the source code that belongs to the module - :param root_directory: The path to the module's root directory - :param file_search: The `FileSearch` that should be used to search for source files + :param language: The programming language of the source code that belongs to the module + :param root_directory: The path to the module's root directory + :param source_file_search: The `FileSearch` that should be used to search for source files """ self.language = language self.root_directory = root_directory - self.file_search = file_search + self.source_file_search = source_file_search def find_source_files(self) -> List[str]: """ @@ -49,4 +49,4 @@ def find_source_files(self) -> List[str]: :return: A list that contains the paths of the source files that have been found """ - return self.file_search.filter_by_language(self.language).list(self.root_directory) + return self.source_file_search.filter_by_language(self.language).list(self.root_directory) diff --git a/scons/code_style/python/__init__.py b/scons/code_style/python/__init__.py index 67811183e..2f7aa0543 100644 --- a/scons/code_style/python/__init__.py +++ b/scons/code_style/python/__init__.py @@ -1,10 +1,13 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Defines targets for checking and enforcing code style definitions for Python and Cython files. +Defines targets and modules for checking and enforcing code style definitions for Python and Cython files. """ +from code_style.modules import CodeModule from code_style.python.targets import CheckCythonCodeStyle, CheckPythonCodeStyle, EnforceCythonCodeStyle, \ EnforcePythonCodeStyle +from util.files import FileSearch +from util.languages import Language from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -16,3 +19,30 @@ .add_phony_target(FORMAT_PYTHON).set_runnables(EnforcePythonCodeStyle(), EnforceCythonCodeStyle()) \ .add_phony_target(TEST_FORMAT_PYTHON).set_runnables(CheckPythonCodeStyle(), CheckCythonCodeStyle()) \ .build() + +MODULES = [ + CodeModule( + language=Language.PYTHON, + root_directory='scons', + ), + CodeModule( + language=Language.PYTHON, + root_directory='python', + source_file_search=FileSearch() \ + .set_recursive(True) \ + .exclude_subdirectories_by_name('build', 'dist', '__pycache__') \ + .exclude_subdirectories_by_substrings(ends_with='.egg.info'), + ), + CodeModule( + language=Language.CYTHON, + root_directory='python', + source_file_search=FileSearch() \ + .set_recursive(True) \ + .exclude_subdirectories_by_name('build', 'dist', '__pycache__') \ + .exclude_subdirectories_by_substrings(ends_with='.egg.info'), + ), + CodeModule( + language=Language.PYTHON, + root_directory='doc', + ), +] diff --git a/scons/code_style/yaml/__init__.py b/scons/code_style/yaml/__init__.py index 6160e5ed6..96616e565 100644 --- a/scons/code_style/yaml/__init__.py +++ b/scons/code_style/yaml/__init__.py @@ -1,9 +1,12 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Defines targets for checking and enforcing code style definitions for YAML files. +Defines targets and modules for checking and enforcing code style definitions for YAML files. """ +from code_style.modules import CodeModule from code_style.yaml.targets import CheckYamlCodeStyle, EnforceYamlCodeStyle +from util.files import FileSearch +from util.languages import Language from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -15,3 +18,15 @@ .add_phony_target(FORMAT_YAML).set_runnables(EnforceYamlCodeStyle()) \ .add_phony_target(TEST_FORMAT_YAML).set_runnables(CheckYamlCodeStyle()) \ .build() + +MODULES = [ + CodeModule( + language=Language.YAML, + root_directory='.', + source_file_search=FileSearch().set_recursive(False).set_hidden(True), + ), + CodeModule( + language=Language.YAML, + root_directory='.github', + ), +] diff --git a/scons/compilation/cpp/__init__.py b/scons/compilation/cpp/__init__.py index 25f15c028..2059b0013 100644 --- a/scons/compilation/cpp/__init__.py +++ b/scons/compilation/cpp/__init__.py @@ -1,10 +1,13 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Defines targets for compiling C++ code. +Defines targets and modules for compiling C++ code. """ from compilation.cpp.targets import CompileCpp, InstallCpp, SetupCpp +from compilation.modules import CompilationModule from dependencies.python import VENV +from util.files import FileSearch +from util.languages import Language from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -25,3 +28,16 @@ .depends_on(COMPILE_CPP) \ .set_runnables(InstallCpp()) \ .build() + +MODULES = [ + CompilationModule( + language=Language.CPP, + root_directory='cpp', + install_directory='python', + installed_file_search=FileSearch() \ + .filter_by_substrings(starts_with='lib', contains='.so') \ + .filter_by_substrings(ends_with='.dylib') \ + .filter_by_substrings(starts_with='mlrl', ends_with='.lib') \ + .filter_by_substrings(ends_with='.dll'), + ), +] diff --git a/scons/compilation/cython/__init__.py b/scons/compilation/cython/__init__.py index be4d69e31..0af0b669c 100644 --- a/scons/compilation/cython/__init__.py +++ b/scons/compilation/cython/__init__.py @@ -1,10 +1,13 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Defines targets for compiling C++ code. +Defines targets and modules for compiling Cython code. """ from compilation.cpp import COMPILE_CPP from compilation.cython.targets import CompileCython, InstallCython, SetupCython +from compilation.modules import CompilationModule +from util.files import FileSearch +from util.languages import Language from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -25,3 +28,14 @@ .depends_on(COMPILE_CYTHON) \ .set_runnables(InstallCython()) \ .build() + +MODULES = [ + CompilationModule( + language=Language.CYTHON, + root_directory='python', + installed_file_search=FileSearch() \ + .filter_by_substrings(not_starts_with='lib', ends_with='.so') \ + .filter_by_substrings(ends_with='.pyd') \ + .filter_by_substrings(not_starts_with='mlrl', ends_with='.lib'), + ), +] diff --git a/scons/compilation/modules.py b/scons/compilation/modules.py index 0e5adc5af..39138fda3 100644 --- a/scons/compilation/modules.py +++ b/scons/compilation/modules.py @@ -35,19 +35,19 @@ def __init__(self, language: Language, root_directory: str, install_directory: Optional[str] = None, - file_search: Optional[FileSearch] = None): + installed_file_search: Optional[FileSearch] = None): """ - :param language: The programming language of the source code that belongs to the module - :param root_directory: The path to the module's root directory - :param install_directory: The path to the directory into which files are installed or None, if the files are - installed into the root directory - :param file_search: The `FileSearch` that should be used to search for installed files or None, if the - module does never contain any installed files + :param language: The programming language of the source code that belongs to the module + :param root_directory: The path to the module's root directory + :param install_directory: The path to the directory into which files are installed or None, if the files + are installed into the root directory + :param installed_file_search: The `FileSearch` that should be used to search for installed files or None, if + the module does never contain any installed files """ self.language = language self.root_directory = root_directory self.install_directory = install_directory if install_directory else root_directory - self.file_search = file_search + self.installed_file_search = installed_file_search @property def build_directory(self) -> str: @@ -62,8 +62,8 @@ def find_installed_files(self) -> List[str]: :return: A list that contains the paths of the requirements files that have been found """ - if self.file_search: - return self.file_search \ + if self.installed_file_search: + return self.installed_file_search \ .set_recursive(True) \ .exclude_subdirectories_by_name(path.basename(self.build_directory)) \ .list(self.install_directory) diff --git a/scons/dependencies/python/__init__.py b/scons/dependencies/python/__init__.py index f59979d88..2103a53f6 100644 --- a/scons/dependencies/python/__init__.py +++ b/scons/dependencies/python/__init__.py @@ -1,9 +1,11 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Defines targets for updating the Python runtime dependencies that are required by the project's source code. +Defines targets and modules for installing Python dependencies that are required by the project. """ +from dependencies.python.modules import DependencyType, PythonDependencyModule from dependencies.python.targets import CheckPythonDependencies, InstallRuntimeDependencies +from util.files import FileSearch from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -13,3 +15,20 @@ .add_phony_target(VENV).set_runnables(InstallRuntimeDependencies()) \ .add_phony_target('check_dependencies').set_runnables(CheckPythonDependencies()) \ .build() + +MODULES = [ + PythonDependencyModule( + dependency_type=DependencyType.BUILD_TIME, + root_directory='scons', + requirements_file_search=FileSearch().set_recursive(True), + ), + PythonDependencyModule( + dependency_type=DependencyType.RUNTIME, + root_directory='python', + ), + PythonDependencyModule( + dependency_type=DependencyType.BUILD_TIME, + root_directory='doc', + requirements_file_search=FileSearch().set_recursive(True), + ), +] diff --git a/scons/dependencies/python/modules.py b/scons/dependencies/python/modules.py index b56cb4b62..c5bb39676 100644 --- a/scons/dependencies/python/modules.py +++ b/scons/dependencies/python/modules.py @@ -39,15 +39,18 @@ def matches(self, module: Module) -> bool: return isinstance(module, PythonDependencyModule) and (not self.dependency_types or module.dependency_type in self.dependency_types) - def __init__(self, dependency_type: DependencyType, root_directory: str, file_search: FileSearch = FileSearch()): + def __init__(self, + dependency_type: DependencyType, + root_directory: str, + requirements_file_search: FileSearch = FileSearch()): """ - :param dependency_type: The type of the Python dependencies - :param root_directory: The path to the module's root directory - :param file_search: The `FileSearch` that should be used to search for requirements files + :param dependency_type: The type of the Python dependencies + :param root_directory: The path to the module's root directory + :param requirements_file_search: The `FileSearch` that should be used to search for requirements files """ self.dependency_type = dependency_type self.root_directory = root_directory - self.file_search = file_search + self.requirements_file_search = requirements_file_search def find_requirements_files(self) -> List[str]: """ @@ -55,4 +58,4 @@ def find_requirements_files(self) -> List[str]: :return: A list that contains the paths of the requirements files that have been found """ - return self.file_search.filter_by_name('requirements.txt').list(self.root_directory) + return self.requirements_file_search.filter_by_name('requirements.txt').list(self.root_directory) diff --git a/scons/testing/cpp/__init__.py b/scons/testing/cpp/__init__.py index 6e3d23dea..6c08abc0c 100644 --- a/scons/testing/cpp/__init__.py +++ b/scons/testing/cpp/__init__.py @@ -1,9 +1,10 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Defines targets for testing C++ code. +Defines targets and modules for testing C++ code. """ from compilation.cpp import COMPILE_CPP +from testing.cpp.modules import CppTestModule from testing.cpp.targets import TestCpp from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -15,3 +16,7 @@ .depends_on(COMPILE_CPP) \ .set_runnables(TestCpp()) \ .build() + +MODULES = [ + CppTestModule(root_directory='cpp'), +] diff --git a/scons/testing/python/__init__.py b/scons/testing/python/__init__.py index fd379679d..07ca88bac 100644 --- a/scons/testing/python/__init__.py +++ b/scons/testing/python/__init__.py @@ -1,8 +1,9 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Defines targets for testing Python code. +Defines targets and modules for testing Python code. """ +from testing.python.modules import PythonTestModule from testing.python.targets import TestPython from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -14,3 +15,7 @@ .add_phony_target(TESTS_PYTHON) \ .set_runnables(TestPython()) \ .build() + +MODULES = [ + PythonTestModule(root_directory='python'), +] diff --git a/scons/testing/python/modules.py b/scons/testing/python/modules.py index c00878806..3003aff22 100644 --- a/scons/testing/python/modules.py +++ b/scons/testing/python/modules.py @@ -26,14 +26,14 @@ def matches(self, module: Module) -> bool: def __init__(self, root_directory: str, - directory_search: DirectorySearch = DirectorySearch().set_recursive(True).exclude_by_name( + test_directory_search: DirectorySearch = DirectorySearch().set_recursive(True).exclude_by_name( 'build').filter_by_name('tests')): """ - :param root_directory: The path to the module's root directory - :param directory_search: The `DirectorySearch` that should be used for directories containing automated tests + :param root_directory: The path to the module's root directory + :param test_directory_search: The `DirectorySearch` that should be used for directories containing tests """ self.root_directory = root_directory - self.directory_search = directory_search + self.test_directory_search = test_directory_search @property def test_result_directory(self) -> str: @@ -48,4 +48,4 @@ def find_test_directories(self) -> List[str]: :return: A list that contains the paths of the directories that have been found """ - return self.directory_search.list(self.root_directory) + return self.test_directory_search.list(self.root_directory) From 9f7daa42bc5d66008e75d16f3659073cb3d06f41 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 4 Dec 2024 17:38:32 +0100 Subject: [PATCH 070/114] Add class PipList. --- scons/dependencies/python/pip.py | 96 ++++++++++++++++++++++++++++ scons/dependencies/python/targets.py | 6 +- scons/util/pip.py | 79 ++--------------------- 3 files changed, 103 insertions(+), 78 deletions(-) create mode 100644 scons/dependencies/python/pip.py diff --git a/scons/dependencies/python/pip.py b/scons/dependencies/python/pip.py new file mode 100644 index 000000000..b2667dac5 --- /dev/null +++ b/scons/dependencies/python/pip.py @@ -0,0 +1,96 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes for listing installed Python dependencies via pip. +""" +from dataclasses import dataclass +from typing import Set + +from util.pip import Package, Pip, Requirement + + +@dataclass +class Dependency: + """ + Provides information about a dependency. + + Attributes: + installed: The version of the dependency that is currently installed + latest: The latest version of the dependency + """ + installed: Requirement + latest: Requirement + + def __eq__(self, other: 'Dependency') -> bool: + return self.installed == other.installed + + def __hash__(self) -> int: + return hash(self.installed) + + +class PipList(Pip): + """ + Allows to list installed Python packages via pip. + """ + + class ListCommand(Pip.Command): + """ + Allows to list information about installed packages via the command `pip list`. + """ + + def __init__(self, outdated: bool = False): + """ + :param outdated: True, if only outdated packages should be listed, False otherwise + """ + super().__init__('list') + self.add_conditional_arguments(outdated, '--outdated') + + def __init__(self, *requirements_files: str): + """ + :param requirements_files: The paths to the requirements files that specify the versions of the packages to be + installed + """ + super().__init__(*requirements_files) + + def install_all_packages(self): + """ + Installs all dependencies in the requirements file. + """ + for requirement in self.requirements.requirements: + Pip.install_requirement(requirement, dry_run=True) + + def list_outdated_dependencies(self) -> Set[Dependency]: + """ + Returns all outdated Python dependencies that are currently installed. + """ + stdout = PipList.ListCommand(outdated=True).print_command(False).capture_output() + stdout_lines = stdout.strip().split('\n') + i = 0 + + for line in stdout_lines: + i += 1 + + if line.startswith('----'): + break + + outdated_dependencies = set() + + for line in stdout_lines[i:]: + parts = line.split() + + if len(parts) < 3: + raise ValueError( + 'Output of command "pip list" is expected to be a table with at least three columns, but got:' + + line) + + package = Package(parts[0]) + requirement = self.requirements.lookup_requirement(package, accept_missing=True) + + if requirement and requirement.version: + installed_version = parts[1] + latest_version = parts[2] + outdated_dependencies.add( + Dependency(installed=Requirement(package, version=installed_version), + latest=Requirement(package, version=latest_version))) + + return outdated_dependencies diff --git a/scons/dependencies/python/targets.py b/scons/dependencies/python/targets.py index 0586e3642..8b9e8c8c3 100644 --- a/scons/dependencies/python/targets.py +++ b/scons/dependencies/python/targets.py @@ -6,9 +6,9 @@ from functools import reduce from dependencies.python.modules import DependencyType, PythonDependencyModule +from dependencies.python.pip import PipList from dependencies.table import Table from util.modules import ModuleRegistry -from util.pip import Pip from util.targets import PhonyTarget from util.units import BuildUnit @@ -22,7 +22,7 @@ def run(self, _: BuildUnit, modules: ModuleRegistry): dependency_modules = modules.lookup(PythonDependencyModule.Filter(DependencyType.RUNTIME)) requirements_files = reduce(lambda aggr, module: aggr + module.find_requirements_files(), dependency_modules, []) - Pip(*requirements_files).install_all_packages() + PipList(*requirements_files).install_all_packages() class CheckPythonDependencies(PhonyTarget.Runnable): @@ -34,7 +34,7 @@ def run(self, build_unit: BuildUnit, modules: ModuleRegistry): dependency_modules = modules.lookup(PythonDependencyModule.Filter()) requirements_files = reduce(lambda aggr, module: aggr + module.find_requirements_files(), dependency_modules, []) - pip = Pip(*requirements_files) + pip = PipList(*requirements_files) print('Installing all dependencies...') pip.install_all_packages() print('Checking for outdated dependencies...') diff --git a/scons/util/pip.py b/scons/util/pip.py index 67bc7382b..28575f7bd 100644 --- a/scons/util/pip.py +++ b/scons/util/pip.py @@ -1,7 +1,7 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Provides utility functions for installing Python packages via pip. +Provides classes for installing Python packages via pip. """ from abc import ABC, abstractmethod from dataclasses import dataclass @@ -75,25 +75,6 @@ def __hash__(self) -> int: return hash(self.package) -@dataclass -class Dependency: - """ - Provides information about a dependency. - - Attributes: - installed: The version of the dependency that is currently installed - latest: The latest version of the dependency - """ - installed: Requirement - latest: Requirement - - def __eq__(self, other: 'Dependency') -> bool: - return self.installed == other.installed - - def __hash__(self) -> int: - return hash(self.installed) - - class Requirements(ABC): """ An abstract base class for all classes that provide access to requirements. @@ -204,18 +185,6 @@ def __init__(self, requirement: Requirement, dry_run: bool = False): super().__init__('install', str(requirement), '--upgrade', '--upgrade-strategy', 'eager', '--prefer-binary') self.add_conditional_arguments(dry_run, '--dry-run') - class ListCommand(Command): - """ - Allows to list information about installed packages via the command `pip list`. - """ - - def __init__(self, outdated: bool = False): - """ - :param outdated: True, if only outdated packages should be listed, False otherwise - """ - super().__init__('list') - self.add_conditional_arguments(outdated, '--outdated') - @staticmethod def __would_install_requirement(requirement: Requirement, stdout: str) -> bool: prefix = 'Would install' @@ -230,7 +199,7 @@ def __would_install_requirement(requirement: Requirement, stdout: str) -> bool: return False @staticmethod - def __install_requirement(requirement: Requirement, dry_run: bool = False): + def install_requirement(requirement: Requirement, dry_run: bool = False): """ Installs a requirement. @@ -250,7 +219,7 @@ def __install_requirement(requirement: Requirement, dry_run: bool = False): else: print(stdout) except RuntimeError: - Pip.__install_requirement(requirement) + Pip.install_requirement(requirement) def __init__(self, *requirements_files: str): """ @@ -281,44 +250,4 @@ def install_packages(self, *package_names: str, accept_missing: bool = False): requirements = self.requirements.lookup_requirements(*packages, accept_missing=accept_missing) for requirement in requirements: - self.__install_requirement(requirement, dry_run=True) - - def install_all_packages(self): - """ - Installs all dependencies in the requirements file. - """ - for requirement in self.requirements.requirements: - self.__install_requirement(requirement, dry_run=True) - - def list_outdated_dependencies(self) -> Set[Dependency]: - stdout = Pip.ListCommand(outdated=True).print_command(False).capture_output() - stdout_lines = stdout.strip().split('\n') - i = 0 - - for line in stdout_lines: - i += 1 - - if line.startswith('----'): - break - - outdated_dependencies = set() - - for line in stdout_lines[i:]: - parts = line.split() - - if len(parts) < 3: - raise ValueError( - 'Output of command "pip list" is expected to be a table with at least three columns, but got:' - + line) - - package = Package(parts[0]) - requirement = self.requirements.lookup_requirement(package, accept_missing=True) - - if requirement and requirement.version: - installed_version = parts[1] - latest_version = parts[2] - outdated_dependencies.add( - Dependency(installed=Requirement(package, version=installed_version), - latest=Requirement(package, version=latest_version))) - - return outdated_dependencies + self.install_requirement(requirement, dry_run=True) From e15ff2411a6e6f49adbbca61bb08edfcc33ed575 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 5 Dec 2024 00:52:37 +0100 Subject: [PATCH 071/114] Add class Project. --- scons/code_style/cpp/__init__.py | 6 +- scons/code_style/markdown/__init__.py | 9 +- scons/code_style/python/__init__.py | 22 ++--- scons/code_style/yaml/__init__.py | 5 +- scons/compilation/cpp/__init__.py | 9 +- scons/compilation/cython/__init__.py | 7 +- scons/compilation/modules.py | 5 +- scons/dependencies/python/__init__.py | 13 +-- scons/sconstruct.py | 3 +- scons/testing/cpp/__init__.py | 6 +- scons/testing/cpp/modules.py | 8 +- scons/testing/python/__init__.py | 8 +- scons/testing/python/modules.py | 21 +++-- scons/util/paths.py | 124 ++++++++++++++++++++++++++ scons/util/units.py | 6 +- 15 files changed, 202 insertions(+), 50 deletions(-) create mode 100644 scons/util/paths.py diff --git a/scons/code_style/cpp/__init__.py b/scons/code_style/cpp/__init__.py index 890a95588..6e3ee36c9 100644 --- a/scons/code_style/cpp/__init__.py +++ b/scons/code_style/cpp/__init__.py @@ -5,8 +5,8 @@ """ from code_style.cpp.targets import CheckCppCodeStyle, EnforceCppCodeStyle from code_style.modules import CodeModule -from util.files import FileSearch from util.languages import Language +from util.paths import Project from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -22,7 +22,7 @@ MODULES = [ CodeModule( language=Language.CPP, - root_directory='cpp', - source_file_search=FileSearch().set_recursive(True).exclude_subdirectories_by_name('build'), + root_directory=Project.Cpp.root_directory, + source_file_search=Project.Cpp.file_search(), ), ] diff --git a/scons/code_style/markdown/__init__.py b/scons/code_style/markdown/__init__.py index e2e496998..a0a92bded 100644 --- a/scons/code_style/markdown/__init__.py +++ b/scons/code_style/markdown/__init__.py @@ -7,6 +7,7 @@ from code_style.modules import CodeModule from util.files import FileSearch from util.languages import Language +from util.paths import Project from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -22,15 +23,17 @@ MODULES = [ CodeModule( language=Language.MARKDOWN, - root_directory='.', + root_directory=Project.root_directory, source_file_search=FileSearch().set_recursive(False), ), CodeModule( language=Language.MARKDOWN, - root_directory='python', + root_directory=Project.Python.root_directory, + source_file_search=Project.Python.file_search(), ), CodeModule( language=Language.MARKDOWN, - root_directory='doc', + root_directory=Project.Documentation.root_directory, + source_file_search=Project.Documentation.file_search(), ), ] diff --git a/scons/code_style/python/__init__.py b/scons/code_style/python/__init__.py index 2f7aa0543..efbacb6dc 100644 --- a/scons/code_style/python/__init__.py +++ b/scons/code_style/python/__init__.py @@ -6,8 +6,8 @@ from code_style.modules import CodeModule from code_style.python.targets import CheckCythonCodeStyle, CheckPythonCodeStyle, EnforceCythonCodeStyle, \ EnforcePythonCodeStyle -from util.files import FileSearch from util.languages import Language +from util.paths import Project from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -23,26 +23,22 @@ MODULES = [ CodeModule( language=Language.PYTHON, - root_directory='scons', + root_directory=Project.BuildSystem.root_directory, + source_file_search=Project.BuildSystem.file_search(), ), CodeModule( language=Language.PYTHON, - root_directory='python', - source_file_search=FileSearch() \ - .set_recursive(True) \ - .exclude_subdirectories_by_name('build', 'dist', '__pycache__') \ - .exclude_subdirectories_by_substrings(ends_with='.egg.info'), + root_directory=Project.Python.root_directory, + source_file_search=Project.Python.file_search(), ), CodeModule( language=Language.CYTHON, - root_directory='python', - source_file_search=FileSearch() \ - .set_recursive(True) \ - .exclude_subdirectories_by_name('build', 'dist', '__pycache__') \ - .exclude_subdirectories_by_substrings(ends_with='.egg.info'), + root_directory=Project.Python.root_directory, + source_file_search=Project.Python.file_search(), ), CodeModule( language=Language.PYTHON, - root_directory='doc', + root_directory=Project.Documentation.root_directory, + source_file_search=Project.Documentation.file_search(), ), ] diff --git a/scons/code_style/yaml/__init__.py b/scons/code_style/yaml/__init__.py index 96616e565..afc9f5483 100644 --- a/scons/code_style/yaml/__init__.py +++ b/scons/code_style/yaml/__init__.py @@ -7,6 +7,7 @@ from code_style.yaml.targets import CheckYamlCodeStyle, EnforceYamlCodeStyle from util.files import FileSearch from util.languages import Language +from util.paths import Project from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -22,11 +23,11 @@ MODULES = [ CodeModule( language=Language.YAML, - root_directory='.', + root_directory=Project.root_directory, source_file_search=FileSearch().set_recursive(False).set_hidden(True), ), CodeModule( language=Language.YAML, - root_directory='.github', + root_directory=Project.Github.root_directory, ), ] diff --git a/scons/compilation/cpp/__init__.py b/scons/compilation/cpp/__init__.py index 2059b0013..e4616dde0 100644 --- a/scons/compilation/cpp/__init__.py +++ b/scons/compilation/cpp/__init__.py @@ -6,8 +6,8 @@ from compilation.cpp.targets import CompileCpp, InstallCpp, SetupCpp from compilation.modules import CompilationModule from dependencies.python import VENV -from util.files import FileSearch from util.languages import Language +from util.paths import Project from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -32,9 +32,10 @@ MODULES = [ CompilationModule( language=Language.CPP, - root_directory='cpp', - install_directory='python', - installed_file_search=FileSearch() \ + root_directory=Project.Cpp.root_directory, + build_directory_name=Project.Cpp.build_directory_name, + install_directory=Project.Python.root_directory, + installed_file_search=Project.Cpp.file_search() \ .filter_by_substrings(starts_with='lib', contains='.so') \ .filter_by_substrings(ends_with='.dylib') \ .filter_by_substrings(starts_with='mlrl', ends_with='.lib') \ diff --git a/scons/compilation/cython/__init__.py b/scons/compilation/cython/__init__.py index 0af0b669c..7b432a7e0 100644 --- a/scons/compilation/cython/__init__.py +++ b/scons/compilation/cython/__init__.py @@ -6,8 +6,8 @@ from compilation.cpp import COMPILE_CPP from compilation.cython.targets import CompileCython, InstallCython, SetupCython from compilation.modules import CompilationModule -from util.files import FileSearch from util.languages import Language +from util.paths import Project from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -32,8 +32,9 @@ MODULES = [ CompilationModule( language=Language.CYTHON, - root_directory='python', - installed_file_search=FileSearch() \ + root_directory=Project.Python.root_directory, + build_directory_name=Project.Python.build_directory_name, + installed_file_search=Project.Python.file_search() \ .filter_by_substrings(not_starts_with='lib', ends_with='.so') \ .filter_by_substrings(ends_with='.pyd') \ .filter_by_substrings(not_starts_with='mlrl', ends_with='.lib'), diff --git a/scons/compilation/modules.py b/scons/compilation/modules.py index 39138fda3..e8716a9ae 100644 --- a/scons/compilation/modules.py +++ b/scons/compilation/modules.py @@ -34,11 +34,13 @@ def matches(self, module: Module) -> bool: def __init__(self, language: Language, root_directory: str, + build_directory_name: str, install_directory: Optional[str] = None, installed_file_search: Optional[FileSearch] = None): """ :param language: The programming language of the source code that belongs to the module :param root_directory: The path to the module's root directory + :param build_directory_name: The name of the module's build directory :param install_directory: The path to the directory into which files are installed or None, if the files are installed into the root directory :param installed_file_search: The `FileSearch` that should be used to search for installed files or None, if @@ -46,6 +48,7 @@ def __init__(self, """ self.language = language self.root_directory = root_directory + self.build_directory_name = build_directory_name self.install_directory = install_directory if install_directory else root_directory self.installed_file_search = installed_file_search @@ -54,7 +57,7 @@ def build_directory(self) -> str: """ The path to the directory, where build files should be stored. """ - return path.join(self.root_directory, 'build') + return path.join(self.root_directory, self.build_directory_name) def find_installed_files(self) -> List[str]: """ diff --git a/scons/dependencies/python/__init__.py b/scons/dependencies/python/__init__.py index 2103a53f6..129a27d82 100644 --- a/scons/dependencies/python/__init__.py +++ b/scons/dependencies/python/__init__.py @@ -5,7 +5,7 @@ """ from dependencies.python.modules import DependencyType, PythonDependencyModule from dependencies.python.targets import CheckPythonDependencies, InstallRuntimeDependencies -from util.files import FileSearch +from util.paths import Project from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -19,16 +19,17 @@ MODULES = [ PythonDependencyModule( dependency_type=DependencyType.BUILD_TIME, - root_directory='scons', - requirements_file_search=FileSearch().set_recursive(True), + root_directory=Project.BuildSystem.root_directory, + requirements_file_search=Project.BuildSystem.file_search(), ), PythonDependencyModule( dependency_type=DependencyType.RUNTIME, - root_directory='python', + root_directory=Project.Python.root_directory, + requirements_file_search=Project.Python.file_search(), ), PythonDependencyModule( dependency_type=DependencyType.BUILD_TIME, - root_directory='doc', - requirements_file_search=FileSearch().set_recursive(True), + root_directory=Project.Documentation.root_directory, + requirements_file_search=Project.Documentation.file_search(), ), ] diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 2b66d999b..29f3e3fb6 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -13,6 +13,7 @@ from util.files import FileSearch from util.format import format_iterable from util.modules import Module, ModuleRegistry +from util.paths import Project from util.reflection import import_source_file from util.targets import Target, TargetRegistry @@ -44,7 +45,7 @@ def __print_if_clean(environment, message: str): DEFAULT_TARGET = TARGET_NAME_INSTALL_WHEELS # Register modules... -init_files = FileSearch().set_recursive(True).filter_by_name('__init__.py').list(BUILD_MODULE.root_dir) +init_files = FileSearch().set_recursive(True).filter_by_name('__init__.py').list(Project.BuildSystem.root_directory) module_registry = ModuleRegistry() for init_file in init_files: diff --git a/scons/testing/cpp/__init__.py b/scons/testing/cpp/__init__.py index 6c08abc0c..e01815a1c 100644 --- a/scons/testing/cpp/__init__.py +++ b/scons/testing/cpp/__init__.py @@ -6,6 +6,7 @@ from compilation.cpp import COMPILE_CPP from testing.cpp.modules import CppTestModule from testing.cpp.targets import TestCpp +from util.paths import Project from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -18,5 +19,8 @@ .build() MODULES = [ - CppTestModule(root_directory='cpp'), + CppTestModule( + root_directory=Project.Cpp.root_directory, + build_directory_name=Project.Cpp.build_directory_name, + ), ] diff --git a/scons/testing/cpp/modules.py b/scons/testing/cpp/modules.py index 6eec3f7c8..fa7e0f9b8 100644 --- a/scons/testing/cpp/modules.py +++ b/scons/testing/cpp/modules.py @@ -22,15 +22,17 @@ class Filter(Module.Filter): def matches(self, module: Module) -> bool: return isinstance(module, CppTestModule) - def __init__(self, root_directory: str): + def __init__(self, root_directory: str, build_directory_name: str): """ - :param root_directory: The path to the module's root directory + :param root_directory: The path to the module's root directory + :param build_directory_name: The name of the module's build directory """ self.root_directory = root_directory + self.build_directory_name = build_directory_name @property def build_directory(self) -> str: """ The path to the directory, where build files are stored. """ - return path.join(self.root_directory, 'build') + return path.join(self.root_directory, self.build_directory_name) diff --git a/scons/testing/python/__init__.py b/scons/testing/python/__init__.py index 07ca88bac..41adfb1d0 100644 --- a/scons/testing/python/__init__.py +++ b/scons/testing/python/__init__.py @@ -5,6 +5,7 @@ """ from testing.python.modules import PythonTestModule from testing.python.targets import TestPython +from util.paths import Project from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -17,5 +18,10 @@ .build() MODULES = [ - PythonTestModule(root_directory='python'), + PythonTestModule( + root_directory=Project.Python.root_directory, + build_directory_name=Project.Python.build_directory_name, + test_file_search=Project.Python.file_search() \ + .filter_subdirectories_by_name(Project.Python.test_directory_name), + ), ] diff --git a/scons/testing/python/modules.py b/scons/testing/python/modules.py index 3003aff22..e40109e56 100644 --- a/scons/testing/python/modules.py +++ b/scons/testing/python/modules.py @@ -7,7 +7,8 @@ from typing import List from testing.modules import TestModule -from util.files import DirectorySearch +from util.files import FileSearch +from util.languages import Language from util.modules import Module @@ -26,21 +27,23 @@ def matches(self, module: Module) -> bool: def __init__(self, root_directory: str, - test_directory_search: DirectorySearch = DirectorySearch().set_recursive(True).exclude_by_name( - 'build').filter_by_name('tests')): + build_directory_name: str, + test_file_search: FileSearch = FileSearch().set_recursive(True)): """ :param root_directory: The path to the module's root directory - :param test_directory_search: The `DirectorySearch` that should be used for directories containing tests + :param build_directory_name: The name of the module's build directory + :param test_file_search: The `FilesSearch` that should be used to search for test files """ self.root_directory = root_directory - self.test_directory_search = test_directory_search + self.build_directory_name = build_directory_name + self.test_file_search = test_file_search @property def test_result_directory(self) -> str: """ The path of the directory where tests results should be stored. """ - return path.join(self.root_directory, 'build', 'test-results') + return path.join(self.root_directory, self.build_directory_name, 'test-results') def find_test_directories(self) -> List[str]: """ @@ -48,4 +51,8 @@ def find_test_directories(self) -> List[str]: :return: A list that contains the paths of the directories that have been found """ - return self.test_directory_search.list(self.root_directory) + return self.test_file_search \ + .exclude_subdirectories_by_name(self.build_directory_name) \ + .filter_by_substrings(starts_with='test_') \ + .filter_by_language(Language.PYTHON) \ + .list(self.root_directory) diff --git a/scons/util/paths.py b/scons/util/paths.py new file mode 100644 index 000000000..973e95fe8 --- /dev/null +++ b/scons/util/paths.py @@ -0,0 +1,124 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides paths within the project that are important for the build system. +""" +from util.files import FileSearch + + +class Project: + """ + Provides paths within the project. + + Attributes: + root_directory: The path to the project's root directory + """ + + root_directory = '.' + + class BuildSystem: + """ + Provides paths within the project's build system. + + Attributes: + root_directory: The path to the build system's root directory + build_directory_name: The name of the build system's build directory + """ + + root_directory = 'scons' + + build_directory_name = 'build' + + @staticmethod + def file_search() -> FileSearch: + """ + Creates and returns a `FileSearch` that allows searching for files within the build system. + + :return: The `FileSearch` that has been created + """ + return FileSearch() \ + .set_recursive(True) \ + .exclude_subdirectories_by_name(Project.BuildSystem.build_directory_name) + + class Python: + """ + Provides paths within the project's Python code. + + Attributes: + root_directory: The path to the Python code's root directory + build_directory_name: The name of the Python code's build directory + test_directory_name: The name fo the directory that contains tests + """ + + root_directory = 'python' + + build_directory_name = 'build' + + test_directory_name = 'tests' + + @staticmethod + def file_search() -> FileSearch: + """ + Creates and returns a `FileSearch` that allows searching for files within the Python code. + + :return: The `FileSearch` that has been created + """ + return FileSearch() \ + .set_recursive(True) \ + .exclude_subdirectories_by_name(Project.Python.build_directory_name, 'dist', '__pycache__') \ + .exclude_subdirectories_by_substrings(ends_with='.egg.info') + + class Cpp: + """ + Provides paths within the project's C++ code. + + Attributes: + root_directory: The path to the C++ code's root directory + build_directory_name: The name of the C++ code's build directory + """ + + root_directory = 'cpp' + + build_directory_name = 'build' + + @staticmethod + def file_search() -> FileSearch: + """ + Creates and returns a `FileSearch` that allows searchin for files within the C++ code. + + :return: The `FileSearch` that has been created + """ + return FileSearch() \ + .set_recursive(True) \ + .exclude_subdirectories_by_name(Project.Cpp.build_directory_name) + + class Documentation: + """ + Provides paths within the project's documentation. + + Attributes: + root_directory: The path to the documentation's root directory + """ + + root_directory = 'doc' + + @staticmethod + def file_search() -> FileSearch: + """ + Creates and returns a `FileSearch` that allows searching for files within the documentation. + + :return: The `FileSearch` that has been created + """ + return FileSearch() \ + .set_recursive(True) \ + .exclude_subdirectories_by_name('_build') + + class Github: + """ + Provides paths within the project's GitHub-related files. + + Attributes: + root_directory: The path to the root directory that contains all GitHub-related files + """ + + root_directory = '.github' diff --git a/scons/util/units.py b/scons/util/units.py index 610606b45..5ba9ce785 100644 --- a/scons/util/units.py +++ b/scons/util/units.py @@ -6,6 +6,8 @@ from os import path from typing import List +from util.paths import Project + class BuildUnit: """ @@ -16,7 +18,7 @@ def __init__(self, *subdirectories: str): """ :param subdirectories: The subdirectories within the build system that lead to the root directory of this unit """ - self.root_directory = path.join('scons', *subdirectories) + self.root_directory = path.join(Project.BuildSystem.root_directory, *subdirectories) def find_requirements_files(self) -> List[str]: """ @@ -25,7 +27,7 @@ def find_requirements_files(self) -> List[str]: requirements_files = [] current_directory = self.root_directory - while path.basename(current_directory) != 'scons': + while path.basename(current_directory) != Project.BuildSystem.root_directory: requirements_file = path.join(current_directory, 'requirements.txt') if path.isfile(requirements_file): From 1c2a18e10d9a4813851df397c4f1813d073e0a24 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 5 Dec 2024 01:35:01 +0100 Subject: [PATCH 072/114] Allow specifying the input files required by a BuildTarget. --- scons/util/targets.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/scons/util/targets.py b/scons/util/targets.py index b874ccf42..e775fb92a 100644 --- a/scons/util/targets.py +++ b/scons/util/targets.py @@ -165,6 +165,15 @@ def run(self, build_unit: BuildUnit, modules: ModuleRegistry): :param modules: A `ModuleRegistry` that can be used by the target for looking up modules """ + def get_input_files(self, modules: ModuleRegistry) -> List[str]: + """ + May be overridden by subclasses in order to return the input files required by the target. + + :param modules: A `ModuleRegistry` that can be used by the target for looking up modules + :return: A list that contains the input files + """ + return [] + def get_output_files(self, modules: ModuleRegistry) -> List[str]: """ May be overridden by subclasses in order to return the output files produced by the target. @@ -229,11 +238,13 @@ def action(): for runnable in self.runnables: runnable.run(self.build_unit, module_registry) + input_files = reduce(lambda aggr, runnable: runnable.get_input_files(module_registry), self.runnables, []) + source = (input_files if len(input_files) > 1 else input_files[0]) if input_files else None output_files = reduce(lambda aggr, runnable: runnable.get_output_files(module_registry), self.runnables, []) target = (output_files if len(output_files) > 1 else output_files[0]) if output_files else None if target: - return environment.Command(target, None, action=lambda **_: action()) + return environment.Command(target, source, action=lambda **_: action()) return environment.AlwaysBuild(environment.Alias(self.name, None, action=lambda **_: action())) From 02ceef477e7fb98fcdd2d60ad043420ab06fea61 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 5 Dec 2024 15:35:45 +0100 Subject: [PATCH 073/114] Add function "set_symlinks" to class FileSearch. --- scons/util/files.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/scons/util/files.py b/scons/util/files.py index f802fee73..923183ffd 100644 --- a/scons/util/files.py +++ b/scons/util/files.py @@ -164,6 +164,7 @@ class FileSearch: def __init__(self): self.hidden = False + self.symlinks = True self.filters = [] self.directory_search = DirectorySearch() @@ -251,10 +252,21 @@ def set_hidden(self, hidden: bool) -> 'FileSearch': Sets whether hidden files should be included or not. :param hidden: True, if hidden files should be included, False otherwise + :return: The `FileSearch` itself """ self.hidden = hidden return self + def set_symlinks(self, symlinks: bool) -> 'FileSearch': + """ + Sets whether symbolic links should be included or not. + + :param symlinks: True, if symbolic links should be included, False otherwise + :return: The `FileSearch` itself + """ + self.symlinks = symlinks + return self + def add_filters(self, *filter_functions: Filter) -> 'FileSearch': """ Adds one or several filters that match files to be included. @@ -342,7 +354,7 @@ def list(self, *directories: str) -> List[str]: subdirectories = self.directory_search.list(*directories) if self.directory_search.recursive else [] def filter_file(file: str) -> bool: - if path.isfile(file): + if path.isfile(file) and (self.symlinks or not path.islink(file)): if not self.filters: return True From 9da4a2a56e44df4a588ce1b69eb420bc65b1355a Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 5 Dec 2024 15:36:21 +0100 Subject: [PATCH 074/114] Dynamically register targets and modules for building and installing wheel packages. --- scons/compilation/__init__.py | 4 +- scons/packaging.py | 58 ----------------------- scons/packaging/__init__.py | 41 ++++++++++++++++ scons/packaging/build.py | 24 ++++++++++ scons/packaging/modules.py | 54 +++++++++++++++++++++ scons/packaging/pip.py | 36 ++++++++++++++ scons/packaging/requirements.txt | 3 ++ scons/packaging/targets.py | 80 ++++++++++++++++++++++++++++++++ scons/sconstruct.py | 40 ++-------------- scons/testing/python/__init__.py | 3 +- scons/util/paths.py | 16 +++++-- scons/util/requirements.txt | 3 -- scons/util/targets.py | 52 +++++++++++++++++++-- 13 files changed, 307 insertions(+), 107 deletions(-) delete mode 100644 scons/packaging.py create mode 100644 scons/packaging/__init__.py create mode 100644 scons/packaging/build.py create mode 100644 scons/packaging/modules.py create mode 100644 scons/packaging/pip.py create mode 100644 scons/packaging/requirements.txt create mode 100644 scons/packaging/targets.py diff --git a/scons/compilation/__init__.py b/scons/compilation/__init__.py index cd1f3a401..97010fd49 100644 --- a/scons/compilation/__init__.py +++ b/scons/compilation/__init__.py @@ -8,11 +8,13 @@ from util.targets import TargetBuilder from util.units import BuildUnit +INSTALL = 'install' + TARGETS = TargetBuilder(BuildUnit('compilation')) \ .add_phony_target('compile') \ .depends_on(COMPILE_CPP, COMPILE_CYTHON, clean_dependencies=True) \ .nop() \ - .add_phony_target('install') \ + .add_phony_target(INSTALL) \ .depends_on(INSTALL_CPP, INSTALL_CYTHON, clean_dependencies=True) \ .nop() \ .build() diff --git a/scons/packaging.py b/scons/packaging.py deleted file mode 100644 index 87c3ed6d0..000000000 --- a/scons/packaging.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Provides utility functions for building and installing Python wheel packages. -""" -from typing import List - -from modules_old import PYTHON_MODULE -from util.run import PythonModule - - -def __build_python_wheel(package_dir: str): - PythonModule('build', '--no-isolation', '--wheel', package_dir) \ - .print_arguments(True) \ - .add_dependencies('wheel', 'setuptools') \ - .run() - - -def __install_python_wheels(wheels: List[str]): - PythonModule('pip', 'install', '--force-reinstall', '--no-deps', '--disable-pip-version-check', *wheels) \ - .print_arguments(True) \ - .install_program(False) \ - .run() - - -# pylint: disable=unused-argument -def build_python_wheel(env, target, source): - """ - Builds a Python wheel package for a single subproject. - - :param env: The scons environment - :param target: The path of the wheel package to be built, if it does already exist, or the path of the directory, - where the wheel package should be stored - :param source: The source files from which the wheel package should be built - """ - if target: - subproject = PYTHON_MODULE.find_subproject(target[0].path) - - if subproject: - print('Building Python wheels for subproject "' + subproject.name + '"...') - __build_python_wheel(subproject.root_dir) - - -# pylint: disable=unused-argument -def install_python_wheels(env, target, source): - """ - Installs all Python wheel packages that have been built for a single subproject. - - :param env: The scons environment - :param target: The path of the subproject's root directory - :param source: The paths of the wheel packages to be installed - """ - if source: - subproject = PYTHON_MODULE.find_subproject(source[0].path) - - if subproject: - print('Installing Python wheels for subproject "' + subproject.name + '"...') - __install_python_wheels(subproject.find_wheels()) diff --git a/scons/packaging/__init__.py b/scons/packaging/__init__.py new file mode 100644 index 000000000..a650546ab --- /dev/null +++ b/scons/packaging/__init__.py @@ -0,0 +1,41 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Defines targets and modules for building and install Python wheel packages. +""" +from os import path + +from compilation import INSTALL +from packaging.modules import PythonPackageModule +from packaging.targets import BuildPythonWheels, InstallPythonWheels +from util.paths import Project +from util.targets import TargetBuilder +from util.units import BuildUnit + +BUILD_WHEELS = 'build_wheels' + +INSTALL_WHEELS = 'install_wheels' + +MODULES = [ + PythonPackageModule( + root_directory=path.dirname(setup_file), + wheel_directory_name=Project.Python.wheel_directory_name, + ) for setup_file in Project.Python.file_search().filter_by_name('setup.py').list(Project.Python.root_directory) +] + +TARGETS = TargetBuilder(BuildUnit('packaging')) \ + .add_phony_target(BUILD_WHEELS) \ + .depends_on(INSTALL) \ + .depends_on_build_targets( + MODULES, + lambda module, target_builder: target_builder.set_runnables(BuildPythonWheels(module.root_directory)) + ) \ + .nop() \ + .add_phony_target(INSTALL_WHEELS) \ + .depends_on(BUILD_WHEELS) \ + .depends_on_phony_targets( + MODULES, + lambda module, target_builder: target_builder.set_runnables(InstallPythonWheels(module.root_directory)) + ) \ + .nop() \ + .build() diff --git a/scons/packaging/build.py b/scons/packaging/build.py new file mode 100644 index 000000000..ba0479142 --- /dev/null +++ b/scons/packaging/build.py @@ -0,0 +1,24 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that allow to build wheel packages via the external program "build". +""" +from packaging.modules import PythonPackageModule +from util.run import PythonModule +from util.units import BuildUnit + + +class Build(PythonModule): + """ + Allows to run the external program "build". + """ + + def __init__(self, build_unit: BuildUnit, module: PythonPackageModule): + """ + :param build_unit: The build unit from which the program should be run + :param module: The module, the program should be applied to + """ + super().__init__('build', '--no-isolation', '--wheel', module.root_directory) + self.print_arguments(True) + self.add_dependencies('wheel', 'setuptools') + self.set_build_unit(build_unit) diff --git a/scons/packaging/modules.py b/scons/packaging/modules.py new file mode 100644 index 000000000..eb3c0e6a2 --- /dev/null +++ b/scons/packaging/modules.py @@ -0,0 +1,54 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that provide access to Python code that can be built as wheel packages. +""" +from os import path +from typing import List + +from util.files import FileSearch +from util.modules import Module + + +class PythonPackageModule(Module): + """ + A module that contains Python code that can be built as wheel packages. + """ + + class Filter(Module.Filter): + """ + A filter that matches modules that contain Python code that can be built as wheel packages. + """ + + def __init__(self, *root_directories: str): + """ + :param root_directories: The root directories of the modules to be matched + """ + self.root_directories = set(root_directories) + + def matches(self, module: Module) -> bool: + return isinstance(module, PythonPackageModule) and (not self.root_directories + or module.root_directory in self.root_directories) + + def __init__(self, root_directory: str, wheel_directory_name: str): + """ + :param root_directory: The path to the module's root directory + :param wheel_directory_name: The name of the directory that contains wheel packages + """ + self.root_directory = root_directory + self.wheel_directory_name = wheel_directory_name + + @property + def wheel_directory(self) -> str: + """ + Returns the path of the directory that contains the wheel packages that have been built for the module. + """ + return path.join(self.root_directory, self.wheel_directory_name) + + def find_wheels(self) -> List[str]: + """ + Finds and returns all wheel packages that have been built for the module. + + :return: A list that contains the paths to the wheel packages + """ + return FileSearch().filter_by_suffix('whl').list(self.wheel_directory) diff --git a/scons/packaging/pip.py b/scons/packaging/pip.py new file mode 100644 index 000000000..7809ce7b1 --- /dev/null +++ b/scons/packaging/pip.py @@ -0,0 +1,36 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes for installing wheel packages via pip. +""" +from util.pip import Pip + + +class PipInstallWheel(Pip): + """ + Allows to install wheel packages via pip. + """ + + class InstallWheelCommand(Pip.Command): + """ + Allows to install wheel packages via the command `pip install`. + """ + + def __init__(self, *wheels: str): + """ + :param wheels: The paths to the wheel packages to be installed + """ + super().__init__('install', '--force-reinstall', '--no-deps', *wheels) + + def __init__(self, *requirements_files: str): + """ + :param requirements_files: The paths to the requirements files that specify the versions of the packages to be + installed + """ + super().__init__(*requirements_files) + + def install_wheels(self, *wheels: str): + """ + Installs several wheel packages. + """ + PipInstallWheel.InstallWheelCommand(*wheels).print_arguments(True).run() diff --git a/scons/packaging/requirements.txt b/scons/packaging/requirements.txt new file mode 100644 index 000000000..a2cb5b645 --- /dev/null +++ b/scons/packaging/requirements.txt @@ -0,0 +1,3 @@ +build >= 1.2, < 1.3 +setuptools +wheel >= 0.45, < 0.46 diff --git a/scons/packaging/targets.py b/scons/packaging/targets.py new file mode 100644 index 000000000..7472c6093 --- /dev/null +++ b/scons/packaging/targets.py @@ -0,0 +1,80 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Implements targets for building and installing wheel packages. +""" +from functools import reduce +from typing import List + +from packaging.build import Build +from packaging.modules import PythonPackageModule +from packaging.pip import PipInstallWheel +from util.files import DirectorySearch +from util.languages import Language +from util.modules import ModuleRegistry +from util.paths import Project +from util.targets import BuildTarget, PhonyTarget +from util.units import BuildUnit + + +class BuildPythonWheels(BuildTarget.Runnable): + """ + Builds Python wheel packages. + """ + + def __init__(self, root_directory: str): + """ + :param root_directory: The root directory of the module for which Python wheel packages should be built + """ + self.module_filter = PythonPackageModule.Filter(root_directory) + + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + for module in modules.lookup(self.module_filter): + print('Building Python wheels for directory "' + module.root_directory + '"...') + Build(build_unit, module).run() + + def get_input_files(self, modules: ModuleRegistry) -> List[str]: + package_modules = modules.lookup(self.module_filter) + file_search = Project.Python.file_search() \ + .set_symlinks(False) \ + .exclude_subdirectories_by_name(Project.Python.test_directory_name) \ + .filter_by_language(Language.PYTHON) \ + .filter_by_suffix('so', 'pyd', 'lib', 'dylib', 'dll') + return reduce(lambda aggr, module: aggr + file_search.list(module.root_directory), package_modules, []) + + def get_output_files(self, modules: ModuleRegistry) -> List[str]: + package_modules = modules.lookup(self.module_filter) + wheels = reduce(lambda aggr, module: module.find_wheels(), package_modules, []) + return wheels if wheels else [module.wheel_directory for module in package_modules] + + def get_clean_files(self, modules: ModuleRegistry) -> List[str]: + clean_files = [] + + for module in modules.lookup(self.module_filter): + print('Removing Python wheels from directory "' + module.root_directory + '"...') + clean_files.append(module.wheel_directory) + clean_files.extend( + DirectorySearch() \ + .filter_by_name(Project.Python.build_directory_name) \ + .filter_by_substrings(ends_with=Project.Python.wheel_metadata_directory_suffix) \ + .list(module.root_directory) + ) + + return clean_files + + +class InstallPythonWheels(PhonyTarget.Runnable): + """ + Installs Python wheel packages. + """ + + def __init__(self, root_directory: str): + """ + :param root_directory: The root directory of the module for which Python wheel packages should be installed + """ + self.module_filter = PythonPackageModule.Filter(root_directory) + + def run(self, _: BuildUnit, modules: ModuleRegistry): + for module in modules.lookup(self.module_filter): + print('Installing Python wheels for directory "' + module.root_directory + '"...') + PipInstallWheel().install_wheels(module.find_wheels()) diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 29f3e3fb6..4d86387de 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -9,7 +9,6 @@ from documentation import apidoc_cpp, apidoc_cpp_tocfile, apidoc_python, apidoc_python_tocfile, doc from modules_old import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE -from packaging import build_python_wheel, install_python_wheels from util.files import FileSearch from util.format import format_iterable from util.modules import Module, ModuleRegistry @@ -30,19 +29,14 @@ def __print_if_clean(environment, message: str): # Define target names... -TARGET_NAME_BUILD_WHEELS = 'build_wheels' -TARGET_NAME_INSTALL_WHEELS = 'install_wheels' TARGET_NAME_APIDOC = 'apidoc' TARGET_NAME_APIDOC_CPP = TARGET_NAME_APIDOC + '_cpp' TARGET_NAME_APIDOC_PYTHON = TARGET_NAME_APIDOC + '_python' TARGET_NAME_DOC = 'doc' -VALID_TARGETS = { - TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS, TARGET_NAME_APIDOC, TARGET_NAME_APIDOC_CPP, - TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC -} +VALID_TARGETS = {TARGET_NAME_APIDOC, TARGET_NAME_APIDOC_CPP, TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC} -DEFAULT_TARGET = TARGET_NAME_INSTALL_WHEELS +DEFAULT_TARGET = 'install_wheels' # Register modules... init_files = FileSearch().set_recursive(True).filter_by_name('__init__.py').list(Project.BuildSystem.root_directory) @@ -75,34 +69,6 @@ def __print_if_clean(environment, message: str): # Create temporary file ".sconsign.dblite" in the build directory... env.SConsignFile(name=path.relpath(path.join(BUILD_MODULE.build_dir, '.sconsign'), BUILD_MODULE.root_dir)) -# Define targets for building and installing Python wheels... -commands_build_wheels = [] -commands_install_wheels = [] - -for subproject in PYTHON_MODULE.find_subprojects(): - wheels = subproject.find_wheels() - targets_build_wheels = wheels if wheels else subproject.dist_dir - - command_build_wheels = env.Command(targets_build_wheels, subproject.find_source_files(), action=build_python_wheel) - commands_build_wheels.append(command_build_wheels) - - command_install_wheels = env.Command(subproject.root_dir, targets_build_wheels, action=install_python_wheels) - env.Depends(command_install_wheels, command_build_wheels) - commands_install_wheels.append(command_install_wheels) - -target_build_wheels = env.Alias(TARGET_NAME_BUILD_WHEELS, None, None) -env.Depends(target_build_wheels, ['target_install'] + commands_build_wheels) - -target_install_wheels = env.Alias(TARGET_NAME_INSTALL_WHEELS, None, None) -env.Depends(target_install_wheels, ['target_install'] + commands_install_wheels) - -# Define target for cleaning up Python wheels and associated build directories... -if not COMMAND_LINE_TARGETS or TARGET_NAME_BUILD_WHEELS in COMMAND_LINE_TARGETS: - __print_if_clean(env, 'Removing Python wheels...') - - for subproject in PYTHON_MODULE.find_subprojects(return_all=True): - env.Clean([target_build_wheels, DEFAULT_TARGET], subproject.build_dirs) - # Define targets for generating the documentation... commands_apidoc_cpp = [] commands_apidoc_python = [] @@ -128,7 +94,7 @@ def __print_if_clean(environment, message: str): targets_apidoc_python = build_files if build_files else apidoc_subproject.build_dir command_apidoc_python = env.Command(targets_apidoc_python, subproject.find_source_files(), action=apidoc_python) env.NoClean(command_apidoc_python) - env.Depends(command_apidoc_python, target_install_wheels) + env.Depends(command_apidoc_python, 'install_wheels') commands_apidoc_python.append(command_apidoc_python) command_apidoc_python_tocfile = env.Command(DOC_MODULE.apidoc_tocfile_python, None, action=apidoc_python_tocfile) diff --git a/scons/testing/python/__init__.py b/scons/testing/python/__init__.py index 41adfb1d0..2affbfbc2 100644 --- a/scons/testing/python/__init__.py +++ b/scons/testing/python/__init__.py @@ -3,6 +3,7 @@ Defines targets and modules for testing Python code. """ +from packaging import INSTALL_WHEELS from testing.python.modules import PythonTestModule from testing.python.targets import TestPython from util.paths import Project @@ -11,9 +12,9 @@ TESTS_PYTHON = 'tests_python' -# TODO .depends_on(INSTALL_WHEELS) TARGETS = TargetBuilder(BuildUnit('testing', 'python')) \ .add_phony_target(TESTS_PYTHON) \ + .depends_on(INSTALL_WHEELS) \ .set_runnables(TestPython()) \ .build() diff --git a/scons/util/paths.py b/scons/util/paths.py index 973e95fe8..718476345 100644 --- a/scons/util/paths.py +++ b/scons/util/paths.py @@ -45,9 +45,11 @@ class Python: Provides paths within the project's Python code. Attributes: - root_directory: The path to the Python code's root directory - build_directory_name: The name of the Python code's build directory + root_directory: The path to the Python code's root directory + build_directory_name: The name of the Python code's build directory test_directory_name: The name fo the directory that contains tests + wheel_directory_name: The name of the directory that contains wheel packages + wheel_metadata_directory_suffix: The suffix of the directory that contains the metadata of wheel packages """ root_directory = 'python' @@ -56,6 +58,10 @@ class Python: test_directory_name = 'tests' + wheel_directory_name = 'dist' + + wheel_metadata_directory_suffix = '.egg-info' + @staticmethod def file_search() -> FileSearch: """ @@ -65,8 +71,10 @@ def file_search() -> FileSearch: """ return FileSearch() \ .set_recursive(True) \ - .exclude_subdirectories_by_name(Project.Python.build_directory_name, 'dist', '__pycache__') \ - .exclude_subdirectories_by_substrings(ends_with='.egg.info') + .exclude_subdirectories_by_name(Project.Python.build_directory_name) \ + .exclude_subdirectories_by_name(Project.Python.wheel_directory_name) \ + .exclude_subdirectories_by_name('__pycache__') \ + .exclude_subdirectories_by_substrings(ends_with=Project.Python.wheel_metadata_directory_suffix) class Cpp: """ diff --git a/scons/util/requirements.txt b/scons/util/requirements.txt index e2710db13..f476b0b6c 100644 --- a/scons/util/requirements.txt +++ b/scons/util/requirements.txt @@ -1,4 +1 @@ -build >= 1.2, < 1.3 -setuptools scons >= 4.8, < 4.9 -wheel >= 0.45, < 0.46 diff --git a/scons/util/targets.py b/scons/util/targets.py index e775fb92a..7a3d20684 100644 --- a/scons/util/targets.py +++ b/scons/util/targets.py @@ -8,7 +8,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from functools import reduce -from typing import Any, Callable, Dict, List, Optional, Set +from typing import Any, Callable, Dict, Iterable, List, Optional, Set from uuid import uuid4 from util.modules import ModuleRegistry @@ -84,6 +84,27 @@ def depends_on_build_target(self, clean_dependency: bool = True) -> 'BuildTarget self.depends_on(target_name, clean_dependencies=clean_dependency) return target_builder + def depends_on_build_targets(self, + iterable: Iterable[Any], + target_configurator: Callable[[Any, 'BuildTarget.Builder'], None], + clean_dependencies: bool = True) -> 'Target.Builder': + """ + Configures multiple build targets, one for each value in a given `Iterable`, this target should depend on. + + :param iterable: An `Iterable` that provides access to the values for which dependencies should + be created + :param target_configurator: A function that accepts one value in the `Iterable` at a time, as well as a + `BuildTarget.Builder` for configuring the corresponding dependency + :param clean_dependencies: True, if output files of the dependencies should also be cleaned when cleaning + the output files of this target, False otherwise + :return: The `Target.Builder` itself + """ + for value in iterable: + target_builder = self.depends_on_build_target(clean_dependency=clean_dependencies) + target_configurator(value, target_builder) + + return self + def depends_on_phony_target(self, clean_dependency: bool = True) -> 'PhonyTarget.Builder': """ Creates and returns a `PhonyTarget.Builder` that allows to configure a phony target, this target should @@ -99,6 +120,27 @@ def depends_on_phony_target(self, clean_dependency: bool = True) -> 'PhonyTarget self.depends_on(target_name, clean_dependencies=clean_dependency) return target_builder + def depends_on_phony_targets(self, + iterable: Iterable[Any], + target_configurator: Callable[[Any, 'PhonyTarget.Builder'], None], + clean_dependencies: bool = True) -> 'Target.Builder': + """ + Configures multiple phony targets, one for each value in a given `Iterable`, this target should depend on. + + :param iterable: An `Iterable` that provides access to the values for which dependencies should + be created + :param target_configurator: A function that accepts one value in the `Iterable` at a time, as well as a + `BuildTarget.Builder` for configuring the corresponding dependency + :param clean_dependencies: True, if output files of the dependencies should also be cleaned when cleaning + the output files of this target, False otherwise + :return: The `Target.Builder` itself + """ + for value in iterable: + target_builder = self.depends_on_phony_target(clean_dependency=clean_dependencies) + target_configurator(value, target_builder) + + return self + @abstractmethod def _build(self, build_unit: BuildUnit) -> 'Target': """ @@ -244,7 +286,10 @@ def action(): target = (output_files if len(output_files) > 1 else output_files[0]) if output_files else None if target: - return environment.Command(target, source, action=lambda **_: action()) + return environment.Depends( + environment.Alias(self.name, None, None), + environment.Command(target, source, action=lambda **_: action()), + ) return environment.AlwaysBuild(environment.Alias(self.name, None, action=lambda **_: action())) @@ -338,7 +383,8 @@ def __init__(self, name: str, dependencies: Set[Target.Dependency], action: Call def register(self, environment: Environment, module_registry: ModuleRegistry) -> Any: return environment.AlwaysBuild( - environment.Alias(self.name, None, action=lambda **_: self.action(module_registry))) + environment.Alias(self.name, None, action=lambda **_: self.action(module_registry)), + ) class TargetBuilder: From 5ded4316be55eb550b65357b70ed20134a3e7a5d Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Fri, 6 Dec 2024 00:44:03 +0100 Subject: [PATCH 075/114] Replace enum Language with class FileType. --- scons/code_style/cpp/__init__.py | 4 +- scons/code_style/cpp/targets.py | 4 +- scons/code_style/markdown/__init__.py | 9 +- scons/code_style/markdown/targets.py | 4 +- scons/code_style/modules.py | 21 +++-- scons/code_style/python/__init__.py | 10 +-- scons/code_style/python/targets.py | 6 +- scons/code_style/yaml/__init__.py | 7 +- scons/code_style/yaml/targets.py | 4 +- scons/compilation/cpp/__init__.py | 10 +-- scons/compilation/cpp/targets.py | 4 +- scons/compilation/cython/__init__.py | 9 +- scons/compilation/cython/targets.py | 4 +- scons/compilation/modules.py | 20 ++--- scons/dependencies/github/actions.py | 5 +- scons/packaging/targets.py | 6 +- scons/testing/python/modules.py | 5 +- scons/util/files.py | 113 ++++++++++++++++++++++++-- scons/util/languages.py | 17 ---- 19 files changed, 166 insertions(+), 96 deletions(-) delete mode 100644 scons/util/languages.py diff --git a/scons/code_style/cpp/__init__.py b/scons/code_style/cpp/__init__.py index 6e3ee36c9..d35ff2043 100644 --- a/scons/code_style/cpp/__init__.py +++ b/scons/code_style/cpp/__init__.py @@ -5,7 +5,7 @@ """ from code_style.cpp.targets import CheckCppCodeStyle, EnforceCppCodeStyle from code_style.modules import CodeModule -from util.languages import Language +from util.files import FileType from util.paths import Project from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -21,7 +21,7 @@ MODULES = [ CodeModule( - language=Language.CPP, + file_type=FileType.cpp(), root_directory=Project.Cpp.root_directory, source_file_search=Project.Cpp.file_search(), ), diff --git a/scons/code_style/cpp/targets.py b/scons/code_style/cpp/targets.py index dbaec1464..9b09fc63b 100644 --- a/scons/code_style/cpp/targets.py +++ b/scons/code_style/cpp/targets.py @@ -6,12 +6,12 @@ from code_style.cpp.clang_format import ClangFormat from code_style.cpp.cpplint import CppLint from code_style.modules import CodeModule -from util.languages import Language +from util.files import FileType from util.modules import ModuleRegistry from util.targets import PhonyTarget from util.units import BuildUnit -MODULE_FILTER = CodeModule.Filter(Language.CPP) +MODULE_FILTER = CodeModule.Filter(FileType.cpp()) class CheckCppCodeStyle(PhonyTarget.Runnable): diff --git a/scons/code_style/markdown/__init__.py b/scons/code_style/markdown/__init__.py index a0a92bded..9b0740eaf 100644 --- a/scons/code_style/markdown/__init__.py +++ b/scons/code_style/markdown/__init__.py @@ -5,8 +5,7 @@ """ from code_style.markdown.targets import CheckMarkdownCodeStyle, EnforceMarkdownCodeStyle from code_style.modules import CodeModule -from util.files import FileSearch -from util.languages import Language +from util.files import FileSearch, FileType from util.paths import Project from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -22,17 +21,17 @@ MODULES = [ CodeModule( - language=Language.MARKDOWN, + file_type=FileType.markdown(), root_directory=Project.root_directory, source_file_search=FileSearch().set_recursive(False), ), CodeModule( - language=Language.MARKDOWN, + file_type=FileType.markdown(), root_directory=Project.Python.root_directory, source_file_search=Project.Python.file_search(), ), CodeModule( - language=Language.MARKDOWN, + file_type=FileType.markdown(), root_directory=Project.Documentation.root_directory, source_file_search=Project.Documentation.file_search(), ), diff --git a/scons/code_style/markdown/targets.py b/scons/code_style/markdown/targets.py index 0274b37f4..79621bc1e 100644 --- a/scons/code_style/markdown/targets.py +++ b/scons/code_style/markdown/targets.py @@ -5,12 +5,12 @@ """ from code_style.markdown.mdformat import MdFormat from code_style.modules import CodeModule -from util.languages import Language +from util.files import FileType from util.modules import ModuleRegistry from util.targets import PhonyTarget from util.units import BuildUnit -MODULE_FILTER = CodeModule.Filter(Language.MARKDOWN) +MODULE_FILTER = CodeModule.Filter(FileType.markdown()) class CheckMarkdownCodeStyle(PhonyTarget.Runnable): diff --git a/scons/code_style/modules.py b/scons/code_style/modules.py index ffd5fc1d9..9931f678a 100644 --- a/scons/code_style/modules.py +++ b/scons/code_style/modules.py @@ -5,8 +5,7 @@ """ from typing import List -from util.files import FileSearch -from util.languages import Language +from util.files import FileSearch, FileType from util.modules import Module @@ -20,26 +19,26 @@ class Filter(Module.Filter): A filter that matches code modules. """ - def __init__(self, *languages: Language): + def __init__(self, *file_types: FileType): """ - :param languages: The languages of the code modules to be matched or None, if no restrictions should be - imposed on the languages + :param file_types: The file types of the code modules to be matched or None, if no restrictions should be + imposed on the file types """ - self.languages = set(languages) + self.file_types = set(file_types) def matches(self, module: Module) -> bool: - return isinstance(module, CodeModule) and (not self.languages or module.language in self.languages) + return isinstance(module, CodeModule) and (not self.file_types or module.file_type in self.file_types) def __init__(self, - language: Language, + file_type: FileType, root_directory: str, source_file_search: FileSearch = FileSearch().set_recursive(True)): """ - :param language: The programming language of the source code that belongs to the module + :param file_type: The `FileType` of the source files that belongs to the module :param root_directory: The path to the module's root directory :param source_file_search: The `FileSearch` that should be used to search for source files """ - self.language = language + self.file_type = file_type self.root_directory = root_directory self.source_file_search = source_file_search @@ -49,4 +48,4 @@ def find_source_files(self) -> List[str]: :return: A list that contains the paths of the source files that have been found """ - return self.source_file_search.filter_by_language(self.language).list(self.root_directory) + return self.source_file_search.filter_by_file_type(self.file_type).list(self.root_directory) diff --git a/scons/code_style/python/__init__.py b/scons/code_style/python/__init__.py index efbacb6dc..3f17e94f9 100644 --- a/scons/code_style/python/__init__.py +++ b/scons/code_style/python/__init__.py @@ -6,7 +6,7 @@ from code_style.modules import CodeModule from code_style.python.targets import CheckCythonCodeStyle, CheckPythonCodeStyle, EnforceCythonCodeStyle, \ EnforcePythonCodeStyle -from util.languages import Language +from util.files import FileType from util.paths import Project from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -22,22 +22,22 @@ MODULES = [ CodeModule( - language=Language.PYTHON, + file_type=FileType.python(), root_directory=Project.BuildSystem.root_directory, source_file_search=Project.BuildSystem.file_search(), ), CodeModule( - language=Language.PYTHON, + file_type=FileType.python(), root_directory=Project.Python.root_directory, source_file_search=Project.Python.file_search(), ), CodeModule( - language=Language.CYTHON, + file_type=FileType.cython(), root_directory=Project.Python.root_directory, source_file_search=Project.Python.file_search(), ), CodeModule( - language=Language.PYTHON, + file_type=FileType.python(), root_directory=Project.Documentation.root_directory, source_file_search=Project.Documentation.file_search(), ), diff --git a/scons/code_style/python/targets.py b/scons/code_style/python/targets.py index cb52c9468..89da3cba8 100644 --- a/scons/code_style/python/targets.py +++ b/scons/code_style/python/targets.py @@ -7,14 +7,14 @@ from code_style.python.isort import ISort from code_style.python.pylint import PyLint from code_style.python.yapf import Yapf -from util.languages import Language +from util.files import FileType from util.modules import ModuleRegistry from util.targets import PhonyTarget from util.units import BuildUnit -PYTHON_MODULE_FILTER = CodeModule.Filter(Language.PYTHON) +PYTHON_MODULE_FILTER = CodeModule.Filter(FileType.python()) -CYTHON_MODULE_FILTER = CodeModule.Filter(Language.CYTHON) +CYTHON_MODULE_FILTER = CodeModule.Filter(FileType.cython()) class CheckPythonCodeStyle(PhonyTarget.Runnable): diff --git a/scons/code_style/yaml/__init__.py b/scons/code_style/yaml/__init__.py index afc9f5483..7580ededa 100644 --- a/scons/code_style/yaml/__init__.py +++ b/scons/code_style/yaml/__init__.py @@ -5,8 +5,7 @@ """ from code_style.modules import CodeModule from code_style.yaml.targets import CheckYamlCodeStyle, EnforceYamlCodeStyle -from util.files import FileSearch -from util.languages import Language +from util.files import FileSearch, FileType from util.paths import Project from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -22,12 +21,12 @@ MODULES = [ CodeModule( - language=Language.YAML, + file_type=FileType.yaml(), root_directory=Project.root_directory, source_file_search=FileSearch().set_recursive(False).set_hidden(True), ), CodeModule( - language=Language.YAML, + file_type=FileType.yaml(), root_directory=Project.Github.root_directory, ), ] diff --git a/scons/code_style/yaml/targets.py b/scons/code_style/yaml/targets.py index af1abc0fb..f6f386267 100644 --- a/scons/code_style/yaml/targets.py +++ b/scons/code_style/yaml/targets.py @@ -5,12 +5,12 @@ """ from code_style.modules import CodeModule from code_style.yaml.yamlfix import YamlFix -from util.languages import Language +from util.files import FileType from util.modules import ModuleRegistry from util.targets import PhonyTarget from util.units import BuildUnit -MODULE_FILTER = CodeModule.Filter(Language.YAML) +MODULE_FILTER = CodeModule.Filter(FileType.yaml()) class CheckYamlCodeStyle(PhonyTarget.Runnable): diff --git a/scons/compilation/cpp/__init__.py b/scons/compilation/cpp/__init__.py index e4616dde0..0b09d64c6 100644 --- a/scons/compilation/cpp/__init__.py +++ b/scons/compilation/cpp/__init__.py @@ -6,7 +6,7 @@ from compilation.cpp.targets import CompileCpp, InstallCpp, SetupCpp from compilation.modules import CompilationModule from dependencies.python import VENV -from util.languages import Language +from util.files import FileType from util.paths import Project from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -31,14 +31,10 @@ MODULES = [ CompilationModule( - language=Language.CPP, + file_type=FileType.cpp(), root_directory=Project.Cpp.root_directory, build_directory_name=Project.Cpp.build_directory_name, install_directory=Project.Python.root_directory, - installed_file_search=Project.Cpp.file_search() \ - .filter_by_substrings(starts_with='lib', contains='.so') \ - .filter_by_substrings(ends_with='.dylib') \ - .filter_by_substrings(starts_with='mlrl', ends_with='.lib') \ - .filter_by_substrings(ends_with='.dll'), + installed_file_search=Project.Cpp.file_search().filter_by_file_type(FileType.shared_library()), ), ] diff --git a/scons/compilation/cpp/targets.py b/scons/compilation/cpp/targets.py index 59a1be11d..fd0f13aaa 100644 --- a/scons/compilation/cpp/targets.py +++ b/scons/compilation/cpp/targets.py @@ -9,12 +9,12 @@ from compilation.build_options import BuildOptions, EnvBuildOption from compilation.meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup from compilation.modules import CompilationModule -from util.languages import Language +from util.files import FileType from util.modules import ModuleRegistry from util.targets import BuildTarget, PhonyTarget from util.units import BuildUnit -MODULE_FILTER = CompilationModule.Filter(Language.CPP) +MODULE_FILTER = CompilationModule.Filter(FileType.cpp()) BUILD_OPTIONS = BuildOptions() \ .add(EnvBuildOption(name='subprojects')) \ diff --git a/scons/compilation/cython/__init__.py b/scons/compilation/cython/__init__.py index 7b432a7e0..8a432ff28 100644 --- a/scons/compilation/cython/__init__.py +++ b/scons/compilation/cython/__init__.py @@ -6,7 +6,7 @@ from compilation.cpp import COMPILE_CPP from compilation.cython.targets import CompileCython, InstallCython, SetupCython from compilation.modules import CompilationModule -from util.languages import Language +from util.files import FileType from util.paths import Project from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -31,12 +31,9 @@ MODULES = [ CompilationModule( - language=Language.CYTHON, + file_type=FileType.cython(), root_directory=Project.Python.root_directory, build_directory_name=Project.Python.build_directory_name, - installed_file_search=Project.Python.file_search() \ - .filter_by_substrings(not_starts_with='lib', ends_with='.so') \ - .filter_by_substrings(ends_with='.pyd') \ - .filter_by_substrings(not_starts_with='mlrl', ends_with='.lib'), + installed_file_search=Project.Python.file_search().filter_by_file_type(FileType.extension_module()), ), ] diff --git a/scons/compilation/cython/targets.py b/scons/compilation/cython/targets.py index 281631643..16e841622 100644 --- a/scons/compilation/cython/targets.py +++ b/scons/compilation/cython/targets.py @@ -9,12 +9,12 @@ from compilation.build_options import BuildOptions, EnvBuildOption from compilation.meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup from compilation.modules import CompilationModule -from util.languages import Language +from util.files import FileType from util.modules import ModuleRegistry from util.targets import BuildTarget, PhonyTarget from util.units import BuildUnit -MODULE_FILTER = CompilationModule.Filter(Language.CYTHON) +MODULE_FILTER = CompilationModule.Filter(FileType.cython()) BUILD_OPTIONS = BuildOptions() \ .add(EnvBuildOption(name='subprojects')) diff --git a/scons/compilation/modules.py b/scons/compilation/modules.py index e8716a9ae..cf32e05ab 100644 --- a/scons/compilation/modules.py +++ b/scons/compilation/modules.py @@ -6,8 +6,7 @@ from os import path from typing import List, Optional -from util.files import FileSearch -from util.languages import Language +from util.files import FileSearch, FileType from util.modules import Module @@ -21,24 +20,25 @@ class Filter(Module.Filter): A filter that matches modules that contain source code that must be compiled. """ - def __init__(self, *languages: Language): + def __init__(self, *file_types: FileType): """ - :param languages: The languages of the source code contained by the modules to be matched or None, if no - restrictions should be imposed on the languages + :param file_types: The file types of the source files contained by the modules to be matched or None, if no + restrictions should be imposed on the file types """ - self.languages = set(languages) + self.file_types = set(file_types) def matches(self, module: Module) -> bool: - return isinstance(module, CompilationModule) and (not self.languages or module.language in self.languages) + return isinstance(module, CompilationModule) and (not self.file_types + or module.file_type in self.file_types) def __init__(self, - language: Language, + file_type: FileType, root_directory: str, build_directory_name: str, install_directory: Optional[str] = None, installed_file_search: Optional[FileSearch] = None): """ - :param language: The programming language of the source code that belongs to the module + :param file_type: The file types of the source files that belongs to the module :param root_directory: The path to the module's root directory :param build_directory_name: The name of the module's build directory :param install_directory: The path to the directory into which files are installed or None, if the files @@ -46,7 +46,7 @@ def __init__(self, :param installed_file_search: The `FileSearch` that should be used to search for installed files or None, if the module does never contain any installed files """ - self.language = language + self.file_type = file_type self.root_directory = root_directory self.build_directory_name = build_directory_name self.install_directory = install_directory if install_directory else root_directory diff --git a/scons/dependencies/github/actions.py b/scons/dependencies/github/actions.py index 8b2715307..432cd9c0a 100644 --- a/scons/dependencies/github/actions.py +++ b/scons/dependencies/github/actions.py @@ -13,8 +13,7 @@ from dependencies.github.pygithub import GithubApi from dependencies.github.pyyaml import YamlFile from util.env import get_env -from util.files import FileSearch -from util.languages import Language +from util.files import FileSearch, FileType from util.units import BuildUnit @@ -293,7 +292,7 @@ def workflows(self) -> Set[Workflow]: """ workflows = set() - for workflow_file in FileSearch().filter_by_language(Language.YAML).list(self.workflow_directory): + for workflow_file in FileSearch().filter_by_file_type(FileType.yaml()).list(self.workflow_directory): print('Searching for GitHub Actions in workflow "' + workflow_file + '"...') workflows.add(Workflow(self.build_unit, workflow_file)) diff --git a/scons/packaging/targets.py b/scons/packaging/targets.py index 7472c6093..6113303b3 100644 --- a/scons/packaging/targets.py +++ b/scons/packaging/targets.py @@ -9,8 +9,7 @@ from packaging.build import Build from packaging.modules import PythonPackageModule from packaging.pip import PipInstallWheel -from util.files import DirectorySearch -from util.languages import Language +from util.files import DirectorySearch, FileType from util.modules import ModuleRegistry from util.paths import Project from util.targets import BuildTarget, PhonyTarget @@ -38,8 +37,7 @@ def get_input_files(self, modules: ModuleRegistry) -> List[str]: file_search = Project.Python.file_search() \ .set_symlinks(False) \ .exclude_subdirectories_by_name(Project.Python.test_directory_name) \ - .filter_by_language(Language.PYTHON) \ - .filter_by_suffix('so', 'pyd', 'lib', 'dylib', 'dll') + .filter_by_file_type(FileType.python(), FileType.extension_module(), FileType.shared_library()) return reduce(lambda aggr, module: aggr + file_search.list(module.root_directory), package_modules, []) def get_output_files(self, modules: ModuleRegistry) -> List[str]: diff --git a/scons/testing/python/modules.py b/scons/testing/python/modules.py index e40109e56..75f63145d 100644 --- a/scons/testing/python/modules.py +++ b/scons/testing/python/modules.py @@ -7,8 +7,7 @@ from typing import List from testing.modules import TestModule -from util.files import FileSearch -from util.languages import Language +from util.files import FileSearch, FileType from util.modules import Module @@ -54,5 +53,5 @@ def find_test_directories(self) -> List[str]: return self.test_file_search \ .exclude_subdirectories_by_name(self.build_directory_name) \ .filter_by_substrings(starts_with='test_') \ - .filter_by_language(Language.PYTHON) \ + .filter_by_file_type(FileType.python()) \ .list(self.root_directory) diff --git a/scons/util/files.py b/scons/util/files.py index 923183ffd..cf8e334fe 100644 --- a/scons/util/files.py +++ b/scons/util/files.py @@ -3,13 +3,12 @@ Provides classes for listing files and directories. """ +from abc import abstractmethod from functools import partial, reduce from glob import glob from os import path from typing import Callable, List, Optional, Set -from util.languages import Language - class DirectorySearch: """ @@ -334,14 +333,17 @@ def filter_file(filtered_suffixes: List[str], _: str, file_name: str): return self.add_filters(partial(filter_file, list(suffixes))) - def filter_by_language(self, *languages: Language) -> 'FileSearch': + def filter_by_file_type(self, *file_types: 'FileType') -> 'FileSearch': """ - Adds one or several filters that match files to be included based on the programming language they belong to. + Adds one or several filters that match files to be included based on a `FileType`. - :param languages: The languages of the files that should be included + :param file_types: The `FileType` of the files that should be included :return: The `FileSearch` itself """ - return self.filter_by_suffix(*reduce(lambda aggr, language: aggr | language.value, languages, set())) + for file_type in file_types: + file_type.file_search_decorator(self) + + return self def list(self, *directories: str) -> List[str]: """ @@ -375,3 +377,102 @@ def filter_file(file: str) -> bool: result.extend(files) return result + + +class FileType: + """ + Represents different types of files. + """ + + def __init__(self, name: str, file_search_decorator: Callable[[FileSearch], None]): + """ + :param name: The name of the file type + :param file_search_decorator: A function that adds a filter for this file type to a `FileSearch` + """ + self.name = name + self.file_search_decorator = file_search_decorator + + @staticmethod + def python() -> 'FileType': + """ + Creates and returns a `FileType` that corresponds to Python source files. + + :return: The `FileType` that has been created + """ + return FileType('Python', lambda file_search: file_search.filter_by_suffix('py')) + + @staticmethod + def cpp() -> 'FileType': + """ + Creates and returns a `FileType` that corresponds to C++ source files. + + :return: The `FileType` that has been created + """ + return FileType('C++', lambda file_search: file_search.filter_by_suffix('cpp', 'hpp')) + + @staticmethod + def cython() -> 'FileType': + """ + Creates and returns a `FileType` that corresponds to Cython source files. + + :return: The `FileType` that has been created + """ + return FileType('Cython', lambda file_search: file_search.filter_by_suffix('pyx', 'pxd')) + + @staticmethod + def markdown() -> 'FileType': + """ + Creates and returns a `FileType` that corresponds to Markdown files. + + :return: The `FileType` that has been created + """ + return FileType('Markdown', lambda file_search: file_search.filter_by_suffix('md')) + + @staticmethod + def yaml() -> 'FileType': + """ + Creates and returns a `FileType` that corresponds to YAML files. + + :return: The `FileType` that has been created + """ + return FileType('YAML', lambda file_search: file_search.filter_by_suffix('yaml', 'yml')) + + @staticmethod + def extension_module() -> 'FileType': + """ + Creates and returns a `FileType` that corresponds to shared libraries. + + :return: The `FileType` that has been created + """ + return FileType( + 'Extension module', + lambda file_search: file_search \ + .filter_by_substrings(not_starts_with='lib', ends_with='.so') \ + .filter_by_substrings(ends_with='.pyd') \ + .filter_by_substrings(not_starts_with='mlrl', ends_with='.lib'), + ) + + @staticmethod + def shared_library() -> 'FileType': + """ + Creates and returns a `FileType` that corresponds to shared libraries. + + :return: The `FileType` that has been created + """ + return FileType( + 'Shared library', + lambda file_search: file_search \ + .filter_by_substrings(starts_with='lib', contains='.so') \ + .filter_by_substrings(ends_with='.dylib') \ + .filter_by_substrings(starts_with='mlrl', ends_with='.lib') \ + .filter_by_substrings(ends_with='.dll'), + ) + + def __str__(self) -> str: + return self.name + + def __eq__(self, other: 'FileType') -> bool: + return self.name == other.name + + def __hash__(self) -> int: + return hash(self.name) diff --git a/scons/util/languages.py b/scons/util/languages.py deleted file mode 100644 index 7be34fd1a..000000000 --- a/scons/util/languages.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Provides classes that help to distinguish between different programming languages. -""" -from enum import Enum - - -class Language(Enum): - """ - Different programming languages. - """ - CPP = {'cpp', 'hpp'} - PYTHON = {'py'} - CYTHON = {'pyx', 'pxd'} - MARKDOWN = {'md'} - YAML = {'yaml', 'yml'} From f89be9d3b6f5f0055f078f42d270a8ca171a1522 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Fri, 6 Dec 2024 15:10:44 +0100 Subject: [PATCH 076/114] Add class Log. --- scons/code_style/cpp/targets.py | 5 +- scons/code_style/markdown/targets.py | 5 +- scons/code_style/python/targets.py | 9 +-- scons/code_style/yaml/targets.py | 5 +- scons/compilation/cpp/targets.py | 9 +-- scons/compilation/cython/targets.py | 9 +-- scons/compilation/meson.py | 3 +- scons/dependencies/github/actions.py | 19 +++--- scons/dependencies/github/targets.py | 11 ++-- scons/dependencies/python/targets.py | 10 ++-- scons/documentation.py | 11 ++-- scons/packaging/targets.py | 7 ++- scons/util/cmd.py | 8 +-- scons/util/env.py | 4 +- scons/util/io.py | 2 +- scons/util/log.py | 87 ++++++++++++++++++++++++++++ scons/util/pip.py | 3 +- scons/versioning/changelog.py | 10 ++-- scons/versioning/versioning.py | 11 ++-- 19 files changed, 162 insertions(+), 66 deletions(-) create mode 100644 scons/util/log.py diff --git a/scons/code_style/cpp/targets.py b/scons/code_style/cpp/targets.py index 9b09fc63b..3d8bcffa6 100644 --- a/scons/code_style/cpp/targets.py +++ b/scons/code_style/cpp/targets.py @@ -7,6 +7,7 @@ from code_style.cpp.cpplint import CppLint from code_style.modules import CodeModule from util.files import FileType +from util.log import Log from util.modules import ModuleRegistry from util.targets import PhonyTarget from util.units import BuildUnit @@ -21,7 +22,7 @@ class CheckCppCodeStyle(PhonyTarget.Runnable): def run(self, build_unit: BuildUnit, modules: ModuleRegistry): for module in modules.lookup(MODULE_FILTER): - print('Checking C++ code style in directory "' + module.root_directory + '"...') + Log.info('Checking C++ code style in directory "%s"...', module.root_directory) ClangFormat(build_unit, module).run() CppLint(build_unit, module).run() @@ -33,5 +34,5 @@ class EnforceCppCodeStyle(PhonyTarget.Runnable): def run(self, build_unit: BuildUnit, modules: ModuleRegistry): for module in modules.lookup(MODULE_FILTER): - print('Formatting C++ code in directory "' + module.root_directory + '"...') + Log.info('Formatting C++ code in directory "%s"...', module.root_directory) ClangFormat(build_unit, module, enforce_changes=True).run() diff --git a/scons/code_style/markdown/targets.py b/scons/code_style/markdown/targets.py index 79621bc1e..16f9925e9 100644 --- a/scons/code_style/markdown/targets.py +++ b/scons/code_style/markdown/targets.py @@ -6,6 +6,7 @@ from code_style.markdown.mdformat import MdFormat from code_style.modules import CodeModule from util.files import FileType +from util.log import Log from util.modules import ModuleRegistry from util.targets import PhonyTarget from util.units import BuildUnit @@ -20,7 +21,7 @@ class CheckMarkdownCodeStyle(PhonyTarget.Runnable): def run(self, build_unit: BuildUnit, modules: ModuleRegistry): for module in modules.lookup(MODULE_FILTER): - print('Checking Markdown code style in the directory "' + module.root_directory + '"...') + Log.info('Checking Markdown code style in the directory "%s"...', module.root_directory) MdFormat(build_unit, module).run() @@ -31,5 +32,5 @@ class EnforceMarkdownCodeStyle(PhonyTarget.Runnable): def run(self, build_unit: BuildUnit, modules: ModuleRegistry): for module in modules.lookup(MODULE_FILTER): - print('Formatting Markdown files in the directory "' + module.root_directory + '"...') + Log.info('Formatting Markdown files in the directory "%s"...', module.root_directory) MdFormat(build_unit, module, enforce_changes=True).run() diff --git a/scons/code_style/python/targets.py b/scons/code_style/python/targets.py index 89da3cba8..fa62c6aa8 100644 --- a/scons/code_style/python/targets.py +++ b/scons/code_style/python/targets.py @@ -8,6 +8,7 @@ from code_style.python.pylint import PyLint from code_style.python.yapf import Yapf from util.files import FileType +from util.log import Log from util.modules import ModuleRegistry from util.targets import PhonyTarget from util.units import BuildUnit @@ -24,7 +25,7 @@ class CheckPythonCodeStyle(PhonyTarget.Runnable): def run(self, build_unit: BuildUnit, modules: ModuleRegistry): for module in modules.lookup(PYTHON_MODULE_FILTER): - print('Checking Python code style in directory "' + module.root_directory + '"...') + Log.info('Checking Python code style in directory "%s"...', module.root_directory) ISort(build_unit, module).run() Yapf(build_unit, module).run() PyLint(build_unit, module).run() @@ -37,7 +38,7 @@ class EnforcePythonCodeStyle(PhonyTarget.Runnable): def run(self, build_unit: BuildUnit, modules: ModuleRegistry): for module in modules.lookup(PYTHON_MODULE_FILTER): - print('Formatting Python code in directory "' + module.root_directory + '"...') + Log.info('Formatting Python code in directory "%s"...', module.root_directory) ISort(build_unit, module, enforce_changes=True).run() Yapf(build_unit, module, enforce_changes=True).run() @@ -49,7 +50,7 @@ class CheckCythonCodeStyle(PhonyTarget.Runnable): def run(self, build_unit: BuildUnit, modules: ModuleRegistry): for module in modules.lookup(CYTHON_MODULE_FILTER): - print('Checking Cython code style in directory "' + module.root_directory + '"...') + Log.info('Checking Cython code style in directory "%s"...', module.root_directory) ISort(build_unit, module).run() @@ -60,5 +61,5 @@ class EnforceCythonCodeStyle(PhonyTarget.Runnable): def run(self, build_unit: BuildUnit, modules: ModuleRegistry): for module in modules.lookup(CYTHON_MODULE_FILTER): - print('Formatting Cython code in directory "' + module.root_directory + '"...') + Log.info('Formatting Cython code in directory "%s"...', module.root_directory) ISort(build_unit, module, enforce_changes=True).run() diff --git a/scons/code_style/yaml/targets.py b/scons/code_style/yaml/targets.py index f6f386267..c1ced0145 100644 --- a/scons/code_style/yaml/targets.py +++ b/scons/code_style/yaml/targets.py @@ -6,6 +6,7 @@ from code_style.modules import CodeModule from code_style.yaml.yamlfix import YamlFix from util.files import FileType +from util.log import Log from util.modules import ModuleRegistry from util.targets import PhonyTarget from util.units import BuildUnit @@ -20,7 +21,7 @@ class CheckYamlCodeStyle(PhonyTarget.Runnable): def run(self, build_unit: BuildUnit, modules: ModuleRegistry): for module in modules.lookup(MODULE_FILTER): - print('Checking YAML files in the directory "' + module.root_directory + '"...') + Log.info('Checking YAML files in the directory "%s"...', module.root_directory) YamlFix(build_unit, module).run() @@ -31,5 +32,5 @@ class EnforceYamlCodeStyle(PhonyTarget.Runnable): def run(self, build_unit: BuildUnit, modules: ModuleRegistry): for module in modules.lookup(MODULE_FILTER): - print('Formatting YAML files in the directory "' + module.root_directory + '"...') + Log.info('Formatting YAML files in the directory "%s"...', module.root_directory) YamlFix(build_unit, module, enforce_changes=True).run() diff --git a/scons/compilation/cpp/targets.py b/scons/compilation/cpp/targets.py index fd0f13aaa..2c2663f45 100644 --- a/scons/compilation/cpp/targets.py +++ b/scons/compilation/cpp/targets.py @@ -10,6 +10,7 @@ from compilation.meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup from compilation.modules import CompilationModule from util.files import FileType +from util.log import Log from util.modules import ModuleRegistry from util.targets import BuildTarget, PhonyTarget from util.units import BuildUnit @@ -36,7 +37,7 @@ def get_output_files(self, modules: ModuleRegistry) -> List[str]: return [module.build_directory for module in modules.lookup(MODULE_FILTER)] def get_clean_files(self, modules: ModuleRegistry) -> List[str]: - print('Removing C++ build files...') + Log.info('Removing C++ build files...') return super().get_clean_files(modules) @@ -46,7 +47,7 @@ class CompileCpp(PhonyTarget.Runnable): """ def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - print('Compiling C++ code...') + Log.info('Compiling C++ code...') for module in modules.lookup(MODULE_FILTER): MesonConfigure(build_unit, module, BUILD_OPTIONS).run() @@ -59,12 +60,12 @@ class InstallCpp(BuildTarget.Runnable): """ def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - print('Installing shared libraries into source tree...') + Log.info('Installing shared libraries into source tree...') for module in modules.lookup(MODULE_FILTER): MesonInstall(build_unit, module).run() def get_clean_files(self, modules: ModuleRegistry) -> List[str]: - print('Removing shared libraries from source tree...') + Log.info('Removing shared libraries from source tree...') compilation_modules = modules.lookup(MODULE_FILTER) return reduce(lambda aggr, module: aggr + module.find_installed_files(), compilation_modules, []) diff --git a/scons/compilation/cython/targets.py b/scons/compilation/cython/targets.py index 16e841622..0fe7ef37a 100644 --- a/scons/compilation/cython/targets.py +++ b/scons/compilation/cython/targets.py @@ -10,6 +10,7 @@ from compilation.meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup from compilation.modules import CompilationModule from util.files import FileType +from util.log import Log from util.modules import ModuleRegistry from util.targets import BuildTarget, PhonyTarget from util.units import BuildUnit @@ -35,7 +36,7 @@ def get_output_files(self, modules: ModuleRegistry) -> List[str]: return [module.build_directory for module in modules.lookup(MODULE_FILTER)] def get_clean_files(self, modules: ModuleRegistry) -> List[str]: - print('Removing Cython build files...') + Log.info('Removing Cython build files...') return super().get_clean_files(modules) @@ -45,7 +46,7 @@ class CompileCython(PhonyTarget.Runnable): """ def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - print('Compiling Cython code...') + Log.info('Compiling Cython code...') for module in modules.lookup(MODULE_FILTER): MesonConfigure(build_unit, module, build_options=BUILD_OPTIONS) @@ -58,12 +59,12 @@ class InstallCython(BuildTarget.Runnable): """ def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - print('Installing extension modules into source tree...') + Log.info('Installing extension modules into source tree...') for module in modules.lookup(MODULE_FILTER): MesonInstall(build_unit, module).run() def get_clean_files(self, modules: ModuleRegistry) -> List[str]: - print('Removing extension modules from source tree...') + Log.info('Removing extension modules from source tree...') compilation_modules = modules.lookup(MODULE_FILTER) return reduce(lambda aggr, module: aggr + module.find_installed_files(), compilation_modules, []) diff --git a/scons/compilation/meson.py b/scons/compilation/meson.py index 82e4e62af..a627bfe34 100644 --- a/scons/compilation/meson.py +++ b/scons/compilation/meson.py @@ -8,6 +8,7 @@ from compilation.build_options import BuildOptions from compilation.modules import CompilationModule +from util.log import Log from util.run import Program from util.units import BuildUnit @@ -80,7 +81,7 @@ def _should_be_skipped(self) -> bool: return not self.build_options def _before(self): - print('Configuring build options according to environment variables...') + Log.info('Configuring build options according to environment variables...') class MesonCompile(Meson): diff --git a/scons/dependencies/github/actions.py b/scons/dependencies/github/actions.py index 432cd9c0a..7a1f37b07 100644 --- a/scons/dependencies/github/actions.py +++ b/scons/dependencies/github/actions.py @@ -3,8 +3,6 @@ Provides utility functions for checking the project's GitHub workflows for outdated Actions. """ -import sys - from dataclasses import dataclass, replace from functools import cached_property, reduce from os import environ, path @@ -14,6 +12,7 @@ from dependencies.github.pyyaml import YamlFile from util.env import get_env from util.files import FileSearch, FileType +from util.log import Log from util.units import BuildUnit @@ -149,8 +148,7 @@ def actions(self) -> Set[Action]: try: actions.add(Action.from_uses_clause(uses_clause)) except ValueError as error: - print('Failed to parse uses-clause in workflow "' + self.file + '": ' + str(error)) - sys.exit(-1) + raise RuntimeError('Failed to parse uses-clause in workflow "' + self.file + '"') from error return actions @@ -243,8 +241,8 @@ def __get_github_token(self) -> Optional[str]: github_token = get_env(environ, self.ENV_GITHUB_TOKEN) if not github_token: - print('No GitHub API token is set. You can specify it via the environment variable ' + self.ENV_GITHUB_TOKEN - + '.') + Log.warning('No GitHub API token is set. You can specify it via the environment variable %s.', + WorkflowUpdater.ENV_GITHUB_TOKEN) return github_token @@ -262,15 +260,14 @@ def __query_latest_action_version(self, action: Action) -> ActionVersion: return ActionVersion(latest_tag) except RuntimeError as error: - print('Unable to determine latest version of action "' + str(action) + '" hosted in repository "' - + repository_name + '": ' + str(error)) - sys.exit(-1) + raise RuntimeError('Unable to determine latest version of action "' + str(action) + + '" hosted in repository "' + repository_name + '"') from error def __get_latest_action_version(self, action: Action) -> ActionVersion: latest_version = self.version_cache.get(action.name) if not latest_version: - print('Checking version of GitHub Action "' + action.name + '"...') + Log.info('Checking version of GitHub Action "%s"...', action.name) latest_version = self.__query_latest_action_version(action) self.version_cache[action.name] = latest_version @@ -293,7 +290,7 @@ def workflows(self) -> Set[Workflow]: workflows = set() for workflow_file in FileSearch().filter_by_file_type(FileType.yaml()).list(self.workflow_directory): - print('Searching for GitHub Actions in workflow "' + workflow_file + '"...') + Log.info('Searching for GitHub Actions in workflow "%s"...', workflow_file) workflows.add(Workflow(self.build_unit, workflow_file)) return workflows diff --git a/scons/dependencies/github/targets.py b/scons/dependencies/github/targets.py index cf8688b4d..8253b5be6 100644 --- a/scons/dependencies/github/targets.py +++ b/scons/dependencies/github/targets.py @@ -5,6 +5,7 @@ """ from dependencies.github.actions import WorkflowUpdater from dependencies.table import Table +from util.log import Log from util.modules import ModuleRegistry from util.targets import PhonyTarget from util.units import BuildUnit @@ -27,10 +28,9 @@ def run(self, build_unit: BuildUnit, _: ModuleRegistry): str(outdated_action.latest_version)) table.sort_rows(0, 1) - print('The following GitHub Actions are outdated:\n') - print(str(table)) + Log.info('The following GitHub Actions are outdated:\n\n%s', str(table)) else: - print('All GitHub Actions are up-to-date!') + Log.info('All GitHub Actions are up-to-date!') class UpdateGithubActions(PhonyTarget.Runnable): @@ -50,7 +50,6 @@ def run(self, build_unit: BuildUnit, _: ModuleRegistry): str(updated_action.previous.action.version), str(updated_action.updated.version)) table.sort_rows(0, 1) - print('The following GitHub Actions have been updated:\n') - print(str(table)) + Log.info('The following GitHub Actions have been updated:\n\n%s', str(table)) else: - print('No GitHub Actions have been updated.') + Log.info('No GitHub Actions have been updated.') diff --git a/scons/dependencies/python/targets.py b/scons/dependencies/python/targets.py index 8b9e8c8c3..64c624d9d 100644 --- a/scons/dependencies/python/targets.py +++ b/scons/dependencies/python/targets.py @@ -8,6 +8,7 @@ from dependencies.python.modules import DependencyType, PythonDependencyModule from dependencies.python.pip import PipList from dependencies.table import Table +from util.log import Log from util.modules import ModuleRegistry from util.targets import PhonyTarget from util.units import BuildUnit @@ -35,9 +36,9 @@ def run(self, build_unit: BuildUnit, modules: ModuleRegistry): requirements_files = reduce(lambda aggr, module: aggr + module.find_requirements_files(), dependency_modules, []) pip = PipList(*requirements_files) - print('Installing all dependencies...') + Log.info('Installing all dependencies...') pip.install_all_packages() - print('Checking for outdated dependencies...') + Log.info('Checking for outdated dependencies...') outdated_dependencies = pip.list_outdated_dependencies() if outdated_dependencies: @@ -48,7 +49,6 @@ def run(self, build_unit: BuildUnit, modules: ModuleRegistry): outdated_dependency.latest.version) table.sort_rows(0, 1) - print('The following dependencies are outdated:\n') - print(str(table)) + Log.info('The following dependencies are outdated:\n\n%s', str(table)) else: - print('All dependencies are up-to-date!') + Log.info('All dependencies are up-to-date!') diff --git a/scons/documentation.py b/scons/documentation.py index d81e8554d..d8e82c010 100644 --- a/scons/documentation.py +++ b/scons/documentation.py @@ -9,6 +9,7 @@ from modules_old import CPP_MODULE, DOC_MODULE, PYTHON_MODULE from util.env import set_env from util.io import read_file, write_file +from util.log import Log from util.run import Program @@ -93,7 +94,7 @@ def apidoc_cpp(env, target, source): if apidoc_subproject: subproject_name = apidoc_subproject.name - print('Generating C++ API documentation for subproject "' + subproject_name + '"...') + Log.info('Generating C++ API documentation for subproject "%s"...', subproject_name) include_dir = path.join(apidoc_subproject.source_subproject.root_dir, 'include') build_dir = apidoc_subproject.build_dir __doxygen(project_name=subproject_name, input_dir=include_dir, output_dir=build_dir) @@ -104,7 +105,7 @@ def apidoc_cpp_tocfile(**_): """ Generates a tocfile referencing the C++ API documentation for all existing subprojects. """ - print('Generating tocfile referencing the C++ API documentation for all subprojects...') + Log.info('Generating tocfile referencing the C++ API documentation for all subprojects...') tocfile_entries = [] for subproject in CPP_MODULE.find_subprojects(): @@ -132,7 +133,7 @@ def apidoc_python(env, target, source): apidoc_subproject = DOC_MODULE.find_python_apidoc_subproject(target[0].path) if apidoc_subproject: - print('Generating Python API documentation for subproject "' + apidoc_subproject.name + '"...') + Log.info('Generating Python API documentation for subproject "%s"...', apidoc_subproject.name) build_dir = apidoc_subproject.build_dir makedirs(build_dir, exist_ok=True) __sphinx_apidoc(source_dir=apidoc_subproject.source_subproject.source_dir, output_dir=build_dir) @@ -142,7 +143,7 @@ def apidoc_python_tocfile(**_): """ Generates a tocfile referencing the Python API documentation for all existing subprojects. """ - print('Generating tocfile referencing the Python API documentation for all subprojects...') + Log.info('Generating tocfile referencing the Python API documentation for all subprojects...') tocfile_entries = [] for subproject in PYTHON_MODULE.find_subprojects(): @@ -160,5 +161,5 @@ def doc(**_): """ Builds the documentation. """ - print('Generating documentation...') + Log.info('Generating documentation...') __sphinx_build(source_dir=DOC_MODULE.root_dir, output_dir=DOC_MODULE.build_dir) diff --git a/scons/packaging/targets.py b/scons/packaging/targets.py index 6113303b3..2324fc8a7 100644 --- a/scons/packaging/targets.py +++ b/scons/packaging/targets.py @@ -10,6 +10,7 @@ from packaging.modules import PythonPackageModule from packaging.pip import PipInstallWheel from util.files import DirectorySearch, FileType +from util.log import Log from util.modules import ModuleRegistry from util.paths import Project from util.targets import BuildTarget, PhonyTarget @@ -29,7 +30,7 @@ def __init__(self, root_directory: str): def run(self, build_unit: BuildUnit, modules: ModuleRegistry): for module in modules.lookup(self.module_filter): - print('Building Python wheels for directory "' + module.root_directory + '"...') + Log.info('Building Python wheels for directory "%s"...', module.root_directory) Build(build_unit, module).run() def get_input_files(self, modules: ModuleRegistry) -> List[str]: @@ -49,7 +50,7 @@ def get_clean_files(self, modules: ModuleRegistry) -> List[str]: clean_files = [] for module in modules.lookup(self.module_filter): - print('Removing Python wheels from directory "' + module.root_directory + '"...') + Log.info('Removing Python wheels from directory "%s"...', module.root_directory) clean_files.append(module.wheel_directory) clean_files.extend( DirectorySearch() \ @@ -74,5 +75,5 @@ def __init__(self, root_directory: str): def run(self, _: BuildUnit, modules: ModuleRegistry): for module in modules.lookup(self.module_filter): - print('Installing Python wheels for directory "' + module.root_directory + '"...') + Log.info('Installing Python wheels for directory "%s"...', module.root_directory) PipInstallWheel().install_wheels(module.find_wheels()) diff --git a/scons/util/cmd.py b/scons/util/cmd.py index a9ad151bd..fd3f046a9 100644 --- a/scons/util/cmd.py +++ b/scons/util/cmd.py @@ -10,6 +10,7 @@ from subprocess import CompletedProcess from util.format import format_iterable +from util.log import Log from util.venv import in_virtual_environment @@ -59,7 +60,7 @@ def run(self, command: 'Command', capture_output: bool) -> CompletedProcess: :return: The output of the program """ if self.print_command: - print('Running external command "' + command.print_options.format(command) + '"...') + Log.info('Running external command "%s"...', command.print_options.format(command)) output = subprocess.run([command.command] + command.arguments, check=False, @@ -74,10 +75,9 @@ def run(self, command: 'Command', capture_output: bool) -> CompletedProcess: if self.exit_on_error: if capture_output: - print(str(output.stderr).strip()) + Log.info('%s', str(output.stderr).strip()) - print(message) - sys.exit(exit_code) + Log.error(message, exit_code=exit_code) else: raise RuntimeError(message) diff --git a/scons/util/env.py b/scons/util/env.py index baf2d278a..54ba70c11 100644 --- a/scons/util/env.py +++ b/scons/util/env.py @@ -5,6 +5,8 @@ """ from typing import List, Optional +from util.log import Log + def get_env(env, name: str, default: Optional[str] = None) -> Optional[str]: """ @@ -57,4 +59,4 @@ def set_env(env, name: str, value: str): :param value: The value to be set """ env[name] = value - print('Set environment variable "' + name + '" to value "' + value + '"') + Log.info('Set environment variable "%s" to value "%s"', name, value) diff --git a/scons/util/io.py b/scons/util/io.py index d710291d6..babc12c06 100644 --- a/scons/util/io.py +++ b/scons/util/io.py @@ -67,7 +67,7 @@ def clear(self): """ Clears the text file. """ - print('Clearing file "' + self.file + '"...') + Log.info('Clearing file "%s"...', self.file) self.write_lines('') def __str__(self) -> str: diff --git a/scons/util/log.py b/scons/util/log.py new file mode 100644 index 000000000..cf5b488c9 --- /dev/null +++ b/scons/util/log.py @@ -0,0 +1,87 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes for writing log messages. +""" +import logging +import sys + +from enum import Enum +from typing import Optional + + +class Log: + """ + Allows to write log messages. + """ + + class Level(Enum): + """ + The log levels supported by the build system. + """ + NONE = logging.NOTSET + ERROR = logging.ERROR + WARNING = logging.WARNING + INFO = logging.INFO + VERBOSE = logging.DEBUG + + @staticmethod + def configure(log_level: Level = Level.INFO): + """ + Configures the logger to be used by the build system. + + :param log_level: The log level to be used + """ + root = logging.getLogger() + root.setLevel(log_level.value) + out_handler = logging.StreamHandler(sys.stdout) + out_handler.setLevel(log_level.value) + out_handler.setFormatter(logging.Formatter('%(message)s')) + root.addHandler(out_handler) + + @staticmethod + def error(message: str, *args, error: Optional[Exception] = None, exit_code: int = 1): + """ + Writes a log message at level `Log.Level.ERROR` and terminates the build system. + + :param message: The log message to be written + :param args: Optional arguments to be included in the log message + :param error: An optional error to be included in the log message + :param exit_code: The exit code to be returned when terminating the build system + """ + if error: + logging.error(message + ': %s', *args, error) + else: + logging.error(message, *args) + + sys.exit(exit_code) + + @staticmethod + def warning(message: str, *args): + """ + Writes a log message at level `Log.Level.WARNING`. + + :param message: The log message to be written + :param args: Optional arguments to be included in the log message + """ + logging.warning(message, *args) + + @staticmethod + def info(message: str, *args): + """ + Writes a log message at level `Log.Level.INFO`. + + :param message: The log message to be written + :param args: Optional arguments to be included in the log message + """ + logging.info(message, *args) + + @staticmethod + def verbose(message: str, *args): + """ + Writes a log message at level `Log.Level.VERBOSE`. + + :param message: The log message to be written + :param args: Optional arguments to be included in the log message + """ + logging.debug(message, *args) diff --git a/scons/util/pip.py b/scons/util/pip.py index 28575f7bd..189ee2ec1 100644 --- a/scons/util/pip.py +++ b/scons/util/pip.py @@ -10,6 +10,7 @@ from util.cmd import Command as Cmd from util.io import TextFile +from util.log import Log from util.units import BuildUnit @@ -217,7 +218,7 @@ def install_requirement(requirement: Requirement, dry_run: bool = False): .print_arguments(True) \ .run() else: - print(stdout) + Log.info(stdout) except RuntimeError: Pip.install_requirement(requirement) diff --git a/scons/versioning/changelog.py b/scons/versioning/changelog.py index a24a0bd24..c04b98dce 100644 --- a/scons/versioning/changelog.py +++ b/scons/versioning/changelog.py @@ -12,6 +12,7 @@ from typing import List, Optional from util.io import TextFile +from util.log import Log from versioning.versioning import Version, get_current_version CHANGESET_FILE_MAIN = '.changelog-main.md' @@ -261,7 +262,7 @@ def add_release(self, release: Release): :param release: The release to be added """ formatted_release = str(release) - print('Adding new release to changelog file "' + self.file + '":\n\n' + formatted_release) + Log.info('Adding new release to changelog file "%s":\n\n%s', self.file, formatted_release) original_lines = self.lines modified_lines = [] offset = 0 @@ -306,11 +307,10 @@ def latest(self) -> str: def __validate_changeset(changeset_file: str): try: - print('Validating changeset file "' + changeset_file + '"...') + Log.info('Validating changeset file "%s"...', changeset_file) ChangesetFile(changeset_file, accept_missing=True).validate() except ValueError as error: - print('Changeset file "' + changeset_file + '" is malformed!\n\n' + str(error)) - sys.exit(-1) + Log.error('Changeset file "%s" is malformed!\n\n%s', changeset_file, str(error)) def __merge_changesets(*changeset_files) -> List[Changeset]: @@ -384,4 +384,4 @@ def print_latest_changelog(): """ Prints the changelog of the latest release. """ - print(ChangelogFile().latest) + Log.info('%s', ChangelogFile().latest) diff --git a/scons/versioning/versioning.py b/scons/versioning/versioning.py index e9877b1c5..4561996be 100644 --- a/scons/versioning/versioning.py +++ b/scons/versioning/versioning.py @@ -8,6 +8,7 @@ from typing import Optional from util.io import TextFile +from util.log import Log @dataclass @@ -90,7 +91,7 @@ def version(self) -> Version: def update(self, version: Version): self.write_lines(str(version)) - print('Updated version to "' + str(version) + '"') + Log.info('Updated version to "%s"', str(version)) def write_lines(self, *lines: str): super().write_lines(lines) @@ -113,7 +114,7 @@ def development_version(self) -> int: def update(self, development_version: int): self.write_lines(str(development_version)) - print('Updated development version to "' + str(development_version) + '"') + Log.info('Updated development version to "%s"', str(development_version)) def write_lines(self, *lines: str): super().write_lines(lines) @@ -122,13 +123,13 @@ def write_lines(self, *lines: str): def __get_version_file() -> VersionFile: version_file = VersionFile() - print('Current version is "' + str(version_file.version) + '"') + Log.info('Current version is "%s"', str(version_file.version)) return version_file def __get_development_version_file() -> DevelopmentVersionFile: version_file = DevelopmentVersionFile() - print('Current development version is "' + str(version_file.development_version) + '"') + Log.info('Current development version is "%s"', str(version_file.development_version)) return version_file @@ -145,7 +146,7 @@ def print_current_version(): """ Prints the project's current version. """ - return print(str(get_current_version())) + return Log.info('%s', str(get_current_version())) def increment_development_version(): From 17ebb44e9acab449b0048352be90e23740047d85 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Fri, 6 Dec 2024 17:57:37 +0100 Subject: [PATCH 077/114] Override "__str__" function in subclasses of the class Module. --- scons/code_style/modules.py | 3 +++ scons/compilation/modules.py | 4 ++++ scons/dependencies/python/modules.py | 7 +++++-- scons/packaging/modules.py | 3 +++ scons/testing/cpp/modules.py | 3 +++ scons/testing/python/modules.py | 3 +++ 6 files changed, 21 insertions(+), 2 deletions(-) diff --git a/scons/code_style/modules.py b/scons/code_style/modules.py index 9931f678a..450d4418a 100644 --- a/scons/code_style/modules.py +++ b/scons/code_style/modules.py @@ -49,3 +49,6 @@ def find_source_files(self) -> List[str]: :return: A list that contains the paths of the source files that have been found """ return self.source_file_search.filter_by_file_type(self.file_type).list(self.root_directory) + + def __str__(self) -> str: + return 'CodeModule {file_type="' + str(self.file_type) + '", root_directory="' + self.root_directory + '"}' diff --git a/scons/compilation/modules.py b/scons/compilation/modules.py index cf32e05ab..4553455e9 100644 --- a/scons/compilation/modules.py +++ b/scons/compilation/modules.py @@ -72,3 +72,7 @@ def find_installed_files(self) -> List[str]: .list(self.install_directory) return [] + + def __str__(self) -> str: + return 'CompilationModule {file_type="' + str( + self.file_type) + '", root_directory=' + self.root_directory + '"}' diff --git a/scons/dependencies/python/modules.py b/scons/dependencies/python/modules.py index c5bb39676..02d98a5a2 100644 --- a/scons/dependencies/python/modules.py +++ b/scons/dependencies/python/modules.py @@ -14,8 +14,8 @@ class DependencyType(Enum): """ The type of the Python dependencies. """ - BUILD_TIME = auto() - RUNTIME = auto() + BUILD_TIME = 'build-time' + RUNTIME = 'runtime' class PythonDependencyModule(Module): @@ -59,3 +59,6 @@ def find_requirements_files(self) -> List[str]: :return: A list that contains the paths of the requirements files that have been found """ return self.requirements_file_search.filter_by_name('requirements.txt').list(self.root_directory) + + def __str__(self) -> str: + return 'PythonDependencyModule {dependency_type="' + self.dependency_type.value + '", root_directory="' + self.root_directory + '"}' diff --git a/scons/packaging/modules.py b/scons/packaging/modules.py index eb3c0e6a2..2d4fe43f9 100644 --- a/scons/packaging/modules.py +++ b/scons/packaging/modules.py @@ -52,3 +52,6 @@ def find_wheels(self) -> List[str]: :return: A list that contains the paths to the wheel packages """ return FileSearch().filter_by_suffix('whl').list(self.wheel_directory) + + def __str__(self) -> str: + return 'PythonPackageModule {root_directory="' + self.root_directory + '"}' diff --git a/scons/testing/cpp/modules.py b/scons/testing/cpp/modules.py index fa7e0f9b8..b21c4f632 100644 --- a/scons/testing/cpp/modules.py +++ b/scons/testing/cpp/modules.py @@ -36,3 +36,6 @@ def build_directory(self) -> str: The path to the directory, where build files are stored. """ return path.join(self.root_directory, self.build_directory_name) + + def __str__(self) -> str: + return 'CppTestModule {root_directory="' + self.root_directory + '"}' diff --git a/scons/testing/python/modules.py b/scons/testing/python/modules.py index 75f63145d..a0e58c94e 100644 --- a/scons/testing/python/modules.py +++ b/scons/testing/python/modules.py @@ -55,3 +55,6 @@ def find_test_directories(self) -> List[str]: .filter_by_substrings(starts_with='test_') \ .filter_by_file_type(FileType.python()) \ .list(self.root_directory) + + def __str__(self) -> str: + return 'PythonTestModule {root_directory="' + self.root_directory + '"}' From 57c84cfc75925af78ada83862c1ed4d92a4f2403 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Fri, 6 Dec 2024 17:59:23 +0100 Subject: [PATCH 078/114] Override "__str__" function in subclasses of the class Target. --- scons/util/targets.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scons/util/targets.py b/scons/util/targets.py index 7a3d20684..de71b3135 100644 --- a/scons/util/targets.py +++ b/scons/util/targets.py @@ -11,6 +11,7 @@ from typing import Any, Callable, Dict, Iterable, List, Optional, Set from uuid import uuid4 +from util.format import format_iterable from util.modules import ModuleRegistry from util.units import BuildUnit @@ -36,6 +37,9 @@ class Dependency: target_name: str clean_dependency: bool + def __str__(self) -> str: + return self.target_name + def __eq__(self, other: 'Target.Dependency') -> bool: return self.target_name == other.target_name @@ -187,6 +191,14 @@ def get_clean_files(self, module_registry: ModuleRegistry) -> Optional[List[str] """ return None + def __str__(self) -> str: + result = type(self).__name__ + '{name="' + self.name + '"' + + if self.dependencies: + result += ', dependencies={' + format_iterable(self.dependencies, delimiter='"') + '}' + + return result + '}' + class BuildTarget(Target): """ @@ -387,6 +399,7 @@ def register(self, environment: Environment, module_registry: ModuleRegistry) -> ) + class TargetBuilder: """ A builder that allows to configure and create multiple targets. From c019318d3499c1ecb4dcd5272449da428305ce2e Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Mon, 9 Dec 2024 00:20:41 +0100 Subject: [PATCH 079/114] Add utility function "delete_files". --- scons/util/io.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/scons/util/io.py b/scons/util/io.py index babc12c06..2d6d3f6c3 100644 --- a/scons/util/io.py +++ b/scons/util/io.py @@ -4,9 +4,12 @@ Provides utility functions for reading and writing files. """ from functools import cached_property -from os import path +from os import path, remove +from shutil import rmtree from typing import List +from util.log import Log + ENCODING_UTF8 = 'utf-8' @@ -28,6 +31,23 @@ def write_file(file: str): return open(file, mode='w', encoding=ENCODING_UTF8) +def delete_files(*files: str, accept_missing: bool = True): + """ + Deletes one or several files or directories. + + :param files: The files or directories to be deleted + :param accept_missing: True, if no error should be raised if the file is missing, False otherwise + """ + for file in files: + if path.isdir(file): + Log.verbose('Deleting directory "%s"...', file) + rmtree(file) + else: + if not accept_missing or path.isfile(file): + Log.verbose('Deleting file "%s"...', file) + remove(file) + + class TextFile: """ Allows to read and write the content of a text file. @@ -70,5 +90,11 @@ def clear(self): Log.info('Clearing file "%s"...', self.file) self.write_lines('') + def delete(self): + """ + Deletes the text file. + """ + delete_files(self.file, accept_missing=self.accept_missing) + def __str__(self) -> str: return self.file From 5bada72e5fae42b2f9e7c632616d6d9db9de9521 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Mon, 9 Dec 2024 02:40:26 +0100 Subject: [PATCH 080/114] Replace scons with custom implementation. --- .github/workflows/test_build.yml | 2 +- .github/workflows/test_changelog.yml | 2 +- .github/workflows/test_doc.yml | 2 +- .github/workflows/test_format.yml | 2 +- .github/workflows/test_publish.yml | 2 +- .gitignore | 2 +- build | 7 +- build.bat | 9 +- .../code_style/__init__.py | 0 .../code_style/cpp/.clang-format | 0 .../code_style/cpp/__init__.py | 0 .../code_style/cpp/clang_format.py | 0 .../code_style/cpp/cpplint.py | 0 .../code_style/cpp/requirements.txt | 0 .../code_style/cpp/targets.py | 0 .../code_style/markdown/__init__.py | 0 .../code_style/markdown/mdformat.py | 0 .../code_style/markdown/requirements.txt | 0 .../code_style/markdown/targets.py | 0 {scons => build_system}/code_style/modules.py | 0 .../code_style/python/.isort.cfg | 4 +- .../code_style/python/.pylintrc | 0 .../code_style/python/.style.yapf | 0 .../code_style/python/__init__.py | 0 .../code_style/python/isort.py | 0 .../code_style/python/pylint.py | 0 .../code_style/python/requirements.txt | 0 .../code_style/python/targets.py | 0 .../code_style/python/yapf.py | 0 .../code_style/yaml/.yamlfix.toml | 0 .../code_style/yaml/__init__.py | 0 .../code_style/yaml/requirements.txt | 0 .../code_style/yaml/targets.py | 0 .../code_style/yaml/yamlfix.py | 0 .../compilation/__init__.py | 0 .../compilation/build_options.py | 0 .../compilation/cpp/__init__.py | 0 .../compilation/cpp/targets.py | 0 .../compilation/cython/__init__.py | 0 .../compilation/cython/requirements.txt | 0 .../compilation/cython/targets.py | 0 {scons => build_system}/compilation/meson.py | 0 .../compilation/modules.py | 0 .../compilation/requirements.txt | 0 .../dependencies/__init__.py | 0 .../dependencies/github/__init__.py | 0 .../dependencies/github/actions.py | 0 .../dependencies/github/pygithub.py | 0 .../dependencies/github/pyyaml.py | 0 .../dependencies/github/requirements.txt | 0 .../dependencies/github/targets.py | 0 .../dependencies/python/__init__.py | 0 .../dependencies/python/modules.py | 0 .../dependencies/python/pip.py | 0 .../dependencies/python/targets.py | 0 .../dependencies/requirements.txt | 0 {scons => build_system}/dependencies/table.py | 0 {scons => build_system}/documentation.py | 0 build_system/main.py | 117 +++ {scons => build_system}/packaging/__init__.py | 0 {scons => build_system}/packaging/build.py | 0 {scons => build_system}/packaging/modules.py | 0 {scons => build_system}/packaging/pip.py | 0 .../packaging/requirements.txt | 0 {scons => build_system}/packaging/targets.py | 0 {scons => build_system}/testing/__init__.py | 0 .../testing/cpp/__init__.py | 0 {scons => build_system}/testing/cpp/meson.py | 0 .../testing/cpp/modules.py | 0 .../testing/cpp/targets.py | 0 {scons => build_system}/testing/modules.py | 0 .../testing/python/__init__.py | 0 .../testing/python/modules.py | 0 .../testing/python/requirements.txt | 0 .../testing/python/targets.py | 0 .../testing/python/unittest.py | 0 {scons => build_system}/util/__init__.py | 0 {scons => build_system}/util/cmd.py | 0 {scons => build_system}/util/env.py | 0 {scons => build_system}/util/files.py | 0 {scons => build_system}/util/format.py | 0 {scons => build_system}/util/io.py | 0 {scons => build_system}/util/log.py | 0 {scons => build_system}/util/modules.py | 0 {scons => build_system}/util/paths.py | 2 +- {scons => build_system}/util/pip.py | 0 {scons => build_system}/util/reflection.py | 0 {scons => build_system}/util/run.py | 0 build_system/util/targets.py | 787 ++++++++++++++++++ {scons => build_system}/util/units.py | 0 {scons => build_system}/util/venv.py | 0 .../versioning/__init__.py | 0 .../versioning/changelog.py | 0 .../versioning/versioning.py | 0 doc/developer_guide/coding_standards.md | 2 +- doc/developer_guide/compilation.md | 2 +- scons/modules_old.py | 514 ------------ scons/sconstruct.py | 143 ---- scons/util/requirements.txt | 1 - scons/util/targets.py | 535 ------------ 100 files changed, 922 insertions(+), 1213 deletions(-) rename {scons => build_system}/code_style/__init__.py (100%) rename {scons => build_system}/code_style/cpp/.clang-format (100%) rename {scons => build_system}/code_style/cpp/__init__.py (100%) rename {scons => build_system}/code_style/cpp/clang_format.py (100%) rename {scons => build_system}/code_style/cpp/cpplint.py (100%) rename {scons => build_system}/code_style/cpp/requirements.txt (100%) rename {scons => build_system}/code_style/cpp/targets.py (100%) rename {scons => build_system}/code_style/markdown/__init__.py (100%) rename {scons => build_system}/code_style/markdown/mdformat.py (100%) rename {scons => build_system}/code_style/markdown/requirements.txt (100%) rename {scons => build_system}/code_style/markdown/targets.py (100%) rename {scons => build_system}/code_style/modules.py (100%) rename {scons => build_system}/code_style/python/.isort.cfg (80%) rename {scons => build_system}/code_style/python/.pylintrc (100%) rename {scons => build_system}/code_style/python/.style.yapf (100%) rename {scons => build_system}/code_style/python/__init__.py (100%) rename {scons => build_system}/code_style/python/isort.py (100%) rename {scons => build_system}/code_style/python/pylint.py (100%) rename {scons => build_system}/code_style/python/requirements.txt (100%) rename {scons => build_system}/code_style/python/targets.py (100%) rename {scons => build_system}/code_style/python/yapf.py (100%) rename {scons => build_system}/code_style/yaml/.yamlfix.toml (100%) rename {scons => build_system}/code_style/yaml/__init__.py (100%) rename {scons => build_system}/code_style/yaml/requirements.txt (100%) rename {scons => build_system}/code_style/yaml/targets.py (100%) rename {scons => build_system}/code_style/yaml/yamlfix.py (100%) rename {scons => build_system}/compilation/__init__.py (100%) rename {scons => build_system}/compilation/build_options.py (100%) rename {scons => build_system}/compilation/cpp/__init__.py (100%) rename {scons => build_system}/compilation/cpp/targets.py (100%) rename {scons => build_system}/compilation/cython/__init__.py (100%) rename {scons => build_system}/compilation/cython/requirements.txt (100%) rename {scons => build_system}/compilation/cython/targets.py (100%) rename {scons => build_system}/compilation/meson.py (100%) rename {scons => build_system}/compilation/modules.py (100%) rename {scons => build_system}/compilation/requirements.txt (100%) rename {scons => build_system}/dependencies/__init__.py (100%) rename {scons => build_system}/dependencies/github/__init__.py (100%) rename {scons => build_system}/dependencies/github/actions.py (100%) rename {scons => build_system}/dependencies/github/pygithub.py (100%) rename {scons => build_system}/dependencies/github/pyyaml.py (100%) rename {scons => build_system}/dependencies/github/requirements.txt (100%) rename {scons => build_system}/dependencies/github/targets.py (100%) rename {scons => build_system}/dependencies/python/__init__.py (100%) rename {scons => build_system}/dependencies/python/modules.py (100%) rename {scons => build_system}/dependencies/python/pip.py (100%) rename {scons => build_system}/dependencies/python/targets.py (100%) rename {scons => build_system}/dependencies/requirements.txt (100%) rename {scons => build_system}/dependencies/table.py (100%) rename {scons => build_system}/documentation.py (100%) create mode 100644 build_system/main.py rename {scons => build_system}/packaging/__init__.py (100%) rename {scons => build_system}/packaging/build.py (100%) rename {scons => build_system}/packaging/modules.py (100%) rename {scons => build_system}/packaging/pip.py (100%) rename {scons => build_system}/packaging/requirements.txt (100%) rename {scons => build_system}/packaging/targets.py (100%) rename {scons => build_system}/testing/__init__.py (100%) rename {scons => build_system}/testing/cpp/__init__.py (100%) rename {scons => build_system}/testing/cpp/meson.py (100%) rename {scons => build_system}/testing/cpp/modules.py (100%) rename {scons => build_system}/testing/cpp/targets.py (100%) rename {scons => build_system}/testing/modules.py (100%) rename {scons => build_system}/testing/python/__init__.py (100%) rename {scons => build_system}/testing/python/modules.py (100%) rename {scons => build_system}/testing/python/requirements.txt (100%) rename {scons => build_system}/testing/python/targets.py (100%) rename {scons => build_system}/testing/python/unittest.py (100%) rename {scons => build_system}/util/__init__.py (100%) rename {scons => build_system}/util/cmd.py (100%) rename {scons => build_system}/util/env.py (100%) rename {scons => build_system}/util/files.py (100%) rename {scons => build_system}/util/format.py (100%) rename {scons => build_system}/util/io.py (100%) rename {scons => build_system}/util/log.py (100%) rename {scons => build_system}/util/modules.py (100%) rename {scons => build_system}/util/paths.py (99%) rename {scons => build_system}/util/pip.py (100%) rename {scons => build_system}/util/reflection.py (100%) rename {scons => build_system}/util/run.py (100%) create mode 100644 build_system/util/targets.py rename {scons => build_system}/util/units.py (100%) rename {scons => build_system}/util/venv.py (100%) rename {scons => build_system}/versioning/__init__.py (100%) rename {scons => build_system}/versioning/changelog.py (100%) rename {scons => build_system}/versioning/versioning.py (100%) delete mode 100644 scons/modules_old.py delete mode 100644 scons/sconstruct.py delete mode 100644 scons/util/requirements.txt delete mode 100644 scons/util/targets.py diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index c136d4e47..7bd3fd7db 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -35,7 +35,7 @@ jobs: - '.github/workflows/test_build.yml' - 'build' - 'build.bat' - - 'scons/**' + - 'build_system/**' cpp: &cpp - *build_files - 'cpp/**/include/**' diff --git a/.github/workflows/test_changelog.yml b/.github/workflows/test_changelog.yml index 2679713e2..869fe882c 100644 --- a/.github/workflows/test_changelog.yml +++ b/.github/workflows/test_changelog.yml @@ -33,7 +33,7 @@ jobs: - '.github/workflows/test_changelog.yml' - 'build' - 'build.bat' - - 'scons/**' + - 'build_system/**' bugfix: - *build_files - '.changelog-bugfix.md' diff --git a/.github/workflows/test_doc.yml b/.github/workflows/test_doc.yml index 84836393d..021a893e5 100644 --- a/.github/workflows/test_doc.yml +++ b/.github/workflows/test_doc.yml @@ -34,7 +34,7 @@ jobs: - '.github/workflows/test_doc.yml' - 'build' - 'build.bat' - - 'scons/**' + - 'build_system/**' cpp: &cpp - *build_files - 'cpp/**/include/**' diff --git a/.github/workflows/test_format.yml b/.github/workflows/test_format.yml index 936dd7825..bae279776 100644 --- a/.github/workflows/test_format.yml +++ b/.github/workflows/test_format.yml @@ -33,7 +33,7 @@ jobs: - '.github/workflows/test_format.yml' - 'build' - 'build.bat' - - 'scons/**' + - 'build_system/**' cpp: - *build_files - '**/*.hpp' diff --git a/.github/workflows/test_publish.yml b/.github/workflows/test_publish.yml index 5d64d7b06..2689105e5 100644 --- a/.github/workflows/test_publish.yml +++ b/.github/workflows/test_publish.yml @@ -36,7 +36,7 @@ jobs: - '.github/workflows/template_publish_pure.yml' - 'build' - 'build.bat' - - 'scons/**' + - 'build_system/**' - name: Read Python version uses: juliangruber/read-file-action@v1 id: python_version diff --git a/.gitignore b/.gitignore index 5cded9341..b9b950e53 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ # Build files __pycache__/ -scons/build/ +build_system/build/ python/**/build/ python/**/dist/ python/**/*egg-info/ diff --git a/build b/build index f5c4091ff..d42154e36 100755 --- a/build +++ b/build @@ -1,7 +1,7 @@ #!/bin/sh VENV_DIR="venv" -SCONS_DIR="scons" +BUILD_SYSTEM_DIR="build_system" CLEAN=false set -e @@ -22,13 +22,12 @@ fi if [ -d "$VENV_DIR" ]; then . $VENV_DIR/bin/activate - python3 -c "import sys; sys.path.append('$SCONS_DIR'); from util.pip import Pip; Pip.for_build_unit().install_packages('scons')" - scons --silent --file $SCONS_DIR/sconstruct.py $@ + python3 $BUILD_SYSTEM_DIR/main.py $@ deactivate fi if [ $CLEAN = true ] && [ -d $VENV_DIR ]; then echo "Removing virtual Python environment..." rm -rf $VENV_DIR - rm -rf $SCONS_DIR/build + rm -rf $BUILD_SYSTEM_DIR/build fi diff --git a/build.bat b/build.bat index d5f247836..425048708 100644 --- a/build.bat +++ b/build.bat @@ -1,7 +1,7 @@ @echo off set "VENV_DIR=venv" -set "SCONS_DIR=scons" +set "BUILD_SYSTEM_DIR=build_system" set "CLEAN=false" if not "%1"=="" if "%2"=="" ( @@ -20,8 +20,7 @@ if not exist "%VENV_DIR%" ( if exist "%VENV_DIR%" ( call %VENV_DIR%\Scripts\activate || exit - .\%VENV_DIR%\Scripts\python -c "import sys;sys.path.append('%SCONS_DIR%');from util.pip import Pip;Pip.for_build_unit().install_packages('scons')" || exit - .\%VENV_DIR%\Scripts\python -m SCons --silent --file %SCONS_DIR%\sconstruct.py %* || exit + .\%VENV_DIR%\Scripts\python %BUILD_SYSTEM_DIR%\main.py %* || exit call deactivate || exit ) @@ -29,7 +28,7 @@ if "%CLEAN%"=="true" if exist "%VENV_DIR%" ( echo Removing virtual Python environment... rd /s /q "%VENV_DIR%" || exit - if exist "%SCONS_DIR%\build" ( - rd /s /q "%SCONS_DIR%\build" || exit + if exist "%BUILD_SYSTEM_DIR%\build" ( + rd /s /q "%BUILD_SYSTEM_DIR%\build" || exit ) ) diff --git a/scons/code_style/__init__.py b/build_system/code_style/__init__.py similarity index 100% rename from scons/code_style/__init__.py rename to build_system/code_style/__init__.py diff --git a/scons/code_style/cpp/.clang-format b/build_system/code_style/cpp/.clang-format similarity index 100% rename from scons/code_style/cpp/.clang-format rename to build_system/code_style/cpp/.clang-format diff --git a/scons/code_style/cpp/__init__.py b/build_system/code_style/cpp/__init__.py similarity index 100% rename from scons/code_style/cpp/__init__.py rename to build_system/code_style/cpp/__init__.py diff --git a/scons/code_style/cpp/clang_format.py b/build_system/code_style/cpp/clang_format.py similarity index 100% rename from scons/code_style/cpp/clang_format.py rename to build_system/code_style/cpp/clang_format.py diff --git a/scons/code_style/cpp/cpplint.py b/build_system/code_style/cpp/cpplint.py similarity index 100% rename from scons/code_style/cpp/cpplint.py rename to build_system/code_style/cpp/cpplint.py diff --git a/scons/code_style/cpp/requirements.txt b/build_system/code_style/cpp/requirements.txt similarity index 100% rename from scons/code_style/cpp/requirements.txt rename to build_system/code_style/cpp/requirements.txt diff --git a/scons/code_style/cpp/targets.py b/build_system/code_style/cpp/targets.py similarity index 100% rename from scons/code_style/cpp/targets.py rename to build_system/code_style/cpp/targets.py diff --git a/scons/code_style/markdown/__init__.py b/build_system/code_style/markdown/__init__.py similarity index 100% rename from scons/code_style/markdown/__init__.py rename to build_system/code_style/markdown/__init__.py diff --git a/scons/code_style/markdown/mdformat.py b/build_system/code_style/markdown/mdformat.py similarity index 100% rename from scons/code_style/markdown/mdformat.py rename to build_system/code_style/markdown/mdformat.py diff --git a/scons/code_style/markdown/requirements.txt b/build_system/code_style/markdown/requirements.txt similarity index 100% rename from scons/code_style/markdown/requirements.txt rename to build_system/code_style/markdown/requirements.txt diff --git a/scons/code_style/markdown/targets.py b/build_system/code_style/markdown/targets.py similarity index 100% rename from scons/code_style/markdown/targets.py rename to build_system/code_style/markdown/targets.py diff --git a/scons/code_style/modules.py b/build_system/code_style/modules.py similarity index 100% rename from scons/code_style/modules.py rename to build_system/code_style/modules.py diff --git a/scons/code_style/python/.isort.cfg b/build_system/code_style/python/.isort.cfg similarity index 80% rename from scons/code_style/python/.isort.cfg rename to build_system/code_style/python/.isort.cfg index 08d9adbfb..0dbfa6c52 100644 --- a/scons/code_style/python/.isort.cfg +++ b/build_system/code_style/python/.isort.cfg @@ -2,8 +2,8 @@ line_length=120 group_by_package=true known_first_party=mlrl -known_third_party=sklearn,scipy,numpy,tabulate,arff,SCons -forced_separate=mlrl.common,mlrl.boosting,mlrl.seco,mlrl.testbed,SCons +known_third_party=sklearn,scipy,numpy,tabulate,arff +forced_separate=mlrl.common,mlrl.boosting,mlrl.seco,mlrl.testbed lines_between_types=1 order_by_type=true multi_line_output=2 diff --git a/scons/code_style/python/.pylintrc b/build_system/code_style/python/.pylintrc similarity index 100% rename from scons/code_style/python/.pylintrc rename to build_system/code_style/python/.pylintrc diff --git a/scons/code_style/python/.style.yapf b/build_system/code_style/python/.style.yapf similarity index 100% rename from scons/code_style/python/.style.yapf rename to build_system/code_style/python/.style.yapf diff --git a/scons/code_style/python/__init__.py b/build_system/code_style/python/__init__.py similarity index 100% rename from scons/code_style/python/__init__.py rename to build_system/code_style/python/__init__.py diff --git a/scons/code_style/python/isort.py b/build_system/code_style/python/isort.py similarity index 100% rename from scons/code_style/python/isort.py rename to build_system/code_style/python/isort.py diff --git a/scons/code_style/python/pylint.py b/build_system/code_style/python/pylint.py similarity index 100% rename from scons/code_style/python/pylint.py rename to build_system/code_style/python/pylint.py diff --git a/scons/code_style/python/requirements.txt b/build_system/code_style/python/requirements.txt similarity index 100% rename from scons/code_style/python/requirements.txt rename to build_system/code_style/python/requirements.txt diff --git a/scons/code_style/python/targets.py b/build_system/code_style/python/targets.py similarity index 100% rename from scons/code_style/python/targets.py rename to build_system/code_style/python/targets.py diff --git a/scons/code_style/python/yapf.py b/build_system/code_style/python/yapf.py similarity index 100% rename from scons/code_style/python/yapf.py rename to build_system/code_style/python/yapf.py diff --git a/scons/code_style/yaml/.yamlfix.toml b/build_system/code_style/yaml/.yamlfix.toml similarity index 100% rename from scons/code_style/yaml/.yamlfix.toml rename to build_system/code_style/yaml/.yamlfix.toml diff --git a/scons/code_style/yaml/__init__.py b/build_system/code_style/yaml/__init__.py similarity index 100% rename from scons/code_style/yaml/__init__.py rename to build_system/code_style/yaml/__init__.py diff --git a/scons/code_style/yaml/requirements.txt b/build_system/code_style/yaml/requirements.txt similarity index 100% rename from scons/code_style/yaml/requirements.txt rename to build_system/code_style/yaml/requirements.txt diff --git a/scons/code_style/yaml/targets.py b/build_system/code_style/yaml/targets.py similarity index 100% rename from scons/code_style/yaml/targets.py rename to build_system/code_style/yaml/targets.py diff --git a/scons/code_style/yaml/yamlfix.py b/build_system/code_style/yaml/yamlfix.py similarity index 100% rename from scons/code_style/yaml/yamlfix.py rename to build_system/code_style/yaml/yamlfix.py diff --git a/scons/compilation/__init__.py b/build_system/compilation/__init__.py similarity index 100% rename from scons/compilation/__init__.py rename to build_system/compilation/__init__.py diff --git a/scons/compilation/build_options.py b/build_system/compilation/build_options.py similarity index 100% rename from scons/compilation/build_options.py rename to build_system/compilation/build_options.py diff --git a/scons/compilation/cpp/__init__.py b/build_system/compilation/cpp/__init__.py similarity index 100% rename from scons/compilation/cpp/__init__.py rename to build_system/compilation/cpp/__init__.py diff --git a/scons/compilation/cpp/targets.py b/build_system/compilation/cpp/targets.py similarity index 100% rename from scons/compilation/cpp/targets.py rename to build_system/compilation/cpp/targets.py diff --git a/scons/compilation/cython/__init__.py b/build_system/compilation/cython/__init__.py similarity index 100% rename from scons/compilation/cython/__init__.py rename to build_system/compilation/cython/__init__.py diff --git a/scons/compilation/cython/requirements.txt b/build_system/compilation/cython/requirements.txt similarity index 100% rename from scons/compilation/cython/requirements.txt rename to build_system/compilation/cython/requirements.txt diff --git a/scons/compilation/cython/targets.py b/build_system/compilation/cython/targets.py similarity index 100% rename from scons/compilation/cython/targets.py rename to build_system/compilation/cython/targets.py diff --git a/scons/compilation/meson.py b/build_system/compilation/meson.py similarity index 100% rename from scons/compilation/meson.py rename to build_system/compilation/meson.py diff --git a/scons/compilation/modules.py b/build_system/compilation/modules.py similarity index 100% rename from scons/compilation/modules.py rename to build_system/compilation/modules.py diff --git a/scons/compilation/requirements.txt b/build_system/compilation/requirements.txt similarity index 100% rename from scons/compilation/requirements.txt rename to build_system/compilation/requirements.txt diff --git a/scons/dependencies/__init__.py b/build_system/dependencies/__init__.py similarity index 100% rename from scons/dependencies/__init__.py rename to build_system/dependencies/__init__.py diff --git a/scons/dependencies/github/__init__.py b/build_system/dependencies/github/__init__.py similarity index 100% rename from scons/dependencies/github/__init__.py rename to build_system/dependencies/github/__init__.py diff --git a/scons/dependencies/github/actions.py b/build_system/dependencies/github/actions.py similarity index 100% rename from scons/dependencies/github/actions.py rename to build_system/dependencies/github/actions.py diff --git a/scons/dependencies/github/pygithub.py b/build_system/dependencies/github/pygithub.py similarity index 100% rename from scons/dependencies/github/pygithub.py rename to build_system/dependencies/github/pygithub.py diff --git a/scons/dependencies/github/pyyaml.py b/build_system/dependencies/github/pyyaml.py similarity index 100% rename from scons/dependencies/github/pyyaml.py rename to build_system/dependencies/github/pyyaml.py diff --git a/scons/dependencies/github/requirements.txt b/build_system/dependencies/github/requirements.txt similarity index 100% rename from scons/dependencies/github/requirements.txt rename to build_system/dependencies/github/requirements.txt diff --git a/scons/dependencies/github/targets.py b/build_system/dependencies/github/targets.py similarity index 100% rename from scons/dependencies/github/targets.py rename to build_system/dependencies/github/targets.py diff --git a/scons/dependencies/python/__init__.py b/build_system/dependencies/python/__init__.py similarity index 100% rename from scons/dependencies/python/__init__.py rename to build_system/dependencies/python/__init__.py diff --git a/scons/dependencies/python/modules.py b/build_system/dependencies/python/modules.py similarity index 100% rename from scons/dependencies/python/modules.py rename to build_system/dependencies/python/modules.py diff --git a/scons/dependencies/python/pip.py b/build_system/dependencies/python/pip.py similarity index 100% rename from scons/dependencies/python/pip.py rename to build_system/dependencies/python/pip.py diff --git a/scons/dependencies/python/targets.py b/build_system/dependencies/python/targets.py similarity index 100% rename from scons/dependencies/python/targets.py rename to build_system/dependencies/python/targets.py diff --git a/scons/dependencies/requirements.txt b/build_system/dependencies/requirements.txt similarity index 100% rename from scons/dependencies/requirements.txt rename to build_system/dependencies/requirements.txt diff --git a/scons/dependencies/table.py b/build_system/dependencies/table.py similarity index 100% rename from scons/dependencies/table.py rename to build_system/dependencies/table.py diff --git a/scons/documentation.py b/build_system/documentation.py similarity index 100% rename from scons/documentation.py rename to build_system/documentation.py diff --git a/build_system/main.py b/build_system/main.py new file mode 100644 index 000000000..e830face8 --- /dev/null +++ b/build_system/main.py @@ -0,0 +1,117 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Initializes the build system and runs targets specified via command line arguments. +""" +from argparse import ArgumentParser +from typing import List + +from util.files import FileSearch +from util.format import format_iterable +from util.log import Log +from util.modules import Module, ModuleRegistry +from util.paths import Project +from util.reflection import import_source_file +from util.targets import DependencyGraph, Target, TargetRegistry + + +def __parse_command_line_arguments(): + parser = ArgumentParser(description='The build system of the project "MLRL-Boomer"') + parser.add_argument('--verbose', action='store_true', help='Enables verbose logging.') + parser.add_argument('--clean', action='store_true', help='Cleans the specified targets.') + parser.add_argument('targets', nargs='*') + return parser.parse_args() + + +def __configure_log(args): + log_level = Log.Level.VERBOSE if args.verbose else Log.Level.INFO + Log.configure(log_level) + + +def __find_init_files() -> List[str]: + return FileSearch() \ + .set_recursive(True) \ + .filter_by_name('__init__.py') \ + .list(Project.BuildSystem.root_directory) + + +def __register_modules(init_files: List[str]) -> ModuleRegistry: + Log.verbose('Registering modules...') + module_registry = ModuleRegistry() + num_modules = 0 + + for init_file in init_files: + modules = [ + module for module in getattr(import_source_file(init_file), 'MODULES', []) if isinstance(module, Module) + ] + + if modules: + Log.verbose('Registering %s modules defined in file "%s":\n', str(len(modules)), init_file) + + for module in modules: + Log.verbose(' - %s', str(module)) + module_registry.register(module) + + Log.verbose('') + num_modules += len(modules) + + Log.verbose('Successfully registered %s modules.\n', str(num_modules)) + return module_registry + + +def __register_targets(init_files: List[str]) -> TargetRegistry: + Log.verbose('Registering targets...') + target_registry = TargetRegistry() + num_targets = 0 + + for init_file in init_files: + targets = [ + target for target in getattr(import_source_file(init_file), 'TARGETS', []) if isinstance(target, Target) + ] + + if targets: + Log.verbose('Registering %s targets defined in file "%s":\n', str(len(targets)), init_file) + + for target in targets: + Log.verbose(' - %s', str(target)) + target_registry.register(target) + + Log.verbose('') + num_targets += len(targets) + + Log.verbose('Successfully registered %s targets.\n', str(num_targets)) + return target_registry + + +def __create_dependency_graph(target_registry: TargetRegistry, args) -> DependencyGraph: + targets = args.targets + clean = args.clean + Log.verbose('Creating dependency graph for %s targets [%s]...', 'cleaning' if clean else 'running', + format_iterable(targets)) + dependency_graph = target_registry.create_dependency_graph(*targets, clean=clean) + Log.verbose('Successfully created dependency graph:\n\n%s\n', str(dependency_graph)) + return dependency_graph + + +def __execute_dependency_graph(dependency_graph: DependencyGraph, module_registry: ModuleRegistry): + Log.verbose('Executing dependency graph...') + dependency_graph.execute(module_registry) + Log.verbose('Successfully executed dependency graph.') + + +def main(): + """ + The main function to be executed when the build system is invoked. + """ + args = __parse_command_line_arguments() + __configure_log(args) + + init_files = __find_init_files() + module_registry = __register_modules(init_files) + target_registry = __register_targets(init_files) + dependency_graph = __create_dependency_graph(target_registry, args) + __execute_dependency_graph(dependency_graph, module_registry) + + +if __name__ == '__main__': + main() diff --git a/scons/packaging/__init__.py b/build_system/packaging/__init__.py similarity index 100% rename from scons/packaging/__init__.py rename to build_system/packaging/__init__.py diff --git a/scons/packaging/build.py b/build_system/packaging/build.py similarity index 100% rename from scons/packaging/build.py rename to build_system/packaging/build.py diff --git a/scons/packaging/modules.py b/build_system/packaging/modules.py similarity index 100% rename from scons/packaging/modules.py rename to build_system/packaging/modules.py diff --git a/scons/packaging/pip.py b/build_system/packaging/pip.py similarity index 100% rename from scons/packaging/pip.py rename to build_system/packaging/pip.py diff --git a/scons/packaging/requirements.txt b/build_system/packaging/requirements.txt similarity index 100% rename from scons/packaging/requirements.txt rename to build_system/packaging/requirements.txt diff --git a/scons/packaging/targets.py b/build_system/packaging/targets.py similarity index 100% rename from scons/packaging/targets.py rename to build_system/packaging/targets.py diff --git a/scons/testing/__init__.py b/build_system/testing/__init__.py similarity index 100% rename from scons/testing/__init__.py rename to build_system/testing/__init__.py diff --git a/scons/testing/cpp/__init__.py b/build_system/testing/cpp/__init__.py similarity index 100% rename from scons/testing/cpp/__init__.py rename to build_system/testing/cpp/__init__.py diff --git a/scons/testing/cpp/meson.py b/build_system/testing/cpp/meson.py similarity index 100% rename from scons/testing/cpp/meson.py rename to build_system/testing/cpp/meson.py diff --git a/scons/testing/cpp/modules.py b/build_system/testing/cpp/modules.py similarity index 100% rename from scons/testing/cpp/modules.py rename to build_system/testing/cpp/modules.py diff --git a/scons/testing/cpp/targets.py b/build_system/testing/cpp/targets.py similarity index 100% rename from scons/testing/cpp/targets.py rename to build_system/testing/cpp/targets.py diff --git a/scons/testing/modules.py b/build_system/testing/modules.py similarity index 100% rename from scons/testing/modules.py rename to build_system/testing/modules.py diff --git a/scons/testing/python/__init__.py b/build_system/testing/python/__init__.py similarity index 100% rename from scons/testing/python/__init__.py rename to build_system/testing/python/__init__.py diff --git a/scons/testing/python/modules.py b/build_system/testing/python/modules.py similarity index 100% rename from scons/testing/python/modules.py rename to build_system/testing/python/modules.py diff --git a/scons/testing/python/requirements.txt b/build_system/testing/python/requirements.txt similarity index 100% rename from scons/testing/python/requirements.txt rename to build_system/testing/python/requirements.txt diff --git a/scons/testing/python/targets.py b/build_system/testing/python/targets.py similarity index 100% rename from scons/testing/python/targets.py rename to build_system/testing/python/targets.py diff --git a/scons/testing/python/unittest.py b/build_system/testing/python/unittest.py similarity index 100% rename from scons/testing/python/unittest.py rename to build_system/testing/python/unittest.py diff --git a/scons/util/__init__.py b/build_system/util/__init__.py similarity index 100% rename from scons/util/__init__.py rename to build_system/util/__init__.py diff --git a/scons/util/cmd.py b/build_system/util/cmd.py similarity index 100% rename from scons/util/cmd.py rename to build_system/util/cmd.py diff --git a/scons/util/env.py b/build_system/util/env.py similarity index 100% rename from scons/util/env.py rename to build_system/util/env.py diff --git a/scons/util/files.py b/build_system/util/files.py similarity index 100% rename from scons/util/files.py rename to build_system/util/files.py diff --git a/scons/util/format.py b/build_system/util/format.py similarity index 100% rename from scons/util/format.py rename to build_system/util/format.py diff --git a/scons/util/io.py b/build_system/util/io.py similarity index 100% rename from scons/util/io.py rename to build_system/util/io.py diff --git a/scons/util/log.py b/build_system/util/log.py similarity index 100% rename from scons/util/log.py rename to build_system/util/log.py diff --git a/scons/util/modules.py b/build_system/util/modules.py similarity index 100% rename from scons/util/modules.py rename to build_system/util/modules.py diff --git a/scons/util/paths.py b/build_system/util/paths.py similarity index 99% rename from scons/util/paths.py rename to build_system/util/paths.py index 718476345..7f3c75a29 100644 --- a/scons/util/paths.py +++ b/build_system/util/paths.py @@ -25,7 +25,7 @@ class BuildSystem: build_directory_name: The name of the build system's build directory """ - root_directory = 'scons' + root_directory = 'build_system' build_directory_name = 'build' diff --git a/scons/util/pip.py b/build_system/util/pip.py similarity index 100% rename from scons/util/pip.py rename to build_system/util/pip.py diff --git a/scons/util/reflection.py b/build_system/util/reflection.py similarity index 100% rename from scons/util/reflection.py rename to build_system/util/reflection.py diff --git a/scons/util/run.py b/build_system/util/run.py similarity index 100% rename from scons/util/run.py rename to build_system/util/run.py diff --git a/build_system/util/targets.py b/build_system/util/targets.py new file mode 100644 index 000000000..1b7d19ad0 --- /dev/null +++ b/build_system/util/targets.py @@ -0,0 +1,787 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides base classes for defining individual targets of the build process. +""" +from abc import ABC, abstractmethod +from dataclasses import dataclass +from functools import reduce +from os import path +from typing import Any, Callable, Dict, Iterable, List, Optional + +from util.format import format_iterable +from util.io import delete_files +from util.log import Log +from util.modules import ModuleRegistry +from util.units import BuildUnit + + +class Target(ABC): + """ + An abstract base class for all targets of the build system. + """ + + @dataclass + class Dependency: + """ + A single dependency of a parent target. + + Attributes: + target_name: The name of the target, the parent target depends on + clean_dependency: True, if the output files of the dependency should also be cleaned when cleaning the + output files of the parent target, False otherwise + """ + target_name: str + clean_dependency: bool = True + + def __str__(self) -> str: + return self.target_name + + def __eq__(self, other) -> bool: + return type(self) == type(other) and self.target_name == other.target_name + + class Builder(ABC): + """ + An abstract base class for all builders that allow to configure and create targets. + """ + + def __anonymous_target_name(self) -> str: + return 'anonymous-dependency-' + str(len(self.dependencies) + 1) + '-of-' + self.target_name + + def __init__(self, parent_builder: Any, target_name: str): + """ + :param parent_builder: The builder, this builder has been created from + :param target_name: The name of the target that is configured via the builder + """ + self.parent_builder = parent_builder + self.target_name = target_name + self.child_builders = [] + self.dependency_names = set() + self.dependencies = [] + + def depends_on(self, *target_names: str, clean_dependencies: bool = False) -> 'Target.Builder': + """ + Adds on or several targets, this target should depend on. + + :param target_names: The names of the targets, this target should depend on + :param clean_dependencies: True, if output files of the dependencies should also be cleaned when cleaning + the output files of this target, False otherwise + :return: The `Target.Builder` itself + """ + for target_name in target_names: + if not target_name in self.dependency_names: + self.dependency_names.add(target_name) + self.dependencies.append( + Target.Dependency(target_name=target_name, clean_dependency=clean_dependencies)) + + return self + + def depends_on_build_target(self, clean_dependency: bool = True) -> 'BuildTarget.Builder': + """ + Creates and returns a `BuildTarget.Builder` that allows to configure a build target, this target should + depend on. + + :param clean_dependency: True, if output files of the dependency should also be cleaned when cleaning the + output files of this target, False otherwise + :return: The `BuildTarget.Builder` that has been created + """ + target_name = self.__anonymous_target_name() + target_builder = BuildTarget.Builder(self, target_name) + self.child_builders.append(target_builder) + self.depends_on(target_name, clean_dependencies=clean_dependency) + return target_builder + + def depends_on_build_targets(self, + iterable: Iterable[Any], + target_configurator: Callable[[Any, 'BuildTarget.Builder'], None], + clean_dependencies: bool = True) -> 'Target.Builder': + """ + Configures multiple build targets, one for each value in a given `Iterable`, this target should depend on. + + :param iterable: An `Iterable` that provides access to the values for which dependencies should + be created + :param target_configurator: A function that accepts one value in the `Iterable` at a time, as well as a + `BuildTarget.Builder` for configuring the corresponding dependency + :param clean_dependencies: True, if output files of the dependencies should also be cleaned when cleaning + the output files of this target, False otherwise + :return: The `Target.Builder` itself + """ + for value in iterable: + target_builder = self.depends_on_build_target(clean_dependency=clean_dependencies) + target_configurator(value, target_builder) + + return self + + def depends_on_phony_target(self, clean_dependency: bool = True) -> 'PhonyTarget.Builder': + """ + Creates and returns a `PhonyTarget.Builder` that allows to configure a phony target, this target should + depend on. + + :param clean_dependency: True, if output files of the dependency should also be cleaned when cleaning the + output files of this target, False otherwise + :return: The `PhonyTarget.Builder` that has been created + """ + target_name = self.__anonymous_target_name() + target_builder = PhonyTarget.Builder(self, target_name) + self.child_builders.append(target_builder) + self.depends_on(target_name, clean_dependencies=clean_dependency) + return target_builder + + def depends_on_phony_targets(self, + iterable: Iterable[Any], + target_configurator: Callable[[Any, 'PhonyTarget.Builder'], None], + clean_dependencies: bool = True) -> 'Target.Builder': + """ + Configures multiple phony targets, one for each value in a given `Iterable`, this target should depend on. + + :param iterable: An `Iterable` that provides access to the values for which dependencies should + be created + :param target_configurator: A function that accepts one value in the `Iterable` at a time, as well as a + `BuildTarget.Builder` for configuring the corresponding dependency + :param clean_dependencies: True, if output files of the dependencies should also be cleaned when cleaning + the output files of this target, False otherwise + :return: The `Target.Builder` itself + """ + for value in iterable: + target_builder = self.depends_on_phony_target(clean_dependency=clean_dependencies) + target_configurator(value, target_builder) + + return self + + @abstractmethod + def _build(self, build_unit: BuildUnit) -> 'Target': + """ + Must be implemented by subclasses in order to create the target that has been configured via the builder. + + :param build_unit: The build unit, the target belongs to + :return: The target that has been created + """ + + def build(self, build_unit: BuildUnit) -> List['Target']: + """ + Creates and returns all targets that have been configured via the builder. + + :param build_unit: The build unit, the target belongs to + :return: The targets that have been created + """ + return [self._build(build_unit)] + reduce(lambda aggr, builder: aggr + builder.build(build_unit), + self.child_builders, []) + + def __init__(self, name: str, dependencies: List['Target.Dependency']): + """ + :param name: The name of the target + :param dependencies: A list that contains all dependencies of the target + """ + self.name = name + self.dependencies = dependencies + + @abstractmethod + def run(self, module_registry: ModuleRegistry): + """ + Must be implemented by subclasses in order to run this target. + + :param module_registry: The `ModuleRegistry` that can be used by the target for looking up modules + """ + + def clean(self, module_registry: ModuleRegistry): + """ + May be overridden by subclasses in order to clean up this target. + + :param module_registry: The `ModuleRegistry` that can be used by the target for looking up modules + """ + + def __str__(self) -> str: + result = type(self).__name__ + '{name="' + self.name + '"' + + if self.dependencies: + result += ', dependencies={' + format_iterable(self.dependencies, delimiter='"') + '}' + + return result + '}' + + +class BuildTarget(Target): + """ + A build target, which executes a certain action and produces one or several output files. + """ + + class Runnable(ABC): + """ + An abstract base class for all classes that can be run via a build target. + """ + + @abstractmethod + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + """ + Must be implemented by subclasses in order to run the target. + + :param build_unit: The build unit, the target belongs to + :param modules: A `ModuleRegistry` that can be used by the target for looking up modules + """ + + def get_input_files(self, modules: ModuleRegistry) -> List[str]: + """ + May be overridden by subclasses in order to return the input files required by the target. + + :param modules: A `ModuleRegistry` that can be used by the target for looking up modules + :return: A list that contains the input files + """ + return [] + + def get_output_files(self, modules: ModuleRegistry) -> List[str]: + """ + May be overridden by subclasses in order to return the output files produced by the target. + + :param modules: A `ModuleRegistry` that can be used by the target for looking up modules + :return: A list that contains the output files + """ + return [] + + def get_clean_files(self, modules: ModuleRegistry) -> List[str]: + """ + May be overridden by subclasses in order to return the output files produced by the target that must be + cleaned. + + :param modules: A `ModuleRegistry` that can be used by the target for looking up modules + :return: A list that contains the files to be cleaned + """ + return self.get_output_files(modules) + + def output_files_exist(self, modules: ModuleRegistry) -> bool: + """ + Returns whether all output files produced by the target exist or not. + + :param modules: A `ModuleRegistry` that can be used by the target for looking up modules + :return: True, if all output files exist, False otherwise + """ + output_files = self.get_output_files(modules) + + if output_files: + missing_files = [output_file for output_file in output_files if not path.exists(output_file)] + + if missing_files: + Log.verbose('Target needs to be run, because the following output files do not exist:') + + for missing_file in missing_files: + Log.verbose(' - %s', missing_file) + + Log.verbose('') + return False + + Log.verbose('All output files already exist.') + return True + + return False + + def input_files_have_changed(self, modules: ModuleRegistry) -> bool: + """ + Returns whether any input files required by the target have changed since the target was run for the last + time. + + :param modules: A `ModuleRegistry` that can be used by the target for looking up modules + :return: True, if any input files have changed, False otherwise + """ + input_files = self.get_input_files(modules) + + if input_files: + # TODO check timestamps or hashes + changed_files = input_files + + if changed_files: + Log.verbose('Target needs to be run, because the following input files have changed:\n') + + for input_file in input_files: + Log.verbose(' - %s', input_file) + + Log.verbose('') + return True + + Log.verbose('No input files have changed.') + return False + + return True + + class Builder(Target.Builder): + """ + A builder that allows to configure and create build targets. + """ + + def __init__(self, parent_builder: Any, target_name: str): + """ + :param parent_builder: The builder, this builder has been created from + :param target_name: The name of the target that is configured via the builder + """ + super().__init__(parent_builder, target_name) + self.runnables = [] + + def set_runnables(self, *runnables: 'BuildTarget.Runnable') -> Any: + """ + Sets one or several `Runnable` objects to be run by the target. + + :param runnables: The `Runnable` objects to be set + :return: The builder, this builder has been created from + """ + self.runnables = list(runnables) + return self.parent_builder + + def _build(self, build_unit: BuildUnit) -> Target: + return BuildTarget(self.target_name, self.dependencies, self.runnables, build_unit) + + def __init__(self, name: str, dependencies: List[Target.Dependency], runnables: List[Runnable], + build_unit: BuildUnit): + """ + :param name: The name of the target or None, if the target does not have a name + :param dependencies: A list that contains all dependencies of the target + :param runnables: The `BuildTarget.Runnable` to the be run by the target + :param build_unit: The `BuildUnit`, the target belongs to + """ + super().__init__(name, dependencies) + self.runnables = runnables + self.build_unit = build_unit + + def run(self, module_registry: ModuleRegistry): + for runnable in self.runnables: + if not runnable.output_files_exist(module_registry): + if runnable.input_files_have_changed(module_registry): + runnable.run(self.build_unit, module_registry) + + def clean(self, module_registry: ModuleRegistry): + for runnable in self.runnables: + clean_files = runnable.get_clean_files(module_registry) + delete_files(*clean_files) + + +class PhonyTarget(Target): + """ + A phony target, which executes a certain action and does not produce any output files. + """ + + Function = Callable[[], None] + + class Runnable(ABC): + """ + An abstract base class for all classes that can be run via a phony target. + """ + + @abstractmethod + def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + """ + Must be implemented by subclasses in order to run the target. + + :param build_unit: The build unit, the target belongs to + :param modules: A `ModuleRegistry` that can be used by the target for looking up modules + """ + + class Builder(Target.Builder): + """ + A builder that allows to configure and create phony targets. + """ + + def __init__(self, parent_builder: Any, target_name: str): + """ + :param parent_builder: The builder, this builder has been created from + :param target_name: The name of the target that is configured via the builder + """ + super().__init__(parent_builder, target_name) + self.functions = [] + self.runnables = [] + + def nop(self) -> Any: + """ + Instructs the target to not execute any action. + + :return: The `TargetBuilder`, this builder has been created from + """ + return self.parent_builder + + def set_functions(self, *functions: 'PhonyTarget.Function') -> Any: + """ + Sets one or several functions to be run by the target. + + :param functions: The functions to be set + :return: The builder, this builder has been created from + """ + self.functions = list(functions) + return self.parent_builder + + def set_runnables(self, *runnables: 'PhonyTarget.Runnable') -> Any: + """ + Sets one or several `Runnable` objects to be run by the target. + + :param runnables: The `Runnable` objects to be set + :return: The builder, this builder has been created from + """ + self.runnables = list(runnables) + return self.parent_builder + + def _build(self, build_unit: BuildUnit) -> Target: + + def action(module_registry: ModuleRegistry): + for function in self.functions: + function() + + for runnable in self.runnables: + runnable.run(build_unit, module_registry) + + return PhonyTarget(self.target_name, self.dependencies, action) + + def __init__(self, name: str, dependencies: List[Target.Dependency], action: Callable[[ModuleRegistry], None]): + """ + :param name: The name of the target + :param dependencies: A list that contains all dependencies of the target + :param action: The action to be executed by the target + """ + super().__init__(name, dependencies) + self.action = action + + def run(self, module_registry: ModuleRegistry): + self.action(module_registry) + + +class TargetBuilder: + """ + A builder that allows to configure and create multiple targets. + """ + + def __init__(self, build_unit: BuildUnit = BuildUnit()): + """ + :param build_unit: The build unit, the targets belong to + """ + self.build_unit = build_unit + self.target_builders = [] + + def add_build_target(self, name: str) -> BuildTarget.Builder: + """ + Adds a build target. + + :param name: The name of the target + :return: A `BuildTarget.Builder` that allows to configure the target + """ + target_builder = BuildTarget.Builder(self, name) + self.target_builders.append(target_builder) + return target_builder + + def add_phony_target(self, name: str) -> PhonyTarget.Builder: + """ + Adds a phony target. + + :param name: The name of the target + :return: A `PhonyTarget.Builder` that allows to configure the target + """ + target_builder = PhonyTarget.Builder(self, name) + self.target_builders.append(target_builder) + return target_builder + + def build(self) -> List[Target]: + """ + Creates and returns the targets that have been configured via the builder. + + :return: A list that stores the targets that have been created + """ + return reduce(lambda aggr, builder: aggr + builder.build(self.build_unit), self.target_builders, []) + + +class DependencyGraph: + """ + A graph that determines the execution order of targets based on the dependencies between them. + """ + + @dataclass + class Node(ABC): + """ + An abstract base class for all nodes in a dependency graph. + + Attributes: + target: The target that corresponds to this node + parent: The parent of this node, if any + child: The child of this node, if any + """ + target: Target + parent: Optional['DependencyGraph.Node'] = None + child: Optional['DependencyGraph.Node'] = None + + @staticmethod + def from_name(targets_by_name: Dict[str, Target], target_name: str, clean: bool) -> 'DependencyGraph.Node': + """ + Creates and returns a new node of a dependency graph corresponding to the target with a specific name. + + :param targets_by_name: A dictionary that stores all available targets by their names + :param target_name: The name of the target, the node should correspond to + :param clean: True, if the target should be cleaned, False otherwise + :return: The node that has been created + """ + target = targets_by_name[target_name] + return DependencyGraph.CleanNode(target) if clean else DependencyGraph.RunNode(target) + + @staticmethod + def from_dependency(targets_by_name: Dict[str, Target], dependency: Target.Dependency, + clean: bool) -> Optional['DependencyGraph.Node']: + """ + Creates and returns a new node of a dependency graph corresponding to the target referred to by a + `Target.Dependency`. + + :param targets_by_name: A dictionary that stores all available targets by their names + :param dependency: The dependency referring to the target, the node should correspond to + :param clean: True, if the target should be cleaned, False otherwise + :return: The node that has been created or None, if the dependency does not require a node to + be created + """ + if not clean or dependency.clean_dependency: + target = targets_by_name[dependency.target_name] + return DependencyGraph.CleanNode(target) if clean else DependencyGraph.RunNode(target) + + return None + + @abstractmethod + def execute(self, module_registry: ModuleRegistry): + """ + Must be implemented by subclasses in order to execute the node. + """ + + @abstractmethod + def copy(self) -> 'DependencyGraph.Node': + """ + Must be implemented by subclasses in order to create a shallow copy of the node. + + :return: The copy that has been created + """ + + def __str__(self) -> str: + return '[' + self.target.name + ']' + + def __eq__(self, other) -> bool: + return type(self) == type(other) and self.target == other.target + + class RunNode(Node): + """ + A node in the dependency graph that runs one or several targets. + """ + + def execute(self, module_registry: ModuleRegistry): + Log.verbose('Running target "%s"...', self.target.name) + self.target.run(module_registry) + + def copy(self) -> 'DependencyGraph.Node': + return DependencyGraph.RunNode(self.target) + + class CleanNode(Node): + """ + A node in the dependency graph that cleans one or several targets. + """ + + def execute(self, module_registry: ModuleRegistry): + Log.verbose('Cleaning target "%s"...', self.target.name) + self.target.clean(module_registry) + + def copy(self) -> 'DependencyGraph.Node': + return DependencyGraph.CleanNode(self.target) + + @dataclass + class Sequence: + """ + A sequence consisting of several nodes in a dependency graph. + + Attributes: + first: The first node in the sequence + last: The last node in the sequence + """ + first: 'DependencyGraph.Node' + last: 'DependencyGraph.Node' + + @staticmethod + def from_node(node: 'DependencyGraph.Node') -> 'DependencyGraph.Sequence': + """ + Creates and returns a new path that consists of a single node. + + :param node: The node + :return: The path that has been created + """ + node.parent = None + node.child = None + return DependencyGraph.Sequence(first=node, last=node) + + def prepend(self, node: 'DependencyGraph.Node'): + """ + Adds a new node at the start of the sequence. + + :param node: The node to be added + """ + first_node = self.first + first_node.parent = node + node.parent = None + node.child = first_node + self.first = node + + def copy(self) -> 'DependencyGraph.Sequence': + """ + Creates a deep copy of the sequence. + + :return: The copy that has been created + """ + current_node = self.last + copy = DependencyGraph.Sequence.from_node(current_node.copy()) + current_node = current_node.parent + + while current_node: + copy.prepend(current_node.copy()) + current_node = current_node.parent + + return copy + + def execute(self, module_registry: ModuleRegistry): + current_node = self.first + + while current_node: + current_node.execute(module_registry) + current_node = current_node.child + + def __str__(self) -> str: + current_node = self.first + result = ' → ' + str(current_node) + current_node = current_node.child + indent = 1 + + while current_node: + result += '\n' + reduce(lambda aggr, _: aggr + ' ', range(indent), '') + ' ↳ ' + str(current_node) + current_node = current_node.child + indent += 1 + + return result + + @staticmethod + def __expand_sequence(targets_by_name: Dict[str, Target], sequence: Sequence, clean: bool) -> List[Sequence]: + sequences = [] + dependencies = sequence.first.target.dependencies + + if dependencies: + for dependency in dependencies: + new_node = DependencyGraph.Node.from_dependency(targets_by_name, dependency, clean=clean) + + if new_node: + new_sequence = sequence.copy() + new_sequence.prepend(new_node) + sequences.extend(DependencyGraph.__expand_sequence(targets_by_name, new_sequence, clean=clean)) + else: + sequences.append(sequence) + else: + sequences.append(sequence) + + return sequences + + @staticmethod + def __create_sequence(targets_by_name: Dict[str, Target], target_name: str, clean: bool) -> List[Sequence]: + node = DependencyGraph.Node.from_name(targets_by_name, target_name, clean=clean) + sequence = DependencyGraph.Sequence.from_node(node) + return DependencyGraph.__expand_sequence(targets_by_name, sequence, clean=clean) + + @staticmethod + def __find_in_parents(node: Node, parent: Optional[Node]) -> Optional[Node]: + while parent: + if parent == node: + return parent + + parent = parent.parent + + return None + + @staticmethod + def __merge_two_sequences(first_sequence: Sequence, second_sequence: Sequence) -> Sequence: + first_node = first_sequence.last + second_node = second_sequence.last + + while second_node: + overlapping_node = DependencyGraph.__find_in_parents(second_node, first_node) + + if overlapping_node: + first_node = overlapping_node.parent + else: + new_node = second_node.copy() + + if first_node: + new_node.parent = first_node + + if first_node.child: + new_node.child = first_node.child + first_node.child.parent = new_node + + first_node.child = new_node + + if first_node == first_sequence.last: + first_sequence.last = new_node + else: + first_sequence.prepend(new_node) + + second_node = second_node.parent + + return first_sequence + + @staticmethod + def __merge_multiple_sequences(sequences: List[Sequence]) -> Sequence: + while len(sequences) > 1: + second_sequence = sequences.pop() + first_sequence = sequences.pop() + merged_sequence = DependencyGraph.__merge_two_sequences(first_sequence, second_sequence) + sequences.append(merged_sequence) + + return sequences[0] + + def __init__(self, targets_by_name: Dict[str, Target], *target_names: str, clean: bool): + """ + :param targets_by_name: A dictionary that stores all available targets by their names + :param target_names: The names of the targets to be included in the graph + :param clean: True, if the targets should be cleaned, False otherwise + """ + self.sequence = self.__merge_multiple_sequences( + reduce(lambda aggr, target_name: aggr + self.__create_sequence(targets_by_name, target_name, clean=clean), + target_names, [])) + + def execute(self, module_registry: ModuleRegistry): + """ + Executes all targets in the graph in the pre-determined order. + + :param module_registry: The `ModuleRegistry` that should be used by targets for looking up modules + """ + self.sequence.execute(module_registry) + + def __str__(self) -> str: + return str(self.sequence) + + +class TargetRegistry: + """ + Allows to register targets. + """ + + def __init__(self): + self.targets_by_name = {} + + def register(self, target: Target): + """ + Registers a new target. + + :param target: The target to be registered + """ + existing = self.targets_by_name.get(target.name) + + if existing: + raise ValueError('Failed to register target ' + str(target) + + ', because a target with the same name has already been registered: ' + str(existing)) + + self.targets_by_name[target.name] = target + + def create_dependency_graph(self, *target_names: str, clean: bool = False) -> DependencyGraph: + """ + Creates and returns a `DependencyGraph` for each of the given targets. + + :param target_names: The names of the targets for which graphs should be created + :param clean: True, if the targets should be cleaned, False otherwise + :return: A list that contains the graphs that have been created + """ + if not target_names: + Log.error('No targets given') + + invalid_targets = [target_name for target_name in target_names if target_name not in self.targets_by_name] + + if invalid_targets: + Log.error('The following targets are invalid: %s', format_iterable(invalid_targets)) + + return DependencyGraph(self.targets_by_name, *target_names, clean=clean) diff --git a/scons/util/units.py b/build_system/util/units.py similarity index 100% rename from scons/util/units.py rename to build_system/util/units.py diff --git a/scons/util/venv.py b/build_system/util/venv.py similarity index 100% rename from scons/util/venv.py rename to build_system/util/venv.py diff --git a/scons/versioning/__init__.py b/build_system/versioning/__init__.py similarity index 100% rename from scons/versioning/__init__.py rename to build_system/versioning/__init__.py diff --git a/scons/versioning/changelog.py b/build_system/versioning/changelog.py similarity index 100% rename from scons/versioning/changelog.py rename to build_system/versioning/changelog.py diff --git a/scons/versioning/versioning.py b/build_system/versioning/versioning.py similarity index 100% rename from scons/versioning/versioning.py rename to build_system/versioning/versioning.py diff --git a/doc/developer_guide/coding_standards.md b/doc/developer_guide/coding_standards.md index dae978cb5..b49e14469 100644 --- a/doc/developer_guide/coding_standards.md +++ b/doc/developer_guide/coding_standards.md @@ -68,7 +68,7 @@ We aim to enforce a consistent code style across the entire project. For this pu - For formatting the C++ code, we use [clang-format](https://clang.llvm.org/docs/ClangFormat.html). The desired C++ code style is defined in the file `.clang-format` in the project's root directory. In addition, [cpplint](https://github.com/cpplint/cpplint) is used for static code analysis. It uses the configuration file `CPPLINT.cfg`. - We use [YAPF](https://github.com/google/yapf) to enforce the Python code style defined in the file `.style.yapf`. In addition, [isort](https://github.com/PyCQA/isort) is used to keep the ordering of imports in Python and Cython source files consistent according to the configuration file `.isort.cfg` and [pylint](https://pylint.org/) is used to check for common issues in the Python code according to the configuration file `.pylintrc`. - For applying a consistent style to Markdown files, including those used for writing the documentation, we use [mdformat](https://github.com/executablebooks/mdformat). -- We apply [yamlfix](https://github.com/lyz-code/yamlfix) to YAML files to enforce the code style defined in the file `scons/code_style/yaml/.yamlfix.toml`. +- We apply [yamlfix](https://github.com/lyz-code/yamlfix) to YAML files to enforce the code style defined in the file `build_system/code_style/yaml/.yamlfix.toml`. If you have modified the project's source code, you can check whether it adheres to our coding standards via the following command: diff --git a/doc/developer_guide/compilation.md b/doc/developer_guide/compilation.md index bd67c2a6d..b2b0e775a 100644 --- a/doc/developer_guide/compilation.md +++ b/doc/developer_guide/compilation.md @@ -4,7 +4,7 @@ As discussed in the previous section {ref}`project-structure`, the algorithms that are provided by this project are implemented in [C++](https://en.wikipedia.org/wiki/C%2B%2B) to ensure maximum efficiency (requires C++ 17 or newer). In addition, a [Python]() wrapper that integrates with the [scikit-learn](https://scikit-learn.org) framework is provided (requires Python 3.10 or newer). To make the underlying C++ implementation accessible from within the Python code, [Cython](https://en.wikipedia.org/wiki/Cython) is used (requires Cython 3.0 or newer). -Unlike pure Python programs, the C++ and Cython source files must be compiled for a particular target platform. To ease the process of compiling the source code, the project comes with a [SCons](https://scons.org/) build that automates the necessary steps. In the following, we discuss the individual steps that are necessary for building the project from scratch. This is necessary if you intend to modify the library's source code. If you want to use the algorithm without any custom modifications, the {ref}`installation ` of pre-built packages is usually a better choice. +Unlike pure Python programs, the C++ and Cython source files must be compiled for a particular target platform. To ease the process of compiling the source code, the project comes with a build system that automates the necessary steps. In the following, we discuss the individual steps that are necessary for building the project from scratch. This is necessary if you intend to modify the library's source code. If you want to use the algorithm without any custom modifications, the {ref}`installation ` of pre-built packages is usually a better choice. ## Prerequisites diff --git a/scons/modules_old.py b/scons/modules_old.py deleted file mode 100644 index d36ad7aa1..000000000 --- a/scons/modules_old.py +++ /dev/null @@ -1,514 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Provides access to directories and files belonging to different modules that are part of the project. -""" -from abc import ABC, abstractmethod -from glob import glob -from os import environ, path, walk -from typing import Callable, List, Optional - -from util.env import get_env_array -from util.units import BuildUnit - - -def find_files_recursively(directory: str, - directory_filter: Callable[[str, str], bool] = lambda *_: True, - file_filter: Callable[[str, str], bool] = lambda *_: True) -> List[str]: - """ - Finds and returns files in a directory and its subdirectories that match a given filter. - - :param directory: The directory to be searched - :param directory_filter: A function to be used for filtering subdirectories - :param file_filter: A function to be used for filtering files - :return: A list that contains the paths of all files that have been found - """ - result = [] - - for parent_directory, subdirectories, files in walk(directory, topdown=True): - subdirectories[:] = [ - subdirectory for subdirectory in subdirectories if directory_filter(parent_directory, subdirectory) - ] - - for file in files: - if file_filter(parent_directory, file): - result.append(path.join(parent_directory, file)) - - return result - - -class Module(BuildUnit, ABC): - """ - An abstract base class for all classes that provide access to directories and files that belong to a module. - """ - - def __init__(self): - super().__init__(path.join(self.root_dir, 'requirements.txt')) - - @property - @abstractmethod - def root_dir(self) -> str: - """ - The path to the module's root directory. - """ - - @property - def build_dir(self) -> str: - """ - The path to the directory, where build files are stored. - """ - return path.join(self.root_dir, 'build') - - -class SourceModule(Module, ABC): - """ - An abstract base class for all classes that provide access to directories and files that belong to a module, which - contains source code. - """ - - class Subproject(ABC): - """ - An abstract base class for all classes that provide access to directories and files that belong to an individual - subproject that is part of a module, which contains source files. - """ - - def __init__(self, parent_module: 'SourceModule', root_dir: str): - """ - :param parent_module: The `SourceModule`, the subproject belongs to - :param root_dir: The root directory of the suproject - """ - self.parent_module = parent_module - self.root_dir = root_dir - - @property - def name(self) -> str: - """ - The name of the subproject. - """ - return path.basename(self.root_dir) - - def is_enabled(self) -> bool: - """ - Returns whether the subproject is enabled or not. - - :return: True, if the subproject is enabled, False otherwise - """ - enabled_subprojects = get_env_array(environ, 'SUBPROJECTS') - return not enabled_subprojects or self.name in enabled_subprojects - - -class PythonModule(SourceModule): - """ - Provides access to directories and files that belong to the project's Python code. - """ - - class Subproject(SourceModule.Subproject): - """ - Provides access to directories and files that belong to an individual subproject that is part of the project's - Python code. - """ - - @staticmethod - def __filter_pycache_directories(_: str, directory: str) -> bool: - return directory != '__pycache__' - - @property - def source_dir(self) -> str: - """ - The directory that contains the subproject's source code. - """ - return path.join(self.root_dir, 'mlrl') - - @property - def test_dir(self) -> str: - """ - The directory that contains the subproject's automated tests. - """ - return path.join(self.root_dir, 'tests') - - @property - def dist_dir(self) -> str: - """ - The directory that contains all wheel packages that have been built for the subproject. - """ - return path.join(self.root_dir, 'dist') - - @property - def build_dirs(self) -> List[str]: - """ - A list that contains all directories, where the subproject's build files are stored. - """ - return [self.dist_dir, path.join(self.root_dir, 'build')] + glob(path.join(self.root_dir, '*.egg-info')) - - def find_wheels(self) -> List[str]: - """ - Finds and returns all wheel packages that have been built for the subproject. - - :return: A list that contains the paths of the wheel packages that have been found - """ - return glob(path.join(self.dist_dir, '*.whl')) - - def find_source_files(self) -> List[str]: - """ - Finds and returns all source files that are contained by the subproject. - - :return: A list that contains the paths of the source files that have been found - """ - return find_files_recursively(self.source_dir, directory_filter=self.__filter_pycache_directories) - - def find_shared_libraries(self) -> List[str]: - """ - Finds and returns all shared libraries that are contained in the subproject's source tree. - - :return: A list that contains all shared libraries that have been found - """ - - def file_filter(_: str, file: str) -> bool: - return (file.startswith('lib') and file.find('.so') >= 0) \ - or file.endswith('.dylib') \ - or (file.startswith('mlrl') and file.endswith('.lib')) \ - or file.endswith('.dll') - - return find_files_recursively(self.source_dir, - directory_filter=self.__filter_pycache_directories, - file_filter=file_filter) - - def find_extension_modules(self) -> List[str]: - """ - Finds and returns all extension modules that are contained in the subproject's source tree. - - :return: A list that contains all extension modules that have been found - """ - - def file_filter(_: str, file: str) -> bool: - return (not file.startswith('lib') and file.endswith('.so')) \ - or file.endswith('.pyd') \ - or (not file.startswith('mlrl') and file.endswith('.lib')) - - return find_files_recursively(self.source_dir, - directory_filter=self.__filter_pycache_directories, - file_filter=file_filter) - - @property - def root_dir(self) -> str: - return 'python' - - def find_subprojects(self, return_all: bool = False) -> List[Subproject]: - """ - Finds and returns all subprojects that are part of the Python code. - - :param return_all: True, if all subprojects should be returned, even if they are disabled, False otherwise - :return: A list that contains all subrojects that have been found - """ - subprojects = [ - PythonModule.Subproject(self, file) for file in glob(path.join(self.root_dir, 'subprojects', '*')) - if path.isdir(file) - ] - return subprojects if return_all else [subproject for subproject in subprojects if subproject.is_enabled()] - - def find_subproject(self, file: str) -> Optional[Subproject]: - """ - Finds and returns the subproject to which a given file belongs. - - :param file: The path of the file - :return: The subproject to which the given file belongs or None, if no such subproject is available - """ - for subproject in self.find_subprojects(): - if file.startswith(subproject.root_dir): - return subproject - - return None - - -class CppModule(SourceModule): - """ - Provides access to directories and files that belong to the project's C++ code. - """ - - class Subproject(SourceModule.Subproject): - """ - Provides access to directories and files that belong to an individual subproject that is part of the project's - C++ code. - """ - - @property - def include_dir(self) -> str: - """ - The directory that contains the header files. - """ - return path.join(self.root_dir, 'include') - - @property - def src_dir(self) -> str: - """ - The directory that contains the source files. - """ - return path.join(self.root_dir, 'src') - - @property - def test_dir(self) -> str: - """ - The directory that contains the source code for automated tests. - """ - return path.join(self.root_dir, 'test') - - def find_source_files(self) -> List[str]: - """ - Finds and returns all source files that are contained by the subproject. - - :return: A list that contains the paths of the source files that have been found - """ - - def file_filter(_: str, file: str) -> bool: - return file.endswith('.hpp') or file.endswith('.cpp') - - return find_files_recursively(self.root_dir, file_filter=file_filter) - - @property - def root_dir(self) -> str: - return 'cpp' - - def find_subprojects(self, return_all: bool = False) -> List[Subproject]: - """ - Finds and returns all subprojects that are part of the C++ code. - - - :param return_all: True, if all subprojects should be returned, even if they are disabled, False otherwise - :return: A list that contains all subprojects that have been found - """ - subprojects = [ - CppModule.Subproject(self, file) for file in glob(path.join(self.root_dir, 'subprojects', '*')) - if path.isdir(file) - ] - return subprojects if return_all else [subproject for subproject in subprojects if subproject.is_enabled()] - - -class BuildModule(Module): - """ - Provides access to directories and files that belong to the build system. - """ - - @property - def root_dir(self) -> str: - return 'scons' - - -class DocumentationModule(Module): - """ - Provides access to directories and files that belong to the project's documentation. - """ - - class ApidocSubproject(ABC): - """ - An abstract base class for all classes that provide access to directories and files that are needed for building - the API documentation of a certain C++ or Python subproject. - """ - - def __init__(self, parent_module: 'DocumentationModule', source_subproject: SourceModule.Subproject): - """ - :param parent_module: The `DocumentationModule` this subproject belongs to - :param source_subproject: The subproject of which the API documentation should be built - """ - self.parent_module = parent_module - self.source_subproject = source_subproject - - @property - def name(self) -> str: - """ - The name of the subproject of which the API documentation should be built. - """ - return self.source_subproject.name - - @property - @abstractmethod - def build_dir(self) -> str: - """ - The directory, where build files should be stored. - """ - - @property - @abstractmethod - def root_file(self) -> str: - """ - The path of the root file of the API documentation. - """ - - def find_build_files(self) -> List[str]: - """ - Finds and returns all build files that have been created when building the API documentation. - - :return: A list that contains the paths of all build files that have been found - """ - return find_files_recursively(self.build_dir) - - class CppApidocSubproject(ApidocSubproject): - """ - Provides access to the directories and files that are necessary for building the API documentation of a certain - C++ subproject. - """ - - @property - def build_dir(self) -> str: - return path.join(self.parent_module.apidoc_dir_cpp, self.name) - - @property - def root_file(self) -> str: - return path.join(self.build_dir, 'filelist.rst') - - class PythonApidocSubproject(ApidocSubproject): - """ - Provides access to the directories and files that are necessary for building the API documentation of a certain - Python subproject. - """ - - @property - def build_dir(self) -> str: - return path.join(self.parent_module.apidoc_dir_python, self.name) - - @property - def root_file(self) -> str: - return path.join(self.build_dir, 'mlrl.' + self.name + '.rst') - - @property - def root_dir(self) -> str: - return 'doc' - - @property - def doxygen_config_file(self) -> str: - """ - The Doxygen config file. - """ - return path.join(self.root_dir, 'Doxyfile') - - @property - def config_file(self) -> str: - """ - The config file that should be used for building the documentation. - """ - return path.join(self.root_dir, 'conf.py') - - @property - def apidoc_dir(self) -> str: - """ - The directory, where API documentations should be stored. - """ - return path.join(self.root_dir, 'developer_guide', 'api') - - @property - def apidoc_dir_python(self) -> str: - """ - The directory, where Python API documentations should be stored. - """ - return path.join(self.apidoc_dir, 'python') - - @property - def apidoc_tocfile_python(self) -> str: - """ - The tocfile referencing all Python API documentations. - """ - return path.join(self.apidoc_dir_python, 'index.md') - - @property - def apidoc_dir_cpp(self) -> str: - """ - The directory, where C++ API documentations should be stored. - """ - return path.join(self.apidoc_dir, 'cpp') - - @property - def apidoc_tocfile_cpp(self) -> str: - """ - The tocfile referencing all C++ API documentations. - """ - return path.join(self.apidoc_dir_cpp, 'index.md') - - @property - def build_dir(self) -> str: - """ - The directory, where the documentation should be stored. - """ - return path.join(self.root_dir, '_build', 'html') - - def find_build_files(self) -> List[str]: - """ - Finds and returns all files that belong to the documentation that has been built. - - :return: A list that contains the paths of the build files that have been found - """ - return find_files_recursively(self.build_dir) - - def find_source_files(self) -> List[str]: - """ - Finds and returns all source files from which the documentation is built. - - :return: A list that contains the paths of the source files that have been found - """ - - def directory_filter(parent_directory: str, directory: str) -> bool: - return path.join(parent_directory, directory) != self.build_dir - - def file_filter(_: str, file: str) -> bool: - return file == 'conf.py' or file.endswith('.rst') or file.endswith('.svg') or file.endswith('.md') - - return find_files_recursively(self.root_dir, directory_filter=directory_filter, file_filter=file_filter) - - def get_cpp_apidoc_subproject(self, cpp_subproject: CppModule.Subproject) -> CppApidocSubproject: - """ - Returns a `CppApidocSubproject` for building the API documentation of a given C++ subproject. - - :param cpp_subproject: The C++ subproject of which the API documentation should be built - :return: A `CppApidocSubproject` - """ - return DocumentationModule.CppApidocSubproject(self, cpp_subproject) - - def get_python_apidoc_subproject(self, python_subproject: PythonModule.Subproject) -> PythonApidocSubproject: - """ - Returns a `PythonApidocSubproject` for building the API documentation of a given Python subproject. - - :param python_subproject: The Python subproject of which the API documentation should be built - :return: A `PythonApidocSubproject` - """ - return DocumentationModule.PythonApidocSubproject(self, python_subproject) - - def find_cpp_apidoc_subproject(self, file: str) -> Optional[CppApidocSubproject]: - """ - Finds and returns the `CppApidocSubproject` to which a given file belongs. - - :param file: The path of the file - :return: The `CppApiSubproject` to which the given file belongs or None, if no such subproject is - available - """ - for subproject in CPP_MODULE.find_subprojects(): - apidoc_subproject = self.get_cpp_apidoc_subproject(subproject) - - if file.startswith(apidoc_subproject.build_dir): - return apidoc_subproject - - return None - - def find_python_apidoc_subproject(self, file: str) -> Optional[PythonApidocSubproject]: - """ - Finds and returns the `PythonApidocSubproject` to which a given file belongs. - - :param file: The path of the file - :return: The `PythonApidocSubproject` to which the given file belongs or None, if no such subproject is - available - """ - for subproject in PYTHON_MODULE.find_subprojects(): - apidoc_subproject = self.get_python_apidoc_subproject(subproject) - - if file.startswith(apidoc_subproject.build_dir): - return apidoc_subproject - - return None - - -BUILD_MODULE = BuildModule() - -PYTHON_MODULE = PythonModule() - -CPP_MODULE = CppModule() - -DOC_MODULE = DocumentationModule() - -ALL_MODULES = [BUILD_MODULE, PYTHON_MODULE, CPP_MODULE, DOC_MODULE] diff --git a/scons/sconstruct.py b/scons/sconstruct.py deleted file mode 100644 index 4d86387de..000000000 --- a/scons/sconstruct.py +++ /dev/null @@ -1,143 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Defines the individual targets of the build process. -""" -import sys - -from os import path - -from documentation import apidoc_cpp, apidoc_cpp_tocfile, apidoc_python, apidoc_python_tocfile, doc -from modules_old import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE -from util.files import FileSearch -from util.format import format_iterable -from util.modules import Module, ModuleRegistry -from util.paths import Project -from util.reflection import import_source_file -from util.targets import Target, TargetRegistry - -from SCons.Script import COMMAND_LINE_TARGETS - - -def __create_phony_target(environment, target_name, action=None): - return environment.AlwaysBuild(environment.Alias(target_name, None, action)) - - -def __print_if_clean(environment, message: str): - if environment.GetOption('clean'): - print(message) - - -# Define target names... -TARGET_NAME_APIDOC = 'apidoc' -TARGET_NAME_APIDOC_CPP = TARGET_NAME_APIDOC + '_cpp' -TARGET_NAME_APIDOC_PYTHON = TARGET_NAME_APIDOC + '_python' -TARGET_NAME_DOC = 'doc' - -VALID_TARGETS = {TARGET_NAME_APIDOC, TARGET_NAME_APIDOC_CPP, TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC} - -DEFAULT_TARGET = 'install_wheels' - -# Register modules... -init_files = FileSearch().set_recursive(True).filter_by_name('__init__.py').list(Project.BuildSystem.root_directory) -module_registry = ModuleRegistry() - -for init_file in init_files: - for module in getattr(import_source_file(init_file), 'MODULES', []): - if isinstance(module, Module): - module_registry.register(module) - -# Register build targets... -target_registry = TargetRegistry(module_registry) -env = target_registry.environment - -for init_file in init_files: - for target in getattr(import_source_file(init_file), 'TARGETS', []): - if isinstance(target, Target): - target_registry.add_target(target) - -target_registry.register() - -# Raise an error if any invalid targets are given... -VALID_TARGETS.update(target_registry.target_names) -invalid_targets = [target for target in COMMAND_LINE_TARGETS if target not in VALID_TARGETS] - -if invalid_targets: - print('The following targets are unknown: ' + format_iterable(invalid_targets)) - sys.exit(-1) - -# Create temporary file ".sconsign.dblite" in the build directory... -env.SConsignFile(name=path.relpath(path.join(BUILD_MODULE.build_dir, '.sconsign'), BUILD_MODULE.root_dir)) - -# Define targets for generating the documentation... -commands_apidoc_cpp = [] -commands_apidoc_python = [] - -for subproject in CPP_MODULE.find_subprojects(): - apidoc_subproject = DOC_MODULE.get_cpp_apidoc_subproject(subproject) - build_files = apidoc_subproject.find_build_files() - targets_apidoc_cpp = build_files if build_files else apidoc_subproject.build_dir - command_apidoc_cpp = env.Command(targets_apidoc_cpp, subproject.find_source_files(), action=apidoc_cpp) - env.NoClean(command_apidoc_cpp) - commands_apidoc_cpp.append(command_apidoc_cpp) - -command_apidoc_cpp_tocfile = env.Command(DOC_MODULE.apidoc_tocfile_cpp, None, action=apidoc_cpp_tocfile) -env.NoClean(command_apidoc_cpp_tocfile) -env.Depends(command_apidoc_cpp_tocfile, commands_apidoc_cpp) - -target_apidoc_cpp = env.Alias(TARGET_NAME_APIDOC_CPP, None, None) -env.Depends(target_apidoc_cpp, command_apidoc_cpp_tocfile) - -for subproject in PYTHON_MODULE.find_subprojects(): - apidoc_subproject = DOC_MODULE.get_python_apidoc_subproject(subproject) - build_files = apidoc_subproject.find_build_files() - targets_apidoc_python = build_files if build_files else apidoc_subproject.build_dir - command_apidoc_python = env.Command(targets_apidoc_python, subproject.find_source_files(), action=apidoc_python) - env.NoClean(command_apidoc_python) - env.Depends(command_apidoc_python, 'install_wheels') - commands_apidoc_python.append(command_apidoc_python) - -command_apidoc_python_tocfile = env.Command(DOC_MODULE.apidoc_tocfile_python, None, action=apidoc_python_tocfile) -env.NoClean(command_apidoc_python_tocfile) -env.Depends(command_apidoc_python_tocfile, commands_apidoc_python) - -target_apidoc_python = env.Alias(TARGET_NAME_APIDOC_PYTHON, None, None) -env.Depends(target_apidoc_python, command_apidoc_python_tocfile) - -target_apidoc = env.Alias(TARGET_NAME_APIDOC, None, None) -env.Depends(target_apidoc, [target_apidoc_cpp, target_apidoc_python]) - -doc_files = DOC_MODULE.find_build_files() -targets_doc = doc_files if doc_files else DOC_MODULE.build_dir -command_doc = env.Command(targets_doc, DOC_MODULE.find_source_files(), action=doc) -env.Depends(command_doc, target_apidoc) -target_doc = env.Alias(TARGET_NAME_DOC, None, None) -env.Depends(target_doc, command_doc) - -# Define target for cleaning up the documentation and associated build directories... -if not COMMAND_LINE_TARGETS \ - or TARGET_NAME_APIDOC_CPP in COMMAND_LINE_TARGETS \ - or TARGET_NAME_APIDOC in COMMAND_LINE_TARGETS: - __print_if_clean(env, 'Removing C++ API documentation...') - env.Clean([target_apidoc_cpp, DEFAULT_TARGET], DOC_MODULE.apidoc_tocfile_cpp) - - for subproject in CPP_MODULE.find_subprojects(return_all=True): - apidoc_subproject = DOC_MODULE.get_cpp_apidoc_subproject(subproject) - env.Clean([target_apidoc_cpp, DEFAULT_TARGET], apidoc_subproject.build_dir) - -if not COMMAND_LINE_TARGETS \ - or TARGET_NAME_APIDOC_PYTHON in COMMAND_LINE_TARGETS \ - or TARGET_NAME_APIDOC in COMMAND_LINE_TARGETS: - __print_if_clean(env, 'Removing Python API documentation...') - env.Clean([target_apidoc_python, DEFAULT_TARGET], DOC_MODULE.apidoc_tocfile_python) - - for subproject in PYTHON_MODULE.find_subprojects(return_all=True): - apidoc_subproject = DOC_MODULE.get_python_apidoc_subproject(subproject) - env.Clean([target_apidoc_python, DEFAULT_TARGET], apidoc_subproject.build_dir) - -if not COMMAND_LINE_TARGETS or TARGET_NAME_DOC in COMMAND_LINE_TARGETS: - __print_if_clean(env, 'Removing documentation...') - env.Clean([target_doc, DEFAULT_TARGET], DOC_MODULE.build_dir) - -# Set the default target... -env.Default(DEFAULT_TARGET) diff --git a/scons/util/requirements.txt b/scons/util/requirements.txt deleted file mode 100644 index f476b0b6c..000000000 --- a/scons/util/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -scons >= 4.8, < 4.9 diff --git a/scons/util/targets.py b/scons/util/targets.py deleted file mode 100644 index de71b3135..000000000 --- a/scons/util/targets.py +++ /dev/null @@ -1,535 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Provides base classes for defining individual targets of the build process. -""" -import sys - -from abc import ABC, abstractmethod -from dataclasses import dataclass -from functools import reduce -from typing import Any, Callable, Dict, Iterable, List, Optional, Set -from uuid import uuid4 - -from util.format import format_iterable -from util.modules import ModuleRegistry -from util.units import BuildUnit - -from SCons.Script import COMMAND_LINE_TARGETS -from SCons.Script.SConscript import SConsEnvironment as Environment - - -class Target(ABC): - """ - An abstract base class for all targets of the build system. - """ - - @dataclass - class Dependency: - """ - A single dependency of a parent target. - - Attributes: - target_name: The name of the target, the parent target depends on - clean_dependency: True, if the output files of the dependency should also be cleaned when cleaning the - output files of the parent target, False otherwise - """ - target_name: str - clean_dependency: bool - - def __str__(self) -> str: - return self.target_name - - def __eq__(self, other: 'Target.Dependency') -> bool: - return self.target_name == other.target_name - - def __hash__(self) -> int: - return hash(self.target_name) - - class Builder(ABC): - """ - An abstract base class for all builders that allow to configure and create targets. - """ - - def __init__(self, parent_builder: Any): - """ - :param parent_builder: The builder, this builder has been created from - """ - self.parent_builder = parent_builder - self.child_builders = [] - self.dependencies = set() - - def depends_on(self, *target_names: str, clean_dependencies: bool = False) -> 'Target.Builder': - """ - Adds on or several targets, this target should depend on. - - :param target_names: The names of the targets, this target should depend on - :param clean_dependencies: True, if output files of the dependencies should also be cleaned when cleaning - the output files of this target, False otherwise - :return: The `Target.Builder` itself - """ - for target_name in target_names: - self.dependencies.add(Target.Dependency(target_name=target_name, clean_dependency=clean_dependencies)) - - return self - - def depends_on_build_target(self, clean_dependency: bool = True) -> 'BuildTarget.Builder': - """ - Creates and returns a `BuildTarget.Builder` that allows to configure a build target, this target should - depend on. - - :param clean_dependency: True, if output files of the dependency should also be cleaned when cleaning the - output files of this target, False otherwise - :return: The `BuildTarget.Builder` that has been created - """ - target_name = str(uuid4()) - target_builder = BuildTarget.Builder(self, target_name) - self.child_builders.append(target_builder) - self.depends_on(target_name, clean_dependencies=clean_dependency) - return target_builder - - def depends_on_build_targets(self, - iterable: Iterable[Any], - target_configurator: Callable[[Any, 'BuildTarget.Builder'], None], - clean_dependencies: bool = True) -> 'Target.Builder': - """ - Configures multiple build targets, one for each value in a given `Iterable`, this target should depend on. - - :param iterable: An `Iterable` that provides access to the values for which dependencies should - be created - :param target_configurator: A function that accepts one value in the `Iterable` at a time, as well as a - `BuildTarget.Builder` for configuring the corresponding dependency - :param clean_dependencies: True, if output files of the dependencies should also be cleaned when cleaning - the output files of this target, False otherwise - :return: The `Target.Builder` itself - """ - for value in iterable: - target_builder = self.depends_on_build_target(clean_dependency=clean_dependencies) - target_configurator(value, target_builder) - - return self - - def depends_on_phony_target(self, clean_dependency: bool = True) -> 'PhonyTarget.Builder': - """ - Creates and returns a `PhonyTarget.Builder` that allows to configure a phony target, this target should - depend on. - - :param clean_dependency: True, if output files of the dependency should also be cleaned when cleaning the - output files of this target, False otherwise - :return: The `PhonyTarget.Builder` that has been created - """ - target_name = str(uuid4()) - target_builder = PhonyTarget.Builder(self, target_name) - self.child_builders.append(target_builder) - self.depends_on(target_name, clean_dependencies=clean_dependency) - return target_builder - - def depends_on_phony_targets(self, - iterable: Iterable[Any], - target_configurator: Callable[[Any, 'PhonyTarget.Builder'], None], - clean_dependencies: bool = True) -> 'Target.Builder': - """ - Configures multiple phony targets, one for each value in a given `Iterable`, this target should depend on. - - :param iterable: An `Iterable` that provides access to the values for which dependencies should - be created - :param target_configurator: A function that accepts one value in the `Iterable` at a time, as well as a - `BuildTarget.Builder` for configuring the corresponding dependency - :param clean_dependencies: True, if output files of the dependencies should also be cleaned when cleaning - the output files of this target, False otherwise - :return: The `Target.Builder` itself - """ - for value in iterable: - target_builder = self.depends_on_phony_target(clean_dependency=clean_dependencies) - target_configurator(value, target_builder) - - return self - - @abstractmethod - def _build(self, build_unit: BuildUnit) -> 'Target': - """ - Must be implemented by subclasses in order to create the target that has been configured via the builder. - - :param build_unit: The build unit, the target belongs to - :return: The target that has been created - """ - - def build(self, build_unit: BuildUnit) -> List['Target']: - """ - Creates and returns all targets that have been configured via the builder. - - :param build_unit: The build unit, the target belongs to - :return: The targets that have been created - """ - return [self._build(build_unit)] + reduce(lambda aggr, builder: aggr + builder.build(build_unit), - self.child_builders, []) - - def __init__(self, name: str, dependencies: Set['Target.Dependency']): - """ - :param name: The name of the target - :param dependencies: The dependencies of the target - """ - self.name = name - self.dependencies = dependencies - - @abstractmethod - def register(self, environment: Environment, module_registry: ModuleRegistry) -> Any: - """ - Must be implemented by subclasses in order to register this target. - - :param environment: The environment, the target should be registered at - :param module_registry: The `ModuleRegistry` that can be used by the target for looking up modules - :return: The scons target that has been created - """ - - def get_clean_files(self, module_registry: ModuleRegistry) -> Optional[List[str]]: - """ - May be overridden by subclasses in order to return the files that should be cleaned up for this target. - - :param module_registry: The `ModuleRegistry` that can be used by the target for looking up modules - :return: A list that contains the files to be cleaned or None, if cleaning is not necessary - """ - return None - - def __str__(self) -> str: - result = type(self).__name__ + '{name="' + self.name + '"' - - if self.dependencies: - result += ', dependencies={' + format_iterable(self.dependencies, delimiter='"') + '}' - - return result + '}' - - -class BuildTarget(Target): - """ - A build target, which executes a certain action and produces one or several output files. - """ - - class Runnable(ABC): - """ - An abstract base class for all classes that can be run via a build target. - """ - - @abstractmethod - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - """ - Must be implemented by subclasses in order to run the target. - - :param build_unit: The build unit, the target belongs to - :param modules: A `ModuleRegistry` that can be used by the target for looking up modules - """ - - def get_input_files(self, modules: ModuleRegistry) -> List[str]: - """ - May be overridden by subclasses in order to return the input files required by the target. - - :param modules: A `ModuleRegistry` that can be used by the target for looking up modules - :return: A list that contains the input files - """ - return [] - - def get_output_files(self, modules: ModuleRegistry) -> List[str]: - """ - May be overridden by subclasses in order to return the output files produced by the target. - - :param modules: A `ModuleRegistry` that can be used by the target for looking up modules - :return: A list that contains the output files - """ - return [] - - def get_clean_files(self, modules: ModuleRegistry) -> List[str]: - """ - May be overridden by subclasses in order to return the output files produced by the target that must be - cleaned. - - :param modules: A `ModuleRegistry` that can be used by the target for looking up modules - :return: A list that contains the files to be cleaned - """ - return self.get_output_files(modules) - - class Builder(Target.Builder): - """ - A builder that allows to configure and create build targets. - """ - - def __init__(self, parent_builder: Any, name: str): - """ - :param parent_builder: The builder, this builder has been created from - :param name: The name of the target - """ - super().__init__(parent_builder) - self.name = name - self.runnables = [] - - def set_runnables(self, *runnables: 'BuildTarget.Runnable') -> Any: - """ - Sets one or several `Runnable` objects to be run by the target. - - :param runnables: The `Runnable` objects to be set - :return: The builder, this builder has been created from - """ - self.runnables = list(runnables) - return self.parent_builder - - def _build(self, build_unit: BuildUnit) -> Target: - return BuildTarget(self.name, self.dependencies, self.runnables, build_unit) - - def __init__(self, name: str, dependencies: Set[Target.Dependency], runnables: List[Runnable], - build_unit: BuildUnit): - """ - :param name: The name of the target or None, if the target does not have a name - :param dependencies: The dependencies of the target - :param runnables: The `BuildTarget.Runnable` to the be run by the target - :param build_unit: The `BuildUnit`, the target belongs to - """ - super().__init__(name, dependencies) - self.runnables = runnables - self.build_unit = build_unit - - def register(self, environment: Environment, module_registry: ModuleRegistry) -> Any: - - def action(): - for runnable in self.runnables: - runnable.run(self.build_unit, module_registry) - - input_files = reduce(lambda aggr, runnable: runnable.get_input_files(module_registry), self.runnables, []) - source = (input_files if len(input_files) > 1 else input_files[0]) if input_files else None - output_files = reduce(lambda aggr, runnable: runnable.get_output_files(module_registry), self.runnables, []) - target = (output_files if len(output_files) > 1 else output_files[0]) if output_files else None - - if target: - return environment.Depends( - environment.Alias(self.name, None, None), - environment.Command(target, source, action=lambda **_: action()), - ) - - return environment.AlwaysBuild(environment.Alias(self.name, None, action=lambda **_: action())) - - def get_clean_files(self, module_registry: ModuleRegistry) -> Optional[List[str]]: - return reduce(lambda aggr, runnable: runnable.get_clean_files(module_registry), self.runnables, []) - - -class PhonyTarget(Target): - """ - A phony target, which executes a certain action and does not produce any output files. - """ - - Function = Callable[[], None] - - class Runnable(ABC): - """ - An abstract base class for all classes that can be run via a phony target. - """ - - @abstractmethod - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - """ - Must be implemented by subclasses in order to run the target. - - :param build_unit: The build unit, the target belongs to - :param modules: A `ModuleRegistry` that can be used by the target for looking up modules - """ - - class Builder(Target.Builder): - """ - A builder that allows to configure and create phony targets. - """ - - def __init__(self, parent_builder: Any, name: str): - """ - :param parent_builder: The builder, this builder has been created from - :param name: The name of the target - """ - super().__init__(parent_builder) - self.name = name - self.functions = [] - self.runnables = [] - - def nop(self) -> Any: - """ - Instructs the target to not execute any action. - - :return: The `TargetBuilder`, this builder has been created from - """ - return self.parent_builder - - def set_functions(self, *functions: 'PhonyTarget.Function') -> Any: - """ - Sets one or several functions to be run by the target. - - :param functions: The functions to be set - :return: The builder, this builder has been created from - """ - self.functions = list(functions) - return self.parent_builder - - def set_runnables(self, *runnables: 'PhonyTarget.Runnable') -> Any: - """ - Sets one or several `Runnable` objects to be run by the target. - - :param runnables: The `Runnable` objects to be set - :return: The builder, this builder has been created from - """ - self.runnables = list(runnables) - return self.parent_builder - - def _build(self, build_unit: BuildUnit) -> Target: - - def action(module_registry: ModuleRegistry): - for function in self.functions: - function() - - for runnable in self.runnables: - runnable.run(build_unit, module_registry) - - return PhonyTarget(self.name, self.dependencies, action) - - def __init__(self, name: str, dependencies: Set[Target.Dependency], action: Callable[[ModuleRegistry], None]): - """ - :param name: The name of the target - :param dependencies: The dependencies of the target - :param action: The action to be executed by the target - """ - super().__init__(name, dependencies) - self.action = action - - def register(self, environment: Environment, module_registry: ModuleRegistry) -> Any: - return environment.AlwaysBuild( - environment.Alias(self.name, None, action=lambda **_: self.action(module_registry)), - ) - - - -class TargetBuilder: - """ - A builder that allows to configure and create multiple targets. - """ - - def __init__(self, build_unit: BuildUnit = BuildUnit()): - """ - :param build_unit: The build unit, the targets belong to - """ - self.build_unit = build_unit - self.target_builders = [] - - def add_build_target(self, name: str) -> BuildTarget.Builder: - """ - Adds a build target. - - :param name: The name of the target - :return: A `BuildTarget.Builder` that allows to configure the target - """ - target_builder = BuildTarget.Builder(self, name) - self.target_builders.append(target_builder) - return target_builder - - def add_phony_target(self, name: str) -> PhonyTarget.Builder: - """ - Adds a phony target. - - :param name: The name of the target - :return: A `PhonyTarget.Builder` that allows to configure the target - """ - target_builder = PhonyTarget.Builder(self, name) - self.target_builders.append(target_builder) - return target_builder - - def build(self) -> List[Target]: - """ - Creates and returns the targets that have been configured via the builder. - - :return: A list that stores the targets that have been created - """ - return reduce(lambda aggr, builder: aggr + builder.build(self.build_unit), self.target_builders, []) - - -class TargetRegistry: - """ - Allows to register targets. - """ - - def __register_scons_targets(self) -> Dict[str, Any]: - scons_targets_by_name = {} - - for target_name, target in self.targets_by_name.items(): - scons_targets_by_name[target_name] = target.register(self.environment, self.module_registry) - - return scons_targets_by_name - - def __register_scons_dependencies(self, scons_targets_by_name: Dict[str, Any]): - for target_name, target in self.targets_by_name.items(): - scons_target = scons_targets_by_name[target_name] - scons_dependencies = [] - - for dependency in target.dependencies: - try: - scons_dependencies.append(scons_targets_by_name[dependency.target_name]) - except KeyError: - print('Dependency "' + dependency.target_name + '" of target "' + target_name - + '" has not been registered') - sys.exit(-1) - - if scons_dependencies: - self.environment.Depends(scons_target, scons_dependencies) - - def __get_parent_targets(self, *target_names: str) -> Set[str]: - result = set() - parent_targets = reduce(lambda aggr, target_name: aggr | self.parent_targets_by_name.get(target_name, set()), - target_names, set()) - - if parent_targets: - result.update(self.__get_parent_targets(*parent_targets)) - - result.update(parent_targets) - return result - - def __register_scons_clean_targets(self, scons_targets_by_name: Dict[str, Any]): - if self.environment.GetOption('clean'): - for target_name, target in self.targets_by_name.items(): - parent_targets = {target_name} | self.__get_parent_targets(target_name) - - if not COMMAND_LINE_TARGETS or reduce( - lambda aggr, parent_target: aggr or parent_target in COMMAND_LINE_TARGETS, parent_targets, - False): - clean_files = target.get_clean_files(self.module_registry) - - if clean_files: - clean_targets = [scons_targets_by_name[parent_target] for parent_target in parent_targets] - self.environment.Clean(clean_targets, clean_files) - - def __init__(self, module_registry: ModuleRegistry): - """ - :param module_registry: The `ModuleRegistry` that should be used by targets for looking up modules - """ - self.environment = Environment() - self.module_registry = module_registry - self.targets_by_name = {} - self.parent_targets_by_name = {} - - def add_target(self, target: Target): - """ - Adds a new target to be registered. - - :param target: The target to be added - """ - self.targets_by_name[target.name] = target - - for dependency in target.dependencies: - if dependency.clean_dependency: - parent_targets = self.parent_targets_by_name.setdefault(dependency.target_name, set()) - parent_targets.add(target.name) - - def register(self): - """ - Registers all targets that have previously been added. - """ - scons_targets_by_name = self.__register_scons_targets() - self.__register_scons_dependencies(scons_targets_by_name) - self.__register_scons_clean_targets(scons_targets_by_name) - - @property - def target_names(self) -> Set[str]: - """ - A set that contains the names of all targets that have previously been added. - """ - return set(self.targets_by_name.keys()) From 5c54d114015d81cd8f204554f2cd92bbf696a6be Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Tue, 10 Dec 2024 03:16:42 +0100 Subject: [PATCH 081/114] Update file paths mentioned in the documentation. --- doc/developer_guide/coding_standards.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/developer_guide/coding_standards.md b/doc/developer_guide/coding_standards.md index b49e14469..d27ef9878 100644 --- a/doc/developer_guide/coding_standards.md +++ b/doc/developer_guide/coding_standards.md @@ -65,8 +65,8 @@ The unit and integration tests are run automatically via {ref}`Continuous Integr We aim to enforce a consistent code style across the entire project. For this purpose, we employ the following tools: -- For formatting the C++ code, we use [clang-format](https://clang.llvm.org/docs/ClangFormat.html). The desired C++ code style is defined in the file `.clang-format` in the project's root directory. In addition, [cpplint](https://github.com/cpplint/cpplint) is used for static code analysis. It uses the configuration file `CPPLINT.cfg`. -- We use [YAPF](https://github.com/google/yapf) to enforce the Python code style defined in the file `.style.yapf`. In addition, [isort](https://github.com/PyCQA/isort) is used to keep the ordering of imports in Python and Cython source files consistent according to the configuration file `.isort.cfg` and [pylint](https://pylint.org/) is used to check for common issues in the Python code according to the configuration file `.pylintrc`. +- For formatting the C++ code, we use [clang-format](https://clang.llvm.org/docs/ClangFormat.html). The desired C++ code style is defined in the file `build_system/code_style/cpp/.clang-format`. In addition, [cpplint](https://github.com/cpplint/cpplint) is used for static code analysis. It is configured according to the `.cpplint.cfg` files located in the directory `cpp` and its subdirectories. +- We use [YAPF](https://github.com/google/yapf) to enforce the Python code style defined in the file `build_system/code_style/python/.style.yapf`. In addition, [isort](https://github.com/PyCQA/isort) is used to keep the ordering of imports in Python and Cython source files consistent according to the configuration file `build_system/code_style/python/.isort.cfg` and [pylint](https://pylint.org/) is used to check for common issues in the Python code according to the configuration file `build_system/code_style/python/.pylintrc`. - For applying a consistent style to Markdown files, including those used for writing the documentation, we use [mdformat](https://github.com/executablebooks/mdformat). - We apply [yamlfix](https://github.com/lyz-code/yamlfix) to YAML files to enforce the code style defined in the file `build_system/code_style/yaml/.yamlfix.toml`. From 8a0db5eeeb5748c4406bb73e258858fcac12c656 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Tue, 10 Dec 2024 03:17:43 +0100 Subject: [PATCH 082/114] Fix name of environment variable mentioned in the documentation. --- doc/developer_guide/coding_standards.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/developer_guide/coding_standards.md b/doc/developer_guide/coding_standards.md index d27ef9878..64badb025 100644 --- a/doc/developer_guide/coding_standards.md +++ b/doc/developer_guide/coding_standards.md @@ -28,23 +28,23 @@ To be able to detect problems with the project's source code early during develo ``` ```` -This will result in all tests being run and their results being reported. If the execution should be aborted as soon as a single test fails, the environment variable `SKIP_EARLY` can be used as shown below: +This will result in all tests being run and their results being reported. If the execution should be aborted as soon as a single test fails, the environment variable `FAIL_FAST` can be used as shown below: ````{tab} Linux ```text - SKIP_EARLY=true ./build tests + FAIL_FAST=true ./build tests ``` ```` ````{tab} macOS ```text - SKIP_EARLY=true ./build tests + FAIL_FAST=true ./build tests ``` ```` ````{tab} Windows ```text - $env:SKIP_EARLY = "true" + $env:FAIL_FAST = "true" build.bat tests ``` ```` From cf4e32529a28accd26cc78f8edd0e02c17581957 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 11 Dec 2024 01:10:45 +0100 Subject: [PATCH 083/114] Pass individual modules to functions of the class PhonyTarget.Runnable and BuildTarget.Runnable. --- build_system/code_style/cpp/targets.py | 24 +++-- build_system/code_style/markdown/targets.py | 22 ++-- build_system/code_style/python/targets.py | 48 +++++---- build_system/code_style/yaml/targets.py | 22 ++-- build_system/compilation/cpp/targets.py | 48 +++++---- build_system/compilation/cython/targets.py | 52 ++++----- build_system/dependencies/github/__init__.py | 6 ++ build_system/dependencies/github/actions.py | 20 ++-- build_system/dependencies/github/modules.py | 42 ++++++++ build_system/dependencies/github/targets.py | 19 +++- build_system/dependencies/python/targets.py | 21 ++-- build_system/packaging/__init__.py | 26 ++--- build_system/packaging/modules.py | 9 +- build_system/packaging/targets.py | 72 ++++++------- build_system/testing/cpp/targets.py | 10 +- build_system/testing/python/targets.py | 10 +- build_system/util/targets.py | 108 +++++++++++++------ 17 files changed, 333 insertions(+), 226 deletions(-) create mode 100644 build_system/dependencies/github/modules.py diff --git a/build_system/code_style/cpp/targets.py b/build_system/code_style/cpp/targets.py index 3d8bcffa6..3761da476 100644 --- a/build_system/code_style/cpp/targets.py +++ b/build_system/code_style/cpp/targets.py @@ -8,7 +8,7 @@ from code_style.modules import CodeModule from util.files import FileType from util.log import Log -from util.modules import ModuleRegistry +from util.modules import Module from util.targets import PhonyTarget from util.units import BuildUnit @@ -20,11 +20,13 @@ class CheckCppCodeStyle(PhonyTarget.Runnable): Checks if C++ source files adhere to the code style definitions. If this is not the case, an error is raised. """ - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - for module in modules.lookup(MODULE_FILTER): - Log.info('Checking C++ code style in directory "%s"...', module.root_directory) - ClangFormat(build_unit, module).run() - CppLint(build_unit, module).run() + def __init__(self): + super().__init__(MODULE_FILTER) + + def run(self, build_unit: BuildUnit, module: Module): + Log.info('Checking C++ code style in directory "%s"...', module.root_directory) + ClangFormat(build_unit, module).run() + CppLint(build_unit, module).run() class EnforceCppCodeStyle(PhonyTarget.Runnable): @@ -32,7 +34,9 @@ class EnforceCppCodeStyle(PhonyTarget.Runnable): Enforces C++ source files to adhere to the code style definitions. """ - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - for module in modules.lookup(MODULE_FILTER): - Log.info('Formatting C++ code in directory "%s"...', module.root_directory) - ClangFormat(build_unit, module, enforce_changes=True).run() + def __init__(self): + super().__init__(MODULE_FILTER) + + def run(self, build_unit: BuildUnit, module: Module): + Log.info('Formatting C++ code in directory "%s"...', module.root_directory) + ClangFormat(build_unit, module, enforce_changes=True).run() diff --git a/build_system/code_style/markdown/targets.py b/build_system/code_style/markdown/targets.py index 16f9925e9..d58042721 100644 --- a/build_system/code_style/markdown/targets.py +++ b/build_system/code_style/markdown/targets.py @@ -7,7 +7,7 @@ from code_style.modules import CodeModule from util.files import FileType from util.log import Log -from util.modules import ModuleRegistry +from util.modules import Module from util.targets import PhonyTarget from util.units import BuildUnit @@ -19,10 +19,12 @@ class CheckMarkdownCodeStyle(PhonyTarget.Runnable): Checks if Markdown files adhere to the code style definitions. If this is not the case, an error is raised. """ - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - for module in modules.lookup(MODULE_FILTER): - Log.info('Checking Markdown code style in the directory "%s"...', module.root_directory) - MdFormat(build_unit, module).run() + def __init__(self): + super().__init__(MODULE_FILTER) + + def run(self, build_unit: BuildUnit, module: Module): + Log.info('Checking Markdown code style in the directory "%s"...', module.root_directory) + MdFormat(build_unit, module).run() class EnforceMarkdownCodeStyle(PhonyTarget.Runnable): @@ -30,7 +32,9 @@ class EnforceMarkdownCodeStyle(PhonyTarget.Runnable): Enforces Markdown files to adhere to the code style definitions. """ - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - for module in modules.lookup(MODULE_FILTER): - Log.info('Formatting Markdown files in the directory "%s"...', module.root_directory) - MdFormat(build_unit, module, enforce_changes=True).run() + def __init__(self): + super().__init__(MODULE_FILTER) + + def run(self, build_unit: BuildUnit, module: Module): + Log.info('Formatting Markdown files in the directory "%s"...', module.root_directory) + MdFormat(build_unit, module, enforce_changes=True).run() diff --git a/build_system/code_style/python/targets.py b/build_system/code_style/python/targets.py index fa62c6aa8..eabf49c30 100644 --- a/build_system/code_style/python/targets.py +++ b/build_system/code_style/python/targets.py @@ -9,7 +9,7 @@ from code_style.python.yapf import Yapf from util.files import FileType from util.log import Log -from util.modules import ModuleRegistry +from util.modules import Module from util.targets import PhonyTarget from util.units import BuildUnit @@ -23,12 +23,14 @@ class CheckPythonCodeStyle(PhonyTarget.Runnable): Checks if Python source files adhere to the code style definitions. If this is not the case, an error is raised. """ - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - for module in modules.lookup(PYTHON_MODULE_FILTER): - Log.info('Checking Python code style in directory "%s"...', module.root_directory) - ISort(build_unit, module).run() - Yapf(build_unit, module).run() - PyLint(build_unit, module).run() + def __init__(self): + super().__init__(PYTHON_MODULE_FILTER) + + def run(self, build_unit: BuildUnit, module: Module): + Log.info('Checking Python code style in directory "%s"...', module.root_directory) + ISort(build_unit, module).run() + Yapf(build_unit, module).run() + PyLint(build_unit, module).run() class EnforcePythonCodeStyle(PhonyTarget.Runnable): @@ -36,11 +38,13 @@ class EnforcePythonCodeStyle(PhonyTarget.Runnable): Enforces Python source files to adhere to the code style definitions. """ - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - for module in modules.lookup(PYTHON_MODULE_FILTER): - Log.info('Formatting Python code in directory "%s"...', module.root_directory) - ISort(build_unit, module, enforce_changes=True).run() - Yapf(build_unit, module, enforce_changes=True).run() + def __init__(self): + super().__init__(PYTHON_MODULE_FILTER) + + def run(self, build_unit: BuildUnit, module: Module): + Log.info('Formatting Python code in directory "%s"...', module.root_directory) + ISort(build_unit, module, enforce_changes=True).run() + Yapf(build_unit, module, enforce_changes=True).run() class CheckCythonCodeStyle(PhonyTarget.Runnable): @@ -48,10 +52,12 @@ class CheckCythonCodeStyle(PhonyTarget.Runnable): Checks if Cython source files adhere to the code style definitions. If this is not the case, an error is raised. """ - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - for module in modules.lookup(CYTHON_MODULE_FILTER): - Log.info('Checking Cython code style in directory "%s"...', module.root_directory) - ISort(build_unit, module).run() + def __init__(self): + super().__init__(CYTHON_MODULE_FILTER) + + def run(self, build_unit: BuildUnit, module: Module): + Log.info('Checking Cython code style in directory "%s"...', module.root_directory) + ISort(build_unit, module).run() class EnforceCythonCodeStyle(PhonyTarget.Runnable): @@ -59,7 +65,9 @@ class EnforceCythonCodeStyle(PhonyTarget.Runnable): Enforces Cython source files to adhere to the code style definitions. """ - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - for module in modules.lookup(CYTHON_MODULE_FILTER): - Log.info('Formatting Cython code in directory "%s"...', module.root_directory) - ISort(build_unit, module, enforce_changes=True).run() + def __init__(self): + super().__init__(CYTHON_MODULE_FILTER) + + def run(self, build_unit: BuildUnit, module: Module): + Log.info('Formatting Cython code in directory "%s"...', module.root_directory) + ISort(build_unit, module, enforce_changes=True).run() diff --git a/build_system/code_style/yaml/targets.py b/build_system/code_style/yaml/targets.py index c1ced0145..b16ad7c1b 100644 --- a/build_system/code_style/yaml/targets.py +++ b/build_system/code_style/yaml/targets.py @@ -7,7 +7,7 @@ from code_style.yaml.yamlfix import YamlFix from util.files import FileType from util.log import Log -from util.modules import ModuleRegistry +from util.modules import Module from util.targets import PhonyTarget from util.units import BuildUnit @@ -19,10 +19,12 @@ class CheckYamlCodeStyle(PhonyTarget.Runnable): Checks if YAML files adhere to the code style definitions. If this is not the case, an error is raised. """ - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - for module in modules.lookup(MODULE_FILTER): - Log.info('Checking YAML files in the directory "%s"...', module.root_directory) - YamlFix(build_unit, module).run() + def __init__(self): + super().__init__(MODULE_FILTER) + + def run(self, build_unit: BuildUnit, module: Module): + Log.info('Checking YAML files in the directory "%s"...', module.root_directory) + YamlFix(build_unit, module).run() class EnforceYamlCodeStyle(PhonyTarget.Runnable): @@ -30,7 +32,9 @@ class EnforceYamlCodeStyle(PhonyTarget.Runnable): Enforces YAML files to adhere to the code style definitions. """ - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - for module in modules.lookup(MODULE_FILTER): - Log.info('Formatting YAML files in the directory "%s"...', module.root_directory) - YamlFix(build_unit, module, enforce_changes=True).run() + def __init__(self): + super().__init__(MODULE_FILTER) + + def run(self, build_unit: BuildUnit, module: Module): + Log.info('Formatting YAML files in the directory "%s"...', module.root_directory) + YamlFix(build_unit, module, enforce_changes=True).run() diff --git a/build_system/compilation/cpp/targets.py b/build_system/compilation/cpp/targets.py index 2c2663f45..3c9d68918 100644 --- a/build_system/compilation/cpp/targets.py +++ b/build_system/compilation/cpp/targets.py @@ -3,7 +3,6 @@ Implements targets for compiling C++ code. """ -from functools import reduce from typing import List from compilation.build_options import BuildOptions, EnvBuildOption @@ -11,7 +10,7 @@ from compilation.modules import CompilationModule from util.files import FileType from util.log import Log -from util.modules import ModuleRegistry +from util.modules import Module from util.targets import BuildTarget, PhonyTarget from util.units import BuildUnit @@ -29,16 +28,18 @@ class SetupCpp(BuildTarget.Runnable): Sets up the build system for compiling C++ code. """ - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - for module in modules.lookup(MODULE_FILTER): - MesonSetup(build_unit, module, build_options=BUILD_OPTIONS).run() + def __init__(self): + super().__init__(MODULE_FILTER) - def get_output_files(self, modules: ModuleRegistry) -> List[str]: - return [module.build_directory for module in modules.lookup(MODULE_FILTER)] + def run(self, build_unit: BuildUnit, module: Module): + MesonSetup(build_unit, module, build_options=BUILD_OPTIONS).run() - def get_clean_files(self, modules: ModuleRegistry) -> List[str]: - Log.info('Removing C++ build files...') - return super().get_clean_files(modules) + def get_output_files(self, module: Module) -> List[str]: + return [module.build_directory] + + def get_clean_files(self, module: Module) -> List[str]: + Log.info('Removing C++ build files from directory "%s"...', module.root_directory) + return super().get_clean_files(module) class CompileCpp(PhonyTarget.Runnable): @@ -46,12 +47,13 @@ class CompileCpp(PhonyTarget.Runnable): Compiles C++ code. """ - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - Log.info('Compiling C++ code...') + def __init__(self): + super().__init__(MODULE_FILTER) - for module in modules.lookup(MODULE_FILTER): - MesonConfigure(build_unit, module, BUILD_OPTIONS).run() - MesonCompile(build_unit, module).run() + def run(self, build_unit: BuildUnit, module: Module): + Log.info('Compiling C++ code in directory "%s"...', module.root_directory) + MesonConfigure(build_unit, module, BUILD_OPTIONS).run() + MesonCompile(build_unit, module).run() class InstallCpp(BuildTarget.Runnable): @@ -59,13 +61,13 @@ class InstallCpp(BuildTarget.Runnable): Installs shared libraries into the source tree. """ - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - Log.info('Installing shared libraries into source tree...') + def __init__(self): + super().__init__(MODULE_FILTER) - for module in modules.lookup(MODULE_FILTER): - MesonInstall(build_unit, module).run() + def run(self, build_unit: BuildUnit, module: Module): + Log.info('Installing shared libraries from directory "%s" into source tree...', module.root_directory) + MesonInstall(build_unit, module).run() - def get_clean_files(self, modules: ModuleRegistry) -> List[str]: - Log.info('Removing shared libraries from source tree...') - compilation_modules = modules.lookup(MODULE_FILTER) - return reduce(lambda aggr, module: aggr + module.find_installed_files(), compilation_modules, []) + def get_clean_files(self, module: Module) -> List[str]: + Log.info('Removing shared libraries installed from directory "%s" from source tree...', module.root_directory) + return module.find_installed_files() diff --git a/build_system/compilation/cython/targets.py b/build_system/compilation/cython/targets.py index 0fe7ef37a..91f56bbc3 100644 --- a/build_system/compilation/cython/targets.py +++ b/build_system/compilation/cython/targets.py @@ -3,7 +3,6 @@ Implements targets for compiling Cython code. """ -from functools import reduce from typing import List from compilation.build_options import BuildOptions, EnvBuildOption @@ -11,7 +10,7 @@ from compilation.modules import CompilationModule from util.files import FileType from util.log import Log -from util.modules import ModuleRegistry +from util.modules import Module from util.targets import BuildTarget, PhonyTarget from util.units import BuildUnit @@ -26,18 +25,20 @@ class SetupCython(BuildTarget.Runnable): Sets up the build system for compiling the Cython code. """ - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - for module in modules.lookup(MODULE_FILTER): - MesonSetup(build_unit, module) \ - .add_dependencies('cython') \ - .run() + def __init__(self): + super().__init__(MODULE_FILTER) - def get_output_files(self, modules: ModuleRegistry) -> List[str]: - return [module.build_directory for module in modules.lookup(MODULE_FILTER)] + def run(self, build_unit: BuildUnit, module: Module): + MesonSetup(build_unit, module) \ + .add_dependencies('cython') \ + .run() - def get_clean_files(self, modules: ModuleRegistry) -> List[str]: - Log.info('Removing Cython build files...') - return super().get_clean_files(modules) + def get_output_files(self, module: Module) -> List[str]: + return [module.build_directory] + + def get_clean_files(self, module: Module) -> List[str]: + Log.info('Removing Cython build files from directory "%s"...', module.root_directory) + return super().get_clean_files(module) class CompileCython(PhonyTarget.Runnable): @@ -45,12 +46,13 @@ class CompileCython(PhonyTarget.Runnable): Compiles the Cython code. """ - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - Log.info('Compiling Cython code...') + def __init__(self): + super().__init__(MODULE_FILTER) - for module in modules.lookup(MODULE_FILTER): - MesonConfigure(build_unit, module, build_options=BUILD_OPTIONS) - MesonCompile(build_unit, module).run() + def run(self, build_unit: BuildUnit, module: Module): + Log.info('Compiling Cython code in directory "%s"...', module.root_directory) + MesonConfigure(build_unit, module, build_options=BUILD_OPTIONS) + MesonCompile(build_unit, module).run() class InstallCython(BuildTarget.Runnable): @@ -58,13 +60,13 @@ class InstallCython(BuildTarget.Runnable): Installs extension modules into the source tree. """ - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - Log.info('Installing extension modules into source tree...') + def __init__(self): + super().__init__(MODULE_FILTER) - for module in modules.lookup(MODULE_FILTER): - MesonInstall(build_unit, module).run() + def run(self, build_unit: BuildUnit, module: Module): + Log.info('Installing extension modules from directory "%s" into source tree...', module.root_directory) + MesonInstall(build_unit, module).run() - def get_clean_files(self, modules: ModuleRegistry) -> List[str]: - Log.info('Removing extension modules from source tree...') - compilation_modules = modules.lookup(MODULE_FILTER) - return reduce(lambda aggr, module: aggr + module.find_installed_files(), compilation_modules, []) + def get_clean_files(self, module: Module) -> List[str]: + Log.info('Removing extension modules installed from directory "%s" from source tree...', module.root_directory) + return module.find_installed_files() diff --git a/build_system/dependencies/github/__init__.py b/build_system/dependencies/github/__init__.py index f625be702..60e8b3783 100644 --- a/build_system/dependencies/github/__init__.py +++ b/build_system/dependencies/github/__init__.py @@ -3,7 +3,9 @@ Defines targets for updating the project's GitHub Actions. """ +from dependencies.github.modules import GithubWorkflowModule from dependencies.github.targets import CheckGithubActions, UpdateGithubActions +from util.paths import Project from util.targets import PhonyTarget, TargetBuilder from util.units import BuildUnit @@ -11,3 +13,7 @@ .add_phony_target('check_github_actions').set_runnables(CheckGithubActions()) \ .add_phony_target('update_github_actions').set_runnables(UpdateGithubActions()) \ .build() + +MODULES = [ + GithubWorkflowModule(root_directory=Project.Github.root_directory), +] diff --git a/build_system/dependencies/github/actions.py b/build_system/dependencies/github/actions.py index 7a1f37b07..883ddec72 100644 --- a/build_system/dependencies/github/actions.py +++ b/build_system/dependencies/github/actions.py @@ -5,13 +5,13 @@ """ from dataclasses import dataclass, replace from functools import cached_property, reduce -from os import environ, path +from os import environ from typing import Dict, List, Optional, Set +from dependencies.github.modules import GithubWorkflowModule from dependencies.github.pygithub import GithubApi from dependencies.github.pyyaml import YamlFile from util.env import get_env -from util.files import FileSearch, FileType from util.log import Log from util.units import BuildUnit @@ -237,8 +237,9 @@ def __eq__(self, other: 'WorkflowUpdater.UpdatedAction') -> bool: def __hash__(self) -> int: return hash(self.updated) - def __get_github_token(self) -> Optional[str]: - github_token = get_env(environ, self.ENV_GITHUB_TOKEN) + @staticmethod + def __get_github_token() -> Optional[str]: + github_token = get_env(environ, WorkflowUpdater.ENV_GITHUB_TOKEN) if not github_token: Log.warning('No GitHub API token is set. You can specify it via the environment variable %s.', @@ -273,14 +274,15 @@ def __get_latest_action_version(self, action: Action) -> ActionVersion: return latest_version - def __init__(self, build_unit: BuildUnit, workflow_directory: str = path.join('.github', 'workflows')): + def __init__(self, build_unit: BuildUnit, module: GithubWorkflowModule): """ - :param build_unit: The build unit from which workflow definition files should be read - :param workflow_directory: The path to the directory where the workflow definition files are located + :param build_unit: The build unit from which workflow definition files should be read + :param module: The module, that contains the workflow definition files """ self.build_unit = build_unit - self.workflow_directory = workflow_directory + self.module = module self.version_cache = {} + self.github_token = WorkflowUpdater.__get_github_token() @cached_property def workflows(self) -> Set[Workflow]: @@ -289,7 +291,7 @@ def workflows(self) -> Set[Workflow]: """ workflows = set() - for workflow_file in FileSearch().filter_by_file_type(FileType.yaml()).list(self.workflow_directory): + for workflow_file in self.module.find_workflow_files(): Log.info('Searching for GitHub Actions in workflow "%s"...', workflow_file) workflows.add(Workflow(self.build_unit, workflow_file)) diff --git a/build_system/dependencies/github/modules.py b/build_system/dependencies/github/modules.py new file mode 100644 index 000000000..d2f6ef054 --- /dev/null +++ b/build_system/dependencies/github/modules.py @@ -0,0 +1,42 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that provide access to GitHub workflows tha belong to individual modules. +""" +from typing import List + +from util.files import FileSearch, FileType +from util.modules import Module + + +class GithubWorkflowModule(Module): + """ + A module that contains GitHub workflows. + """ + + class Filter(Module.Filter): + """ + A filter that matches modules that contain GitHub workflows. + """ + + def matches(self, module: Module) -> bool: + return isinstance(module, GithubWorkflowModule) + + def __init__(self, root_directory: str, workflow_file_search: FileSearch = FileSearch().set_recursive(True)): + """ + :param root_directory: The path to the module's root directory + :param workflow_file_search: The `FileSearch` that should be used to search for workflow definition files + """ + self.root_directory = root_directory + self.workflow_file_search = workflow_file_search + + def find_workflow_files(self) -> List[str]: + """ + Finds and returns all workflow definition files that belong to the module. + + :return: A list that contains the paths of the workflow definition files that have been found + """ + return self.workflow_file_search.filter_by_file_type(FileType.yaml()).list(self.root_directory) + + def __str__(self) -> str: + return 'GithubWorkflowModule {root_directory="' + self.root_directory + '"}' diff --git a/build_system/dependencies/github/targets.py b/build_system/dependencies/github/targets.py index 8253b5be6..8181a902c 100644 --- a/build_system/dependencies/github/targets.py +++ b/build_system/dependencies/github/targets.py @@ -4,20 +4,26 @@ Implements targets for updating the project's GitHub Actions. """ from dependencies.github.actions import WorkflowUpdater +from dependencies.github.modules import GithubWorkflowModule from dependencies.table import Table from util.log import Log -from util.modules import ModuleRegistry +from util.modules import Module from util.targets import PhonyTarget from util.units import BuildUnit +MODULE_FILTER = GithubWorkflowModule.Filter() + class CheckGithubActions(PhonyTarget.Runnable): """ Prints all outdated Actions used in the project's GitHub workflows. """ - def run(self, build_unit: BuildUnit, _: ModuleRegistry): - outdated_workflows = WorkflowUpdater(build_unit).find_outdated_workflows() + def __init__(self): + super().__init__(MODULE_FILTER) + + def run(self, build_unit: BuildUnit, module: Module): + outdated_workflows = WorkflowUpdater(build_unit, module).find_outdated_workflows() if outdated_workflows: table = Table(build_unit, 'Workflow', 'Action', 'Current version', 'Latest version') @@ -38,8 +44,11 @@ class UpdateGithubActions(PhonyTarget.Runnable): Updates and prints all outdated Actions used in the project's GitHub workflows. """ - def run(self, build_unit: BuildUnit, _: ModuleRegistry): - updated_workflows = WorkflowUpdater(build_unit).update_outdated_workflows() + def __init__(self): + super().__init__(MODULE_FILTER) + + def run(self, build_unit: BuildUnit, module: Module): + updated_workflows = WorkflowUpdater(build_unit, module).update_outdated_workflows() if updated_workflows: table = Table(build_unit, 'Workflow', 'Action', 'Previous version', 'Updated version') diff --git a/build_system/dependencies/python/targets.py b/build_system/dependencies/python/targets.py index 64c624d9d..7161fbc8c 100644 --- a/build_system/dependencies/python/targets.py +++ b/build_system/dependencies/python/targets.py @@ -4,12 +4,13 @@ Implements targets for installing runtime requirements that are required by the project's source code. """ from functools import reduce +from typing import List from dependencies.python.modules import DependencyType, PythonDependencyModule from dependencies.python.pip import PipList from dependencies.table import Table from util.log import Log -from util.modules import ModuleRegistry +from util.modules import Module from util.targets import PhonyTarget from util.units import BuildUnit @@ -19,10 +20,11 @@ class InstallRuntimeDependencies(PhonyTarget.Runnable): Installs all runtime dependencies that are required by the project's source code. """ - def run(self, _: BuildUnit, modules: ModuleRegistry): - dependency_modules = modules.lookup(PythonDependencyModule.Filter(DependencyType.RUNTIME)) - requirements_files = reduce(lambda aggr, module: aggr + module.find_requirements_files(), dependency_modules, - []) + def __init__(self): + super().__init__(PythonDependencyModule.Filter(DependencyType.RUNTIME)) + + def run_all(self, _: BuildUnit, modules: List[Module]): + requirements_files = reduce(lambda aggr, module: aggr + module.find_requirements_files(), modules, []) PipList(*requirements_files).install_all_packages() @@ -31,10 +33,11 @@ class CheckPythonDependencies(PhonyTarget.Runnable): Installs all Python dependencies used by the project and checks for outdated ones. """ - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - dependency_modules = modules.lookup(PythonDependencyModule.Filter()) - requirements_files = reduce(lambda aggr, module: aggr + module.find_requirements_files(), dependency_modules, - []) + def __init__(self): + super().__init__(PythonDependencyModule.Filter()) + + def run_all(self, build_unit: BuildUnit, modules: List[Module]): + requirements_files = reduce(lambda aggr, module: aggr + module.find_requirements_files(), modules, []) pip = PipList(*requirements_files) Log.info('Installing all dependencies...') pip.install_all_packages() diff --git a/build_system/packaging/__init__.py b/build_system/packaging/__init__.py index a650546ab..140710511 100644 --- a/build_system/packaging/__init__.py +++ b/build_system/packaging/__init__.py @@ -16,26 +16,18 @@ INSTALL_WHEELS = 'install_wheels' +TARGETS = TargetBuilder(BuildUnit('packaging')) \ + .add_build_target(BUILD_WHEELS) \ + .depends_on(INSTALL) \ + .set_runnables(BuildPythonWheels()) \ + .add_build_target(INSTALL_WHEELS) \ + .depends_on(BUILD_WHEELS) \ + .set_runnables(InstallPythonWheels()) \ + .build() + MODULES = [ PythonPackageModule( root_directory=path.dirname(setup_file), wheel_directory_name=Project.Python.wheel_directory_name, ) for setup_file in Project.Python.file_search().filter_by_name('setup.py').list(Project.Python.root_directory) ] - -TARGETS = TargetBuilder(BuildUnit('packaging')) \ - .add_phony_target(BUILD_WHEELS) \ - .depends_on(INSTALL) \ - .depends_on_build_targets( - MODULES, - lambda module, target_builder: target_builder.set_runnables(BuildPythonWheels(module.root_directory)) - ) \ - .nop() \ - .add_phony_target(INSTALL_WHEELS) \ - .depends_on(BUILD_WHEELS) \ - .depends_on_phony_targets( - MODULES, - lambda module, target_builder: target_builder.set_runnables(InstallPythonWheels(module.root_directory)) - ) \ - .nop() \ - .build() diff --git a/build_system/packaging/modules.py b/build_system/packaging/modules.py index 2d4fe43f9..f3ba8d993 100644 --- a/build_system/packaging/modules.py +++ b/build_system/packaging/modules.py @@ -20,15 +20,8 @@ class Filter(Module.Filter): A filter that matches modules that contain Python code that can be built as wheel packages. """ - def __init__(self, *root_directories: str): - """ - :param root_directories: The root directories of the modules to be matched - """ - self.root_directories = set(root_directories) - def matches(self, module: Module) -> bool: - return isinstance(module, PythonPackageModule) and (not self.root_directories - or module.root_directory in self.root_directories) + return isinstance(module, PythonPackageModule) def __init__(self, root_directory: str, wheel_directory_name: str): """ diff --git a/build_system/packaging/targets.py b/build_system/packaging/targets.py index 2324fc8a7..6dacfc733 100644 --- a/build_system/packaging/targets.py +++ b/build_system/packaging/targets.py @@ -3,7 +3,6 @@ Implements targets for building and installing wheel packages. """ -from functools import reduce from typing import List from packaging.build import Build @@ -11,69 +10,60 @@ from packaging.pip import PipInstallWheel from util.files import DirectorySearch, FileType from util.log import Log -from util.modules import ModuleRegistry +from util.modules import Module from util.paths import Project -from util.targets import BuildTarget, PhonyTarget +from util.targets import BuildTarget from util.units import BuildUnit +MODULE_FILTER = PythonPackageModule.Filter() + class BuildPythonWheels(BuildTarget.Runnable): """ Builds Python wheel packages. """ - def __init__(self, root_directory: str): - """ - :param root_directory: The root directory of the module for which Python wheel packages should be built - """ - self.module_filter = PythonPackageModule.Filter(root_directory) + def __init__(self): + super().__init__(MODULE_FILTER) - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - for module in modules.lookup(self.module_filter): - Log.info('Building Python wheels for directory "%s"...', module.root_directory) - Build(build_unit, module).run() + def run(self, build_unit: BuildUnit, module: Module): + Log.info('Building Python wheels for directory "%s"...', module.root_directory) + Build(build_unit, module).run() - def get_input_files(self, modules: ModuleRegistry) -> List[str]: - package_modules = modules.lookup(self.module_filter) + def get_input_files(self, module: Module) -> List[str]: file_search = Project.Python.file_search() \ .set_symlinks(False) \ .exclude_subdirectories_by_name(Project.Python.test_directory_name) \ .filter_by_file_type(FileType.python(), FileType.extension_module(), FileType.shared_library()) - return reduce(lambda aggr, module: aggr + file_search.list(module.root_directory), package_modules, []) + return file_search.list(module.root_directory) - def get_output_files(self, modules: ModuleRegistry) -> List[str]: - package_modules = modules.lookup(self.module_filter) - wheels = reduce(lambda aggr, module: module.find_wheels(), package_modules, []) - return wheels if wheels else [module.wheel_directory for module in package_modules] + def get_output_files(self, module: Module) -> List[str]: + return module.wheel_directory - def get_clean_files(self, modules: ModuleRegistry) -> List[str]: + def get_clean_files(self, module: Module) -> List[str]: clean_files = [] - - for module in modules.lookup(self.module_filter): - Log.info('Removing Python wheels from directory "%s"...', module.root_directory) - clean_files.append(module.wheel_directory) - clean_files.extend( - DirectorySearch() \ - .filter_by_name(Project.Python.build_directory_name) \ - .filter_by_substrings(ends_with=Project.Python.wheel_metadata_directory_suffix) \ - .list(module.root_directory) - ) - + Log.info('Removing Python wheels from directory "%s"...', module.root_directory) + clean_files.append(module.wheel_directory) + clean_files.extend( + DirectorySearch() \ + .filter_by_name(Project.Python.build_directory_name) \ + .filter_by_substrings(ends_with=Project.Python.wheel_metadata_directory_suffix) \ + .list(module.root_directory) + ) return clean_files -class InstallPythonWheels(PhonyTarget.Runnable): +class InstallPythonWheels(BuildTarget.Runnable): """ Installs Python wheel packages. """ - def __init__(self, root_directory: str): - """ - :param root_directory: The root directory of the module for which Python wheel packages should be installed - """ - self.module_filter = PythonPackageModule.Filter(root_directory) + def __init__(self): + super().__init__(MODULE_FILTER) + + def run(self, _: BuildUnit, module: Module): + Log.info('Installing Python wheels for directory "%s"...', module.root_directory) + PipInstallWheel().install_wheels(*module.find_wheels()) - def run(self, _: BuildUnit, modules: ModuleRegistry): - for module in modules.lookup(self.module_filter): - Log.info('Installing Python wheels for directory "%s"...', module.root_directory) - PipInstallWheel().install_wheels(module.find_wheels()) + def get_input_files(self, module: Module) -> List[str]: + return module.find_wheels() diff --git a/build_system/testing/cpp/targets.py b/build_system/testing/cpp/targets.py index 13eddba6e..db1b07faf 100644 --- a/build_system/testing/cpp/targets.py +++ b/build_system/testing/cpp/targets.py @@ -5,7 +5,7 @@ """ from testing.cpp.meson import MesonTest from testing.cpp.modules import CppTestModule -from util.modules import ModuleRegistry +from util.modules import Module from util.targets import BuildTarget, PhonyTarget from util.units import BuildUnit @@ -15,6 +15,8 @@ class TestCpp(PhonyTarget.Runnable): Runs automated tests for C++ code. """ - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - for module in modules.lookup(CppTestModule.Filter()): - MesonTest(build_unit, module).run() + def __init__(self): + super().__init__(CppTestModule.Filter()) + + def run(self, build_unit: BuildUnit, module: Module): + MesonTest(build_unit, module).run() diff --git a/build_system/testing/python/targets.py b/build_system/testing/python/targets.py index 9778db28d..81c7bd0f8 100644 --- a/build_system/testing/python/targets.py +++ b/build_system/testing/python/targets.py @@ -5,7 +5,7 @@ """ from testing.python.modules import PythonTestModule from testing.python.unittest import UnitTest -from util.modules import ModuleRegistry +from util.modules import Module from util.targets import PhonyTarget from util.units import BuildUnit @@ -15,6 +15,8 @@ class TestPython(PhonyTarget.Runnable): Runs automated tests for Python code. """ - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): - for module in modules.lookup(PythonTestModule.Filter()): - UnitTest(build_unit, module).run() + def __init__(self): + super().__init__(PythonTestModule.Filter()) + + def run(self, build_unit: BuildUnit, module: Module): + UnitTest(build_unit, module).run() diff --git a/build_system/util/targets.py b/build_system/util/targets.py index 1b7d19ad0..d467e500a 100644 --- a/build_system/util/targets.py +++ b/build_system/util/targets.py @@ -12,7 +12,7 @@ from util.format import format_iterable from util.io import delete_files from util.log import Log -from util.modules import ModuleRegistry +from util.modules import Module, ModuleRegistry from util.units import BuildUnit @@ -201,7 +201,7 @@ def __str__(self) -> str: class BuildTarget(Target): """ - A build target, which executes a certain action and produces one or several output files. + A build target, which produces one or several output files from given input files. """ class Runnable(ABC): @@ -209,57 +209,66 @@ class Runnable(ABC): An abstract base class for all classes that can be run via a build target. """ + def __init__(self, module_filter: Module.Filter): + """ + :param module_filter: A filter that matches the modules, the target should be applied to + """ + self.module_filter = module_filter + @abstractmethod - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + def run(self, build_unit: BuildUnit, module: Module): """ - Must be implemented by subclasses in order to run the target. + may be overridden by subclasses in order to apply the target to an individual module that matches the + filter. :param build_unit: The build unit, the target belongs to - :param modules: A `ModuleRegistry` that can be used by the target for looking up modules + :param module: The module, the target should be applied to """ - def get_input_files(self, modules: ModuleRegistry) -> List[str]: + def get_input_files(self, module: Module) -> List[str]: """ May be overridden by subclasses in order to return the input files required by the target. - :param modules: A `ModuleRegistry` that can be used by the target for looking up modules + :param module: The module, the target should be applied to :return: A list that contains the input files """ return [] - def get_output_files(self, modules: ModuleRegistry) -> List[str]: + def get_output_files(self, module: Module) -> List[str]: """ May be overridden by subclasses in order to return the output files produced by the target. - :param modules: A `ModuleRegistry` that can be used by the target for looking up modules + :param module: The module, the target should be applied to :return: A list that contains the output files """ return [] - def get_clean_files(self, modules: ModuleRegistry) -> List[str]: + def get_clean_files(self, module: Module) -> List[str]: """ May be overridden by subclasses in order to return the output files produced by the target that must be cleaned. - :param modules: A `ModuleRegistry` that can be used by the target for looking up modules + :param module: The module, the target should be applied to :return: A list that contains the files to be cleaned """ - return self.get_output_files(modules) + return self.get_output_files(module) - def output_files_exist(self, modules: ModuleRegistry) -> bool: + def output_files_exist(self, module: Module) -> bool: """ Returns whether all output files produced by the target exist or not. - :param modules: A `ModuleRegistry` that can be used by the target for looking up modules + :param module: The module, the target should be applied to :return: True, if all output files exist, False otherwise """ - output_files = self.get_output_files(modules) + output_files = self.get_output_files(module) if output_files: missing_files = [output_file for output_file in output_files if not path.exists(output_file)] if missing_files: - Log.verbose('Target needs to be run, because the following output files do not exist:') + Log.verbose( + 'Target needs to be applied to module %s, because the following output files do not exist:', + str(module)) for missing_file in missing_files: Log.verbose(' - %s', missing_file) @@ -267,27 +276,29 @@ def output_files_exist(self, modules: ModuleRegistry) -> bool: Log.verbose('') return False - Log.verbose('All output files already exist.') + Log.verbose('All output files of module %s already exist.', str(module)) return True return False - def input_files_have_changed(self, modules: ModuleRegistry) -> bool: + def input_files_have_changed(self, module: Module) -> bool: """ Returns whether any input files required by the target have changed since the target was run for the last time. - :param modules: A `ModuleRegistry` that can be used by the target for looking up modules + :param module: The module, the target should be applied to :return: True, if any input files have changed, False otherwise """ - input_files = self.get_input_files(modules) + input_files = self.get_input_files(module) if input_files: # TODO check timestamps or hashes changed_files = input_files if changed_files: - Log.verbose('Target needs to be run, because the following input files have changed:\n') + Log.verbose( + 'Target needs to be applied to module %s, because the following input files have changed:\n', + str(module)) for input_file in input_files: Log.verbose(' - %s', input_file) @@ -295,7 +306,7 @@ def input_files_have_changed(self, modules: ModuleRegistry) -> bool: Log.verbose('') return True - Log.verbose('No input files have changed.') + Log.verbose('No input files of module %s have changed.', str(module)) return False return True @@ -340,19 +351,24 @@ def __init__(self, name: str, dependencies: List[Target.Dependency], runnables: def run(self, module_registry: ModuleRegistry): for runnable in self.runnables: - if not runnable.output_files_exist(module_registry): - if runnable.input_files_have_changed(module_registry): - runnable.run(self.build_unit, module_registry) + modules = module_registry.lookup(runnable.module_filter) + + for module in modules: + if not runnable.output_files_exist(module) or runnable.input_files_have_changed(module): + runnable.run(self.build_unit, module) def clean(self, module_registry: ModuleRegistry): for runnable in self.runnables: - clean_files = runnable.get_clean_files(module_registry) - delete_files(*clean_files) + modules = module_registry.lookup(runnable.module_filter) + + for module in modules: + clean_files = runnable.get_clean_files(module) + delete_files(*clean_files, accept_missing=True) class PhonyTarget(Target): """ - A phony target, which executes a certain action and does not produce any output files. + A phony target, which executes a certain action unconditionally. """ Function = Callable[[], None] @@ -362,14 +378,30 @@ class Runnable(ABC): An abstract base class for all classes that can be run via a phony target. """ - @abstractmethod - def run(self, build_unit: BuildUnit, modules: ModuleRegistry): + def __init__(self, module_filter: Module.Filter): + """ + :param module_filter: A filter that matches the modules, the target should be applied to + """ + self.module_filter = module_filter + + def run_all(self, build_unit: BuildUnit, module: List[Module]): + """ + May be overridden by subclasses in order to apply the target to all modules that match the filter. + + :param build_unit: The build unit, the target belongs to + :param module: A list that contains the modules, the target should be applied to + """ + raise NotImplementedError('Class ' + type(self).__name__ + ' does not implement the "run_all" method') + + def run(self, build_unit: BuildUnit, module: Module): """ - Must be implemented by subclasses in order to run the target. + May be overridden by subclasses in order to apply the target to an individual module that matches the + filter. :param build_unit: The build unit, the target belongs to - :param modules: A `ModuleRegistry` that can be used by the target for looking up modules + :param module: The module, the target should be applied to """ + raise NotImplementedError('Class ' + type(self).__name__ + ' does not implement the "run" method') class Builder(Target.Builder): """ @@ -420,7 +452,17 @@ def action(module_registry: ModuleRegistry): function() for runnable in self.runnables: - runnable.run(build_unit, module_registry) + modules = module_registry.lookup(runnable.module_filter) + + try: + runnable.run_all(build_unit, modules) + except NotImplementedError: + try: + for module in modules: + runnable.run(build_unit, module) + except NotImplementedError as error: + raise RuntimeError('Class ' + type(runnable).__name__ + + ' must implement either the "run_all" or "run" method') from error return PhonyTarget(self.target_name, self.dependencies, action) From 02446e9684e649d3b0d7addf6e7088edc41ea162 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 11 Dec 2024 02:43:49 +0100 Subject: [PATCH 084/114] Move files into new package "core". --- build_system/code_style/__init__.py | 4 ++-- build_system/code_style/cpp/__init__.py | 4 ++-- build_system/code_style/cpp/clang_format.py | 2 +- build_system/code_style/cpp/cpplint.py | 2 +- build_system/code_style/cpp/targets.py | 6 +++--- build_system/code_style/markdown/__init__.py | 4 ++-- build_system/code_style/markdown/mdformat.py | 2 +- build_system/code_style/markdown/targets.py | 6 +++--- build_system/code_style/modules.py | 2 +- build_system/code_style/python/__init__.py | 4 ++-- build_system/code_style/python/isort.py | 2 +- build_system/code_style/python/pylint.py | 2 +- build_system/code_style/python/targets.py | 6 +++--- build_system/code_style/python/yapf.py | 2 +- build_system/code_style/yaml/__init__.py | 4 ++-- build_system/code_style/yaml/targets.py | 6 +++--- build_system/code_style/yaml/yamlfix.py | 2 +- build_system/compilation/__init__.py | 4 ++-- build_system/compilation/cpp/__init__.py | 4 ++-- build_system/compilation/cpp/targets.py | 6 +++--- build_system/compilation/cython/__init__.py | 4 ++-- build_system/compilation/cython/targets.py | 6 +++--- build_system/compilation/meson.py | 2 +- build_system/compilation/modules.py | 2 +- build_system/core/__init__.py | 0 build_system/{util/units.py => core/build_unit.py} | 0 build_system/{util => core}/modules.py | 2 +- build_system/{util => core}/targets.py | 4 ++-- build_system/dependencies/github/__init__.py | 4 ++-- build_system/dependencies/github/actions.py | 2 +- build_system/dependencies/github/modules.py | 2 +- build_system/dependencies/github/pygithub.py | 2 +- build_system/dependencies/github/pyyaml.py | 2 +- build_system/dependencies/github/targets.py | 6 +++--- build_system/dependencies/python/__init__.py | 4 ++-- build_system/dependencies/python/modules.py | 2 +- build_system/dependencies/python/targets.py | 6 +++--- build_system/dependencies/table.py | 2 +- build_system/main.py | 4 ++-- build_system/packaging/__init__.py | 4 ++-- build_system/packaging/build.py | 2 +- build_system/packaging/modules.py | 2 +- build_system/packaging/targets.py | 6 +++--- build_system/testing/__init__.py | 4 ++-- build_system/testing/cpp/__init__.py | 4 ++-- build_system/testing/cpp/meson.py | 2 +- build_system/testing/cpp/modules.py | 2 +- build_system/testing/cpp/targets.py | 6 +++--- build_system/testing/modules.py | 2 +- build_system/testing/python/__init__.py | 4 ++-- build_system/testing/python/modules.py | 2 +- build_system/testing/python/targets.py | 6 +++--- build_system/testing/python/unittest.py | 2 +- build_system/util/pip.py | 2 +- build_system/util/run.py | 2 +- build_system/versioning/__init__.py | 4 ++-- 56 files changed, 93 insertions(+), 93 deletions(-) create mode 100644 build_system/core/__init__.py rename build_system/{util/units.py => core/build_unit.py} (100%) rename build_system/{util => core}/modules.py (97%) rename build_system/{util => core}/targets.py (99%) diff --git a/build_system/code_style/__init__.py b/build_system/code_style/__init__.py index b12b23f14..bb3904ec5 100644 --- a/build_system/code_style/__init__.py +++ b/build_system/code_style/__init__.py @@ -7,8 +7,8 @@ from code_style.markdown import FORMAT_MARKDOWN, TEST_FORMAT_MARKDOWN from code_style.python import FORMAT_PYTHON, TEST_FORMAT_PYTHON from code_style.yaml import FORMAT_YAML, TEST_FORMAT_YAML -from util.targets import TargetBuilder -from util.units import BuildUnit +from core.build_unit import BuildUnit +from core.targets import TargetBuilder TARGETS = TargetBuilder(BuildUnit('code_style')) \ .add_phony_target('format') \ diff --git a/build_system/code_style/cpp/__init__.py b/build_system/code_style/cpp/__init__.py index d35ff2043..cdc050b11 100644 --- a/build_system/code_style/cpp/__init__.py +++ b/build_system/code_style/cpp/__init__.py @@ -5,10 +5,10 @@ """ from code_style.cpp.targets import CheckCppCodeStyle, EnforceCppCodeStyle from code_style.modules import CodeModule +from core.build_unit import BuildUnit +from core.targets import PhonyTarget, TargetBuilder from util.files import FileType from util.paths import Project -from util.targets import PhonyTarget, TargetBuilder -from util.units import BuildUnit FORMAT_CPP = 'format_cpp' diff --git a/build_system/code_style/cpp/clang_format.py b/build_system/code_style/cpp/clang_format.py index 00e9cdab6..f43fcaed1 100644 --- a/build_system/code_style/cpp/clang_format.py +++ b/build_system/code_style/cpp/clang_format.py @@ -6,8 +6,8 @@ from os import path from code_style.modules import CodeModule +from core.build_unit import BuildUnit from util.run import Program -from util.units import BuildUnit class ClangFormat(Program): diff --git a/build_system/code_style/cpp/cpplint.py b/build_system/code_style/cpp/cpplint.py index 43997a279..d214a2316 100644 --- a/build_system/code_style/cpp/cpplint.py +++ b/build_system/code_style/cpp/cpplint.py @@ -4,8 +4,8 @@ Provides classes that allow to run the external program "cpplint". """ from code_style.modules import CodeModule +from core.build_unit import BuildUnit from util.run import Program -from util.units import BuildUnit class CppLint(Program): diff --git a/build_system/code_style/cpp/targets.py b/build_system/code_style/cpp/targets.py index 3761da476..a73fc6aa7 100644 --- a/build_system/code_style/cpp/targets.py +++ b/build_system/code_style/cpp/targets.py @@ -6,11 +6,11 @@ from code_style.cpp.clang_format import ClangFormat from code_style.cpp.cpplint import CppLint from code_style.modules import CodeModule +from core.build_unit import BuildUnit +from core.modules import Module +from core.targets import PhonyTarget from util.files import FileType from util.log import Log -from util.modules import Module -from util.targets import PhonyTarget -from util.units import BuildUnit MODULE_FILTER = CodeModule.Filter(FileType.cpp()) diff --git a/build_system/code_style/markdown/__init__.py b/build_system/code_style/markdown/__init__.py index 9b0740eaf..b2840ea38 100644 --- a/build_system/code_style/markdown/__init__.py +++ b/build_system/code_style/markdown/__init__.py @@ -5,10 +5,10 @@ """ from code_style.markdown.targets import CheckMarkdownCodeStyle, EnforceMarkdownCodeStyle from code_style.modules import CodeModule +from core.build_unit import BuildUnit +from core.targets import PhonyTarget, TargetBuilder from util.files import FileSearch, FileType from util.paths import Project -from util.targets import PhonyTarget, TargetBuilder -from util.units import BuildUnit FORMAT_MARKDOWN = 'format_md' diff --git a/build_system/code_style/markdown/mdformat.py b/build_system/code_style/markdown/mdformat.py index 5ec2c960c..ce56745a7 100644 --- a/build_system/code_style/markdown/mdformat.py +++ b/build_system/code_style/markdown/mdformat.py @@ -4,8 +4,8 @@ Provides classes that allow to run the external program "mdformat". """ from code_style.modules import CodeModule +from core.build_unit import BuildUnit from util.run import Program -from util.units import BuildUnit class MdFormat(Program): diff --git a/build_system/code_style/markdown/targets.py b/build_system/code_style/markdown/targets.py index d58042721..4e50e5e25 100644 --- a/build_system/code_style/markdown/targets.py +++ b/build_system/code_style/markdown/targets.py @@ -5,11 +5,11 @@ """ from code_style.markdown.mdformat import MdFormat from code_style.modules import CodeModule +from core.build_unit import BuildUnit +from core.modules import Module +from core.targets import PhonyTarget from util.files import FileType from util.log import Log -from util.modules import Module -from util.targets import PhonyTarget -from util.units import BuildUnit MODULE_FILTER = CodeModule.Filter(FileType.markdown()) diff --git a/build_system/code_style/modules.py b/build_system/code_style/modules.py index 450d4418a..04eb37830 100644 --- a/build_system/code_style/modules.py +++ b/build_system/code_style/modules.py @@ -5,8 +5,8 @@ """ from typing import List +from core.modules import Module from util.files import FileSearch, FileType -from util.modules import Module class CodeModule(Module): diff --git a/build_system/code_style/python/__init__.py b/build_system/code_style/python/__init__.py index 3f17e94f9..a559b45de 100644 --- a/build_system/code_style/python/__init__.py +++ b/build_system/code_style/python/__init__.py @@ -6,10 +6,10 @@ from code_style.modules import CodeModule from code_style.python.targets import CheckCythonCodeStyle, CheckPythonCodeStyle, EnforceCythonCodeStyle, \ EnforcePythonCodeStyle +from core.build_unit import BuildUnit +from core.targets import PhonyTarget, TargetBuilder from util.files import FileType from util.paths import Project -from util.targets import PhonyTarget, TargetBuilder -from util.units import BuildUnit FORMAT_PYTHON = 'format_python' diff --git a/build_system/code_style/python/isort.py b/build_system/code_style/python/isort.py index 5a6ea7121..9e4fe656e 100644 --- a/build_system/code_style/python/isort.py +++ b/build_system/code_style/python/isort.py @@ -4,8 +4,8 @@ Provides classes that allow to run the external program "isort". """ from code_style.modules import CodeModule +from core.build_unit import BuildUnit from util.run import Program -from util.units import BuildUnit class ISort(Program): diff --git a/build_system/code_style/python/pylint.py b/build_system/code_style/python/pylint.py index 55e276799..7c98fda87 100644 --- a/build_system/code_style/python/pylint.py +++ b/build_system/code_style/python/pylint.py @@ -6,8 +6,8 @@ from os import path from code_style.modules import CodeModule +from core.build_unit import BuildUnit from util.run import Program -from util.units import BuildUnit class PyLint(Program): diff --git a/build_system/code_style/python/targets.py b/build_system/code_style/python/targets.py index eabf49c30..d629c0f54 100644 --- a/build_system/code_style/python/targets.py +++ b/build_system/code_style/python/targets.py @@ -7,11 +7,11 @@ from code_style.python.isort import ISort from code_style.python.pylint import PyLint from code_style.python.yapf import Yapf +from core.build_unit import BuildUnit +from core.modules import Module +from core.targets import PhonyTarget from util.files import FileType from util.log import Log -from util.modules import Module -from util.targets import PhonyTarget -from util.units import BuildUnit PYTHON_MODULE_FILTER = CodeModule.Filter(FileType.python()) diff --git a/build_system/code_style/python/yapf.py b/build_system/code_style/python/yapf.py index 92f41ecce..81e97cd96 100644 --- a/build_system/code_style/python/yapf.py +++ b/build_system/code_style/python/yapf.py @@ -6,8 +6,8 @@ from os import path from code_style.modules import CodeModule +from core.build_unit import BuildUnit from util.run import Program -from util.units import BuildUnit class Yapf(Program): diff --git a/build_system/code_style/yaml/__init__.py b/build_system/code_style/yaml/__init__.py index 7580ededa..44f5754d0 100644 --- a/build_system/code_style/yaml/__init__.py +++ b/build_system/code_style/yaml/__init__.py @@ -5,10 +5,10 @@ """ from code_style.modules import CodeModule from code_style.yaml.targets import CheckYamlCodeStyle, EnforceYamlCodeStyle +from core.build_unit import BuildUnit +from core.targets import PhonyTarget, TargetBuilder from util.files import FileSearch, FileType from util.paths import Project -from util.targets import PhonyTarget, TargetBuilder -from util.units import BuildUnit FORMAT_YAML = 'format_yaml' diff --git a/build_system/code_style/yaml/targets.py b/build_system/code_style/yaml/targets.py index b16ad7c1b..2ec44620d 100644 --- a/build_system/code_style/yaml/targets.py +++ b/build_system/code_style/yaml/targets.py @@ -5,11 +5,11 @@ """ from code_style.modules import CodeModule from code_style.yaml.yamlfix import YamlFix +from core.build_unit import BuildUnit +from core.modules import Module +from core.targets import PhonyTarget from util.files import FileType from util.log import Log -from util.modules import Module -from util.targets import PhonyTarget -from util.units import BuildUnit MODULE_FILTER = CodeModule.Filter(FileType.yaml()) diff --git a/build_system/code_style/yaml/yamlfix.py b/build_system/code_style/yaml/yamlfix.py index e437fe1e2..546b466b4 100644 --- a/build_system/code_style/yaml/yamlfix.py +++ b/build_system/code_style/yaml/yamlfix.py @@ -6,8 +6,8 @@ from os import path from code_style.modules import CodeModule +from core.build_unit import BuildUnit from util.run import Program -from util.units import BuildUnit class YamlFix(Program): diff --git a/build_system/compilation/__init__.py b/build_system/compilation/__init__.py index 97010fd49..31871ed73 100644 --- a/build_system/compilation/__init__.py +++ b/build_system/compilation/__init__.py @@ -5,8 +5,8 @@ """ from compilation.cpp import COMPILE_CPP, INSTALL_CPP from compilation.cython import COMPILE_CYTHON, INSTALL_CYTHON -from util.targets import TargetBuilder -from util.units import BuildUnit +from core.build_unit import BuildUnit +from core.targets import TargetBuilder INSTALL = 'install' diff --git a/build_system/compilation/cpp/__init__.py b/build_system/compilation/cpp/__init__.py index 0b09d64c6..3989a96a0 100644 --- a/build_system/compilation/cpp/__init__.py +++ b/build_system/compilation/cpp/__init__.py @@ -5,11 +5,11 @@ """ from compilation.cpp.targets import CompileCpp, InstallCpp, SetupCpp from compilation.modules import CompilationModule +from core.build_unit import BuildUnit +from core.targets import PhonyTarget, TargetBuilder from dependencies.python import VENV from util.files import FileType from util.paths import Project -from util.targets import PhonyTarget, TargetBuilder -from util.units import BuildUnit SETUP_CPP = 'setup_cpp' diff --git a/build_system/compilation/cpp/targets.py b/build_system/compilation/cpp/targets.py index 3c9d68918..9e2d3a83f 100644 --- a/build_system/compilation/cpp/targets.py +++ b/build_system/compilation/cpp/targets.py @@ -8,11 +8,11 @@ from compilation.build_options import BuildOptions, EnvBuildOption from compilation.meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup from compilation.modules import CompilationModule +from core.build_unit import BuildUnit +from core.modules import Module +from core.targets import BuildTarget, PhonyTarget from util.files import FileType from util.log import Log -from util.modules import Module -from util.targets import BuildTarget, PhonyTarget -from util.units import BuildUnit MODULE_FILTER = CompilationModule.Filter(FileType.cpp()) diff --git a/build_system/compilation/cython/__init__.py b/build_system/compilation/cython/__init__.py index 8a432ff28..a03bb505a 100644 --- a/build_system/compilation/cython/__init__.py +++ b/build_system/compilation/cython/__init__.py @@ -6,10 +6,10 @@ from compilation.cpp import COMPILE_CPP from compilation.cython.targets import CompileCython, InstallCython, SetupCython from compilation.modules import CompilationModule +from core.build_unit import BuildUnit +from core.targets import PhonyTarget, TargetBuilder from util.files import FileType from util.paths import Project -from util.targets import PhonyTarget, TargetBuilder -from util.units import BuildUnit SETUP_CYTHON = 'setup_cython' diff --git a/build_system/compilation/cython/targets.py b/build_system/compilation/cython/targets.py index 91f56bbc3..6730fa371 100644 --- a/build_system/compilation/cython/targets.py +++ b/build_system/compilation/cython/targets.py @@ -8,11 +8,11 @@ from compilation.build_options import BuildOptions, EnvBuildOption from compilation.meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup from compilation.modules import CompilationModule +from core.build_unit import BuildUnit +from core.modules import Module +from core.targets import BuildTarget, PhonyTarget from util.files import FileType from util.log import Log -from util.modules import Module -from util.targets import BuildTarget, PhonyTarget -from util.units import BuildUnit MODULE_FILTER = CompilationModule.Filter(FileType.cython()) diff --git a/build_system/compilation/meson.py b/build_system/compilation/meson.py index a627bfe34..3e12abd62 100644 --- a/build_system/compilation/meson.py +++ b/build_system/compilation/meson.py @@ -8,9 +8,9 @@ from compilation.build_options import BuildOptions from compilation.modules import CompilationModule +from core.build_unit import BuildUnit from util.log import Log from util.run import Program -from util.units import BuildUnit def build_options_as_meson_arguments(build_options: BuildOptions) -> List[str]: diff --git a/build_system/compilation/modules.py b/build_system/compilation/modules.py index 4553455e9..772942496 100644 --- a/build_system/compilation/modules.py +++ b/build_system/compilation/modules.py @@ -6,8 +6,8 @@ from os import path from typing import List, Optional +from core.modules import Module from util.files import FileSearch, FileType -from util.modules import Module class CompilationModule(Module): diff --git a/build_system/core/__init__.py b/build_system/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/build_system/util/units.py b/build_system/core/build_unit.py similarity index 100% rename from build_system/util/units.py rename to build_system/core/build_unit.py diff --git a/build_system/util/modules.py b/build_system/core/modules.py similarity index 97% rename from build_system/util/modules.py rename to build_system/core/modules.py index 61ef8060b..bc0bbe2d4 100644 --- a/build_system/util/modules.py +++ b/build_system/core/modules.py @@ -2,7 +2,7 @@ Author: Michael Rapp (michael.rapp.ml@gmail.com) Provides classes that provide information about files and directories that belong to individual modules of the project -to be dealt with by the build system. +to be dealt with by the targets of the build system. """ from abc import ABC from functools import reduce diff --git a/build_system/util/targets.py b/build_system/core/targets.py similarity index 99% rename from build_system/util/targets.py rename to build_system/core/targets.py index d467e500a..048fd55b5 100644 --- a/build_system/util/targets.py +++ b/build_system/core/targets.py @@ -9,11 +9,11 @@ from os import path from typing import Any, Callable, Dict, Iterable, List, Optional +from core.build_unit import BuildUnit +from core.modules import Module, ModuleRegistry from util.format import format_iterable from util.io import delete_files from util.log import Log -from util.modules import Module, ModuleRegistry -from util.units import BuildUnit class Target(ABC): diff --git a/build_system/dependencies/github/__init__.py b/build_system/dependencies/github/__init__.py index 60e8b3783..6e85d9f4b 100644 --- a/build_system/dependencies/github/__init__.py +++ b/build_system/dependencies/github/__init__.py @@ -3,11 +3,11 @@ Defines targets for updating the project's GitHub Actions. """ +from core.build_unit import BuildUnit +from core.targets import PhonyTarget, TargetBuilder from dependencies.github.modules import GithubWorkflowModule from dependencies.github.targets import CheckGithubActions, UpdateGithubActions from util.paths import Project -from util.targets import PhonyTarget, TargetBuilder -from util.units import BuildUnit TARGETS = TargetBuilder(BuildUnit('dependencies', 'github')) \ .add_phony_target('check_github_actions').set_runnables(CheckGithubActions()) \ diff --git a/build_system/dependencies/github/actions.py b/build_system/dependencies/github/actions.py index 883ddec72..1355a5d4a 100644 --- a/build_system/dependencies/github/actions.py +++ b/build_system/dependencies/github/actions.py @@ -8,12 +8,12 @@ from os import environ from typing import Dict, List, Optional, Set +from core.build_unit import BuildUnit from dependencies.github.modules import GithubWorkflowModule from dependencies.github.pygithub import GithubApi from dependencies.github.pyyaml import YamlFile from util.env import get_env from util.log import Log -from util.units import BuildUnit @dataclass diff --git a/build_system/dependencies/github/modules.py b/build_system/dependencies/github/modules.py index d2f6ef054..4ac3cbe3d 100644 --- a/build_system/dependencies/github/modules.py +++ b/build_system/dependencies/github/modules.py @@ -5,8 +5,8 @@ """ from typing import List +from core.modules import Module from util.files import FileSearch, FileType -from util.modules import Module class GithubWorkflowModule(Module): diff --git a/build_system/dependencies/github/pygithub.py b/build_system/dependencies/github/pygithub.py index 7eba4170f..3b7773cab 100644 --- a/build_system/dependencies/github/pygithub.py +++ b/build_system/dependencies/github/pygithub.py @@ -5,8 +5,8 @@ """ from typing import Optional +from core.build_unit import BuildUnit from util.pip import Pip -from util.units import BuildUnit class GithubApi: diff --git a/build_system/dependencies/github/pyyaml.py b/build_system/dependencies/github/pyyaml.py index 5111d1aa4..cfdf262d4 100644 --- a/build_system/dependencies/github/pyyaml.py +++ b/build_system/dependencies/github/pyyaml.py @@ -6,9 +6,9 @@ from functools import cached_property from typing import Dict +from core.build_unit import BuildUnit from util.io import TextFile, read_file from util.pip import Pip -from util.units import BuildUnit class YamlFile(TextFile): diff --git a/build_system/dependencies/github/targets.py b/build_system/dependencies/github/targets.py index 8181a902c..7f9eb6782 100644 --- a/build_system/dependencies/github/targets.py +++ b/build_system/dependencies/github/targets.py @@ -3,13 +3,13 @@ Implements targets for updating the project's GitHub Actions. """ +from core.build_unit import BuildUnit +from core.modules import Module +from core.targets import PhonyTarget from dependencies.github.actions import WorkflowUpdater from dependencies.github.modules import GithubWorkflowModule from dependencies.table import Table from util.log import Log -from util.modules import Module -from util.targets import PhonyTarget -from util.units import BuildUnit MODULE_FILTER = GithubWorkflowModule.Filter() diff --git a/build_system/dependencies/python/__init__.py b/build_system/dependencies/python/__init__.py index 129a27d82..165e06289 100644 --- a/build_system/dependencies/python/__init__.py +++ b/build_system/dependencies/python/__init__.py @@ -3,11 +3,11 @@ Defines targets and modules for installing Python dependencies that are required by the project. """ +from core.build_unit import BuildUnit +from core.targets import PhonyTarget, TargetBuilder from dependencies.python.modules import DependencyType, PythonDependencyModule from dependencies.python.targets import CheckPythonDependencies, InstallRuntimeDependencies from util.paths import Project -from util.targets import PhonyTarget, TargetBuilder -from util.units import BuildUnit VENV = 'venv' diff --git a/build_system/dependencies/python/modules.py b/build_system/dependencies/python/modules.py index 02d98a5a2..45efc7d04 100644 --- a/build_system/dependencies/python/modules.py +++ b/build_system/dependencies/python/modules.py @@ -6,8 +6,8 @@ from enum import Enum, auto from typing import List +from core.modules import Module from util.files import FileSearch -from util.modules import Module class DependencyType(Enum): diff --git a/build_system/dependencies/python/targets.py b/build_system/dependencies/python/targets.py index 7161fbc8c..20d526ded 100644 --- a/build_system/dependencies/python/targets.py +++ b/build_system/dependencies/python/targets.py @@ -6,13 +6,13 @@ from functools import reduce from typing import List +from core.build_unit import BuildUnit +from core.modules import Module +from core.targets import PhonyTarget from dependencies.python.modules import DependencyType, PythonDependencyModule from dependencies.python.pip import PipList from dependencies.table import Table from util.log import Log -from util.modules import Module -from util.targets import PhonyTarget -from util.units import BuildUnit class InstallRuntimeDependencies(PhonyTarget.Runnable): diff --git a/build_system/dependencies/table.py b/build_system/dependencies/table.py index daafedb47..cb7a0f01d 100644 --- a/build_system/dependencies/table.py +++ b/build_system/dependencies/table.py @@ -3,8 +3,8 @@ Provides classes for creating tables. """ +from core.build_unit import BuildUnit from util.pip import Pip -from util.units import BuildUnit class Table: diff --git a/build_system/main.py b/build_system/main.py index e830face8..e87d72f02 100644 --- a/build_system/main.py +++ b/build_system/main.py @@ -6,13 +6,13 @@ from argparse import ArgumentParser from typing import List +from core.modules import Module, ModuleRegistry +from core.targets import DependencyGraph, Target, TargetRegistry from util.files import FileSearch from util.format import format_iterable from util.log import Log -from util.modules import Module, ModuleRegistry from util.paths import Project from util.reflection import import_source_file -from util.targets import DependencyGraph, Target, TargetRegistry def __parse_command_line_arguments(): diff --git a/build_system/packaging/__init__.py b/build_system/packaging/__init__.py index 140710511..67a2ac010 100644 --- a/build_system/packaging/__init__.py +++ b/build_system/packaging/__init__.py @@ -6,11 +6,11 @@ from os import path from compilation import INSTALL +from core.build_unit import BuildUnit +from core.targets import TargetBuilder from packaging.modules import PythonPackageModule from packaging.targets import BuildPythonWheels, InstallPythonWheels from util.paths import Project -from util.targets import TargetBuilder -from util.units import BuildUnit BUILD_WHEELS = 'build_wheels' diff --git a/build_system/packaging/build.py b/build_system/packaging/build.py index ba0479142..c2a755292 100644 --- a/build_system/packaging/build.py +++ b/build_system/packaging/build.py @@ -3,9 +3,9 @@ Provides classes that allow to build wheel packages via the external program "build". """ +from core.build_unit import BuildUnit from packaging.modules import PythonPackageModule from util.run import PythonModule -from util.units import BuildUnit class Build(PythonModule): diff --git a/build_system/packaging/modules.py b/build_system/packaging/modules.py index f3ba8d993..c4bbfd2f3 100644 --- a/build_system/packaging/modules.py +++ b/build_system/packaging/modules.py @@ -6,8 +6,8 @@ from os import path from typing import List +from core.modules import Module from util.files import FileSearch -from util.modules import Module class PythonPackageModule(Module): diff --git a/build_system/packaging/targets.py b/build_system/packaging/targets.py index 6dacfc733..f2b0416d0 100644 --- a/build_system/packaging/targets.py +++ b/build_system/packaging/targets.py @@ -5,15 +5,15 @@ """ from typing import List +from core.build_unit import BuildUnit +from core.modules import Module +from core.targets import BuildTarget from packaging.build import Build from packaging.modules import PythonPackageModule from packaging.pip import PipInstallWheel from util.files import DirectorySearch, FileType from util.log import Log -from util.modules import Module from util.paths import Project -from util.targets import BuildTarget -from util.units import BuildUnit MODULE_FILTER = PythonPackageModule.Filter() diff --git a/build_system/testing/__init__.py b/build_system/testing/__init__.py index 63e4a000c..8fa2ad4de 100644 --- a/build_system/testing/__init__.py +++ b/build_system/testing/__init__.py @@ -3,10 +3,10 @@ Defines targets for testing code. """ +from core.build_unit import BuildUnit +from core.targets import TargetBuilder from testing.cpp import TESTS_CPP from testing.python import TESTS_PYTHON -from util.targets import TargetBuilder -from util.units import BuildUnit TARGETS = TargetBuilder(BuildUnit('testing')) \ .add_phony_target('tests') \ diff --git a/build_system/testing/cpp/__init__.py b/build_system/testing/cpp/__init__.py index e01815a1c..650628790 100644 --- a/build_system/testing/cpp/__init__.py +++ b/build_system/testing/cpp/__init__.py @@ -4,11 +4,11 @@ Defines targets and modules for testing C++ code. """ from compilation.cpp import COMPILE_CPP +from core.build_unit import BuildUnit +from core.targets import PhonyTarget, TargetBuilder from testing.cpp.modules import CppTestModule from testing.cpp.targets import TestCpp from util.paths import Project -from util.targets import PhonyTarget, TargetBuilder -from util.units import BuildUnit TESTS_CPP = 'tests_cpp' diff --git a/build_system/testing/cpp/meson.py b/build_system/testing/cpp/meson.py index 74103db7a..ff6499cc8 100644 --- a/build_system/testing/cpp/meson.py +++ b/build_system/testing/cpp/meson.py @@ -4,8 +4,8 @@ Provides classes that allow to run automated tests via the external program "meson". """ from compilation.meson import Meson +from core.build_unit import BuildUnit from testing.cpp.modules import CppTestModule -from util.units import BuildUnit class MesonTest(Meson): diff --git a/build_system/testing/cpp/modules.py b/build_system/testing/cpp/modules.py index b21c4f632..816b2e35a 100644 --- a/build_system/testing/cpp/modules.py +++ b/build_system/testing/cpp/modules.py @@ -5,8 +5,8 @@ """ from os import path +from core.modules import Module from testing.modules import TestModule -from util.modules import Module class CppTestModule(TestModule): diff --git a/build_system/testing/cpp/targets.py b/build_system/testing/cpp/targets.py index db1b07faf..6bb65591c 100644 --- a/build_system/testing/cpp/targets.py +++ b/build_system/testing/cpp/targets.py @@ -3,11 +3,11 @@ Implements targets for testing C++ code. """ +from core.build_unit import BuildUnit +from core.modules import Module +from core.targets import BuildTarget, PhonyTarget from testing.cpp.meson import MesonTest from testing.cpp.modules import CppTestModule -from util.modules import Module -from util.targets import BuildTarget, PhonyTarget -from util.units import BuildUnit class TestCpp(PhonyTarget.Runnable): diff --git a/build_system/testing/modules.py b/build_system/testing/modules.py index 5563beefd..ca0b283f1 100644 --- a/build_system/testing/modules.py +++ b/build_system/testing/modules.py @@ -6,8 +6,8 @@ from abc import ABC from os import environ +from core.modules import Module from util.env import get_env_bool -from util.modules import Module class TestModule(Module, ABC): diff --git a/build_system/testing/python/__init__.py b/build_system/testing/python/__init__.py index 2affbfbc2..ca31332fe 100644 --- a/build_system/testing/python/__init__.py +++ b/build_system/testing/python/__init__.py @@ -3,12 +3,12 @@ Defines targets and modules for testing Python code. """ +from core.build_unit import BuildUnit +from core.targets import PhonyTarget, TargetBuilder from packaging import INSTALL_WHEELS from testing.python.modules import PythonTestModule from testing.python.targets import TestPython from util.paths import Project -from util.targets import PhonyTarget, TargetBuilder -from util.units import BuildUnit TESTS_PYTHON = 'tests_python' diff --git a/build_system/testing/python/modules.py b/build_system/testing/python/modules.py index a0e58c94e..d1475ea48 100644 --- a/build_system/testing/python/modules.py +++ b/build_system/testing/python/modules.py @@ -6,9 +6,9 @@ from os import path from typing import List +from core.modules import Module from testing.modules import TestModule from util.files import FileSearch, FileType -from util.modules import Module class PythonTestModule(TestModule): diff --git a/build_system/testing/python/targets.py b/build_system/testing/python/targets.py index 81c7bd0f8..38105ccb7 100644 --- a/build_system/testing/python/targets.py +++ b/build_system/testing/python/targets.py @@ -3,11 +3,11 @@ Implements targets for testing Python code. """ +from core.build_unit import BuildUnit +from core.modules import Module +from core.targets import PhonyTarget from testing.python.modules import PythonTestModule from testing.python.unittest import UnitTest -from util.modules import Module -from util.targets import PhonyTarget -from util.units import BuildUnit class TestPython(PhonyTarget.Runnable): diff --git a/build_system/testing/python/unittest.py b/build_system/testing/python/unittest.py index 69a747309..e258c3976 100644 --- a/build_system/testing/python/unittest.py +++ b/build_system/testing/python/unittest.py @@ -3,9 +3,9 @@ Provides classes that allow to run automated tests via the external program "unittest". """ +from core.build_unit import BuildUnit from testing.python.modules import PythonTestModule from util.run import PythonModule -from util.units import BuildUnit class UnitTest: diff --git a/build_system/util/pip.py b/build_system/util/pip.py index 189ee2ec1..b33741348 100644 --- a/build_system/util/pip.py +++ b/build_system/util/pip.py @@ -8,10 +8,10 @@ from functools import reduce from typing import Dict, Optional, Set +from core.build_unit import BuildUnit from util.cmd import Command as Cmd from util.io import TextFile from util.log import Log -from util.units import BuildUnit @dataclass diff --git a/build_system/util/run.py b/build_system/util/run.py index bdf10f18a..19ca0f45c 100644 --- a/build_system/util/run.py +++ b/build_system/util/run.py @@ -5,9 +5,9 @@ """ from subprocess import CompletedProcess +from core.build_unit import BuildUnit from util.cmd import Command from util.pip import Pip -from util.units import BuildUnit class Program(Command): diff --git a/build_system/versioning/__init__.py b/build_system/versioning/__init__.py index d524dc4b4..6a7464090 100644 --- a/build_system/versioning/__init__.py +++ b/build_system/versioning/__init__.py @@ -1,8 +1,8 @@ """ Defines build targets for updating the project's version and changelog. """ -from util.targets import PhonyTarget, TargetBuilder -from util.units import BuildUnit +from core.build_unit import BuildUnit +from core.targets import PhonyTarget, TargetBuilder from versioning.changelog import print_latest_changelog, update_changelog_bugfix, update_changelog_feature, \ update_changelog_main, validate_changelog_bugfix, validate_changelog_feature, validate_changelog_main from versioning.versioning import apply_development_version, increment_development_version, increment_major_version, \ From b88e609deb67820989c51db59162deb661b42603 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 11 Dec 2024 02:51:24 +0100 Subject: [PATCH 085/114] Remove file venv.py. --- build_system/util/cmd.py | 7 +++++-- build_system/util/venv.py | 15 --------------- 2 files changed, 5 insertions(+), 17 deletions(-) delete mode 100644 build_system/util/venv.py diff --git a/build_system/util/cmd.py b/build_system/util/cmd.py index fd3f046a9..539dce827 100644 --- a/build_system/util/cmd.py +++ b/build_system/util/cmd.py @@ -11,7 +11,6 @@ from util.format import format_iterable from util.log import Log -from util.venv import in_virtual_environment class Command: @@ -83,6 +82,10 @@ def run(self, command: 'Command', capture_output: bool) -> CompletedProcess: return output + @staticmethod + def __in_virtual_environment() -> bool: + return sys.prefix != sys.base_prefix + def __init__(self, command: str, *arguments: str, @@ -97,7 +100,7 @@ def __init__(self, """ self.command = command - if in_virtual_environment(): + if self.__in_virtual_environment(): # On Windows, we use the relative path to the command's executable within the virtual environment, if such # an executable exists. This circumvents situations where the PATH environment variable has not been updated # after activating the virtual environment. This can prevent the executables from being found or can lead to diff --git a/build_system/util/venv.py b/build_system/util/venv.py deleted file mode 100644 index 90c64dfe9..000000000 --- a/build_system/util/venv.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Provides utility functions for dealing with virtual Python environments. -""" -import sys - - -def in_virtual_environment() -> bool: - """ - Returns whether the current process is executed in a virtual environment or not. - - :return: True, if the current process is executed in a virtual environment, False otherwise - """ - return sys.prefix != sys.base_prefix From 7be5ce4fc5b4c1dcec115745f2367614d6178890 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 11 Dec 2024 02:54:52 +0100 Subject: [PATCH 086/114] Remove file reflection.py. --- build_system/main.py | 20 +++++++++++++++++--- build_system/util/reflection.py | 26 -------------------------- 2 files changed, 17 insertions(+), 29 deletions(-) delete mode 100644 build_system/util/reflection.py diff --git a/build_system/main.py b/build_system/main.py index e87d72f02..bf7779cee 100644 --- a/build_system/main.py +++ b/build_system/main.py @@ -3,7 +3,11 @@ Initializes the build system and runs targets specified via command line arguments. """ +import sys + from argparse import ArgumentParser +from importlib.util import module_from_spec, spec_from_file_location +from types import ModuleType from typing import List from core.modules import Module, ModuleRegistry @@ -12,7 +16,6 @@ from util.format import format_iterable from util.log import Log from util.paths import Project -from util.reflection import import_source_file def __parse_command_line_arguments(): @@ -35,6 +38,17 @@ def __find_init_files() -> List[str]: .list(Project.BuildSystem.root_directory) +def __import_source_file(source_file: str) -> ModuleType: + try: + spec = spec_from_file_location(source_file, source_file) + module = module_from_spec(spec) + sys.modules[source_file] = module + spec.loader.exec_module(module) + return module + except FileNotFoundError as error: + raise ImportError('Source file "' + source_file + '" not found') from error + + def __register_modules(init_files: List[str]) -> ModuleRegistry: Log.verbose('Registering modules...') module_registry = ModuleRegistry() @@ -42,7 +56,7 @@ def __register_modules(init_files: List[str]) -> ModuleRegistry: for init_file in init_files: modules = [ - module for module in getattr(import_source_file(init_file), 'MODULES', []) if isinstance(module, Module) + module for module in getattr(__import_source_file(init_file), 'MODULES', []) if isinstance(module, Module) ] if modules: @@ -66,7 +80,7 @@ def __register_targets(init_files: List[str]) -> TargetRegistry: for init_file in init_files: targets = [ - target for target in getattr(import_source_file(init_file), 'TARGETS', []) if isinstance(target, Target) + target for target in getattr(__import_source_file(init_file), 'TARGETS', []) if isinstance(target, Target) ] if targets: diff --git a/build_system/util/reflection.py b/build_system/util/reflection.py deleted file mode 100644 index 2129dea22..000000000 --- a/build_system/util/reflection.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Provides utility functions for importing and executing Python code at runtime. -""" -import sys - -from importlib.util import module_from_spec, spec_from_file_location -from types import ModuleType as Module - - -def import_source_file(source_file: str) -> Module: - """ - Imports a given source file. - - :param source_file: The path to the source file - :return: The module that has been imported - """ - try: - spec = spec_from_file_location(source_file, source_file) - module = module_from_spec(spec) - sys.modules[source_file] = module - spec.loader.exec_module(module) - return module - except FileNotFoundError as error: - raise ImportError('Source file "' + source_file + '" not found') from error From 046cd43690846d02776ff2728789177507d89707 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 11 Dec 2024 02:57:27 +0100 Subject: [PATCH 087/114] Move files into new package "targets". --- build_system/core/build_unit.py | 8 ++++---- build_system/main.py | 4 ++-- build_system/{dependencies => targets}/__init__.py | 0 build_system/{ => targets}/code_style/__init__.py | 11 ++++++----- .../{ => targets}/code_style/cpp/.clang-format | 0 build_system/{ => targets}/code_style/cpp/__init__.py | 9 +++++---- .../{ => targets}/code_style/cpp/clang_format.py | 3 ++- build_system/{ => targets}/code_style/cpp/cpplint.py | 3 ++- .../{ => targets}/code_style/cpp/requirements.txt | 0 build_system/{ => targets}/code_style/cpp/targets.py | 7 ++++--- .../{ => targets}/code_style/markdown/__init__.py | 9 +++++---- .../{ => targets}/code_style/markdown/mdformat.py | 3 ++- .../code_style/markdown/requirements.txt | 0 .../{ => targets}/code_style/markdown/targets.py | 5 +++-- build_system/{ => targets}/code_style/modules.py | 0 .../{ => targets}/code_style/python/.isort.cfg | 0 .../{ => targets}/code_style/python/.pylintrc | 0 .../{ => targets}/code_style/python/.style.yapf | 0 .../{ => targets}/code_style/python/__init__.py | 11 ++++++----- build_system/{ => targets}/code_style/python/isort.py | 3 ++- .../{ => targets}/code_style/python/pylint.py | 3 ++- .../{ => targets}/code_style/python/requirements.txt | 0 .../{ => targets}/code_style/python/targets.py | 9 +++++---- build_system/{ => targets}/code_style/python/yapf.py | 3 ++- .../{ => targets}/code_style/yaml/.yamlfix.toml | 0 .../{ => targets}/code_style/yaml/__init__.py | 9 +++++---- .../{ => targets}/code_style/yaml/requirements.txt | 0 build_system/{ => targets}/code_style/yaml/targets.py | 5 +++-- build_system/{ => targets}/code_style/yaml/yamlfix.py | 3 ++- build_system/{ => targets}/compilation/__init__.py | 7 ++++--- .../{ => targets}/compilation/build_options.py | 0 .../{ => targets}/compilation/cpp/__init__.py | 11 ++++++----- build_system/{ => targets}/compilation/cpp/targets.py | 7 ++++--- .../{ => targets}/compilation/cython/__init__.py | 11 ++++++----- .../{ => targets}/compilation/cython/requirements.txt | 0 .../{ => targets}/compilation/cython/targets.py | 7 ++++--- build_system/{ => targets}/compilation/meson.py | 5 +++-- build_system/{ => targets}/compilation/modules.py | 0 .../{ => targets}/compilation/requirements.txt | 0 build_system/targets/dependencies/__init__.py | 0 .../{ => targets}/dependencies/github/__init__.py | 9 +++++---- .../{ => targets}/dependencies/github/actions.py | 7 ++++--- .../{ => targets}/dependencies/github/modules.py | 0 .../{ => targets}/dependencies/github/pygithub.py | 0 .../{ => targets}/dependencies/github/pyyaml.py | 0 .../dependencies/github/requirements.txt | 0 .../{ => targets}/dependencies/github/targets.py | 7 ++++--- .../{ => targets}/dependencies/python/__init__.py | 9 +++++---- .../{ => targets}/dependencies/python/modules.py | 2 +- build_system/{ => targets}/dependencies/python/pip.py | 0 .../{ => targets}/dependencies/python/targets.py | 7 ++++--- .../{ => targets}/dependencies/requirements.txt | 0 build_system/{ => targets}/dependencies/table.py | 0 build_system/{ => targets}/packaging/__init__.py | 11 ++++++----- build_system/{ => targets}/packaging/build.py | 3 ++- build_system/{ => targets}/packaging/modules.py | 0 build_system/{ => targets}/packaging/pip.py | 0 build_system/{ => targets}/packaging/requirements.txt | 0 build_system/{ => targets}/packaging/targets.py | 9 +++++---- build_system/{util => targets}/paths.py | 0 build_system/{ => targets}/testing/__init__.py | 7 ++++--- build_system/{ => targets}/testing/cpp/__init__.py | 11 ++++++----- build_system/{ => targets}/testing/cpp/meson.py | 5 +++-- build_system/{ => targets}/testing/cpp/modules.py | 3 ++- build_system/{ => targets}/testing/cpp/targets.py | 5 +++-- build_system/{ => targets}/testing/modules.py | 0 build_system/{ => targets}/testing/python/__init__.py | 11 ++++++----- build_system/{ => targets}/testing/python/modules.py | 3 ++- .../{ => targets}/testing/python/requirements.txt | 0 build_system/{ => targets}/testing/python/targets.py | 5 +++-- build_system/{ => targets}/testing/python/unittest.py | 3 ++- build_system/{ => targets}/versioning/__init__.py | 8 +++++--- build_system/{ => targets}/versioning/changelog.py | 3 ++- build_system/{ => targets}/versioning/versioning.py | 0 74 files changed, 163 insertions(+), 121 deletions(-) rename build_system/{dependencies => targets}/__init__.py (100%) rename build_system/{ => targets}/code_style/__init__.py (59%) rename build_system/{ => targets}/code_style/cpp/.clang-format (100%) rename build_system/{ => targets}/code_style/cpp/__init__.py (74%) rename build_system/{ => targets}/code_style/cpp/clang_format.py (95%) rename build_system/{ => targets}/code_style/cpp/cpplint.py (92%) rename build_system/{ => targets}/code_style/cpp/requirements.txt (100%) rename build_system/{ => targets}/code_style/cpp/targets.py (88%) rename build_system/{ => targets}/code_style/markdown/__init__.py (80%) rename build_system/{ => targets}/code_style/markdown/mdformat.py (94%) rename build_system/{ => targets}/code_style/markdown/requirements.txt (100%) rename build_system/{ => targets}/code_style/markdown/targets.py (91%) rename build_system/{ => targets}/code_style/modules.py (100%) rename build_system/{ => targets}/code_style/python/.isort.cfg (100%) rename build_system/{ => targets}/code_style/python/.pylintrc (100%) rename build_system/{ => targets}/code_style/python/.style.yapf (100%) rename build_system/{ => targets}/code_style/python/__init__.py (83%) rename build_system/{ => targets}/code_style/python/isort.py (94%) rename build_system/{ => targets}/code_style/python/pylint.py (93%) rename build_system/{ => targets}/code_style/python/requirements.txt (100%) rename build_system/{ => targets}/code_style/python/targets.py (92%) rename build_system/{ => targets}/code_style/python/yapf.py (94%) rename build_system/{ => targets}/code_style/yaml/.yamlfix.toml (100%) rename build_system/{ => targets}/code_style/yaml/__init__.py (77%) rename build_system/{ => targets}/code_style/yaml/requirements.txt (100%) rename build_system/{ => targets}/code_style/yaml/targets.py (92%) rename build_system/{ => targets}/code_style/yaml/yamlfix.py (95%) rename build_system/{ => targets}/compilation/__init__.py (70%) rename build_system/{ => targets}/compilation/build_options.py (100%) rename build_system/{ => targets}/compilation/cpp/__init__.py (77%) rename build_system/{ => targets}/compilation/cpp/targets.py (91%) rename build_system/{ => targets}/compilation/cython/__init__.py (76%) rename build_system/{ => targets}/compilation/cython/requirements.txt (100%) rename build_system/{ => targets}/compilation/cython/targets.py (90%) rename build_system/{ => targets}/compilation/meson.py (96%) rename build_system/{ => targets}/compilation/modules.py (100%) rename build_system/{ => targets}/compilation/requirements.txt (100%) create mode 100644 build_system/targets/dependencies/__init__.py rename build_system/{ => targets}/dependencies/github/__init__.py (64%) rename build_system/{ => targets}/dependencies/github/actions.py (98%) rename build_system/{ => targets}/dependencies/github/modules.py (100%) rename build_system/{ => targets}/dependencies/github/pygithub.py (100%) rename build_system/{ => targets}/dependencies/github/pyyaml.py (100%) rename build_system/{ => targets}/dependencies/github/requirements.txt (100%) rename build_system/{ => targets}/dependencies/github/targets.py (92%) rename build_system/{ => targets}/dependencies/python/__init__.py (78%) rename build_system/{ => targets}/dependencies/python/modules.py (98%) rename build_system/{ => targets}/dependencies/python/pip.py (100%) rename build_system/{ => targets}/dependencies/python/targets.py (91%) rename build_system/{ => targets}/dependencies/requirements.txt (100%) rename build_system/{ => targets}/dependencies/table.py (100%) rename build_system/{ => targets}/packaging/__init__.py (75%) rename build_system/{ => targets}/packaging/build.py (92%) rename build_system/{ => targets}/packaging/modules.py (100%) rename build_system/{ => targets}/packaging/pip.py (100%) rename build_system/{ => targets}/packaging/requirements.txt (100%) rename build_system/{ => targets}/packaging/targets.py (92%) rename build_system/{util => targets}/paths.py (100%) rename build_system/{ => targets}/testing/__init__.py (65%) rename build_system/{ => targets}/testing/cpp/__init__.py (66%) rename build_system/{ => targets}/testing/cpp/meson.py (87%) rename build_system/{ => targets}/testing/cpp/modules.py (96%) rename build_system/{ => targets}/testing/cpp/targets.py (82%) rename build_system/{ => targets}/testing/modules.py (100%) rename build_system/{ => targets}/testing/python/__init__.py (71%) rename build_system/{ => targets}/testing/python/modules.py (97%) rename build_system/{ => targets}/testing/python/requirements.txt (100%) rename build_system/{ => targets}/testing/python/targets.py (81%) rename build_system/{ => targets}/testing/python/unittest.py (95%) rename build_system/{ => targets}/versioning/__init__.py (81%) rename build_system/{ => targets}/versioning/changelog.py (99%) rename build_system/{ => targets}/versioning/versioning.py (100%) diff --git a/build_system/core/build_unit.py b/build_system/core/build_unit.py index 5ba9ce785..afd7dfa5a 100644 --- a/build_system/core/build_unit.py +++ b/build_system/core/build_unit.py @@ -6,19 +6,19 @@ from os import path from typing import List -from util.paths import Project - class BuildUnit: """ An independent unit of the build system that may come with its own built-time dependencies. """ + ROOT_DIRECTORY = 'build_system' + def __init__(self, *subdirectories: str): """ :param subdirectories: The subdirectories within the build system that lead to the root directory of this unit """ - self.root_directory = path.join(Project.BuildSystem.root_directory, *subdirectories) + self.root_directory = path.join(self.ROOT_DIRECTORY, *subdirectories) def find_requirements_files(self) -> List[str]: """ @@ -27,7 +27,7 @@ def find_requirements_files(self) -> List[str]: requirements_files = [] current_directory = self.root_directory - while path.basename(current_directory) != Project.BuildSystem.root_directory: + while path.basename(current_directory) != self.ROOT_DIRECTORY: requirements_file = path.join(current_directory, 'requirements.txt') if path.isfile(requirements_file): diff --git a/build_system/main.py b/build_system/main.py index bf7779cee..462c66401 100644 --- a/build_system/main.py +++ b/build_system/main.py @@ -7,6 +7,7 @@ from argparse import ArgumentParser from importlib.util import module_from_spec, spec_from_file_location +from os import path from types import ModuleType from typing import List @@ -15,7 +16,6 @@ from util.files import FileSearch from util.format import format_iterable from util.log import Log -from util.paths import Project def __parse_command_line_arguments(): @@ -35,7 +35,7 @@ def __find_init_files() -> List[str]: return FileSearch() \ .set_recursive(True) \ .filter_by_name('__init__.py') \ - .list(Project.BuildSystem.root_directory) + .list(path.dirname(__file__)) def __import_source_file(source_file: str) -> ModuleType: diff --git a/build_system/dependencies/__init__.py b/build_system/targets/__init__.py similarity index 100% rename from build_system/dependencies/__init__.py rename to build_system/targets/__init__.py diff --git a/build_system/code_style/__init__.py b/build_system/targets/code_style/__init__.py similarity index 59% rename from build_system/code_style/__init__.py rename to build_system/targets/code_style/__init__.py index bb3904ec5..3222f189c 100644 --- a/build_system/code_style/__init__.py +++ b/build_system/targets/code_style/__init__.py @@ -3,14 +3,15 @@ Defines targets for checking and enforcing code style definitions. """ -from code_style.cpp import FORMAT_CPP, TEST_FORMAT_CPP -from code_style.markdown import FORMAT_MARKDOWN, TEST_FORMAT_MARKDOWN -from code_style.python import FORMAT_PYTHON, TEST_FORMAT_PYTHON -from code_style.yaml import FORMAT_YAML, TEST_FORMAT_YAML from core.build_unit import BuildUnit from core.targets import TargetBuilder -TARGETS = TargetBuilder(BuildUnit('code_style')) \ +from targets.code_style.cpp import FORMAT_CPP, TEST_FORMAT_CPP +from targets.code_style.markdown import FORMAT_MARKDOWN, TEST_FORMAT_MARKDOWN +from targets.code_style.python import FORMAT_PYTHON, TEST_FORMAT_PYTHON +from targets.code_style.yaml import FORMAT_YAML, TEST_FORMAT_YAML + +TARGETS = TargetBuilder(BuildUnit('targets', 'code_style')) \ .add_phony_target('format') \ .depends_on(FORMAT_PYTHON, FORMAT_CPP, FORMAT_MARKDOWN, FORMAT_YAML) \ .nop() \ diff --git a/build_system/code_style/cpp/.clang-format b/build_system/targets/code_style/cpp/.clang-format similarity index 100% rename from build_system/code_style/cpp/.clang-format rename to build_system/targets/code_style/cpp/.clang-format diff --git a/build_system/code_style/cpp/__init__.py b/build_system/targets/code_style/cpp/__init__.py similarity index 74% rename from build_system/code_style/cpp/__init__.py rename to build_system/targets/code_style/cpp/__init__.py index cdc050b11..ac877fcea 100644 --- a/build_system/code_style/cpp/__init__.py +++ b/build_system/targets/code_style/cpp/__init__.py @@ -3,18 +3,19 @@ Defines targets and modules for checking and enforcing code style definitions for C++ files. """ -from code_style.cpp.targets import CheckCppCodeStyle, EnforceCppCodeStyle -from code_style.modules import CodeModule from core.build_unit import BuildUnit from core.targets import PhonyTarget, TargetBuilder from util.files import FileType -from util.paths import Project + +from targets.code_style.cpp.targets import CheckCppCodeStyle, EnforceCppCodeStyle +from targets.code_style.modules import CodeModule +from targets.paths import Project FORMAT_CPP = 'format_cpp' TEST_FORMAT_CPP = 'test_format_cpp' -TARGETS = TargetBuilder(BuildUnit('code_style', 'cpp')) \ +TARGETS = TargetBuilder(BuildUnit('targets', 'code_style', 'cpp')) \ .add_phony_target(FORMAT_CPP).set_runnables(EnforceCppCodeStyle()) \ .add_phony_target(TEST_FORMAT_CPP).set_runnables(CheckCppCodeStyle()) \ .build() diff --git a/build_system/code_style/cpp/clang_format.py b/build_system/targets/code_style/cpp/clang_format.py similarity index 95% rename from build_system/code_style/cpp/clang_format.py rename to build_system/targets/code_style/cpp/clang_format.py index f43fcaed1..1c86e16ac 100644 --- a/build_system/code_style/cpp/clang_format.py +++ b/build_system/targets/code_style/cpp/clang_format.py @@ -5,10 +5,11 @@ """ from os import path -from code_style.modules import CodeModule from core.build_unit import BuildUnit from util.run import Program +from targets.code_style.modules import CodeModule + class ClangFormat(Program): """ diff --git a/build_system/code_style/cpp/cpplint.py b/build_system/targets/code_style/cpp/cpplint.py similarity index 92% rename from build_system/code_style/cpp/cpplint.py rename to build_system/targets/code_style/cpp/cpplint.py index d214a2316..eba93b6e4 100644 --- a/build_system/code_style/cpp/cpplint.py +++ b/build_system/targets/code_style/cpp/cpplint.py @@ -3,10 +3,11 @@ Provides classes that allow to run the external program "cpplint". """ -from code_style.modules import CodeModule from core.build_unit import BuildUnit from util.run import Program +from targets.code_style.modules import CodeModule + class CppLint(Program): """ diff --git a/build_system/code_style/cpp/requirements.txt b/build_system/targets/code_style/cpp/requirements.txt similarity index 100% rename from build_system/code_style/cpp/requirements.txt rename to build_system/targets/code_style/cpp/requirements.txt diff --git a/build_system/code_style/cpp/targets.py b/build_system/targets/code_style/cpp/targets.py similarity index 88% rename from build_system/code_style/cpp/targets.py rename to build_system/targets/code_style/cpp/targets.py index a73fc6aa7..3bfce0286 100644 --- a/build_system/code_style/cpp/targets.py +++ b/build_system/targets/code_style/cpp/targets.py @@ -3,15 +3,16 @@ Implements targets for checking and enforcing code style definitions for C++ files. """ -from code_style.cpp.clang_format import ClangFormat -from code_style.cpp.cpplint import CppLint -from code_style.modules import CodeModule from core.build_unit import BuildUnit from core.modules import Module from core.targets import PhonyTarget from util.files import FileType from util.log import Log +from targets.code_style.cpp.clang_format import ClangFormat +from targets.code_style.cpp.cpplint import CppLint +from targets.code_style.modules import CodeModule + MODULE_FILTER = CodeModule.Filter(FileType.cpp()) diff --git a/build_system/code_style/markdown/__init__.py b/build_system/targets/code_style/markdown/__init__.py similarity index 80% rename from build_system/code_style/markdown/__init__.py rename to build_system/targets/code_style/markdown/__init__.py index b2840ea38..ff50f2395 100644 --- a/build_system/code_style/markdown/__init__.py +++ b/build_system/targets/code_style/markdown/__init__.py @@ -3,18 +3,19 @@ Defines targets and modules for checking and enforcing code style definitions for Markdown files. """ -from code_style.markdown.targets import CheckMarkdownCodeStyle, EnforceMarkdownCodeStyle -from code_style.modules import CodeModule from core.build_unit import BuildUnit from core.targets import PhonyTarget, TargetBuilder from util.files import FileSearch, FileType -from util.paths import Project + +from targets.code_style.markdown.targets import CheckMarkdownCodeStyle, EnforceMarkdownCodeStyle +from targets.code_style.modules import CodeModule +from targets.paths import Project FORMAT_MARKDOWN = 'format_md' TEST_FORMAT_MARKDOWN = 'test_format_md' -TARGETS = TargetBuilder(BuildUnit('code_style', 'markdown')) \ +TARGETS = TargetBuilder(BuildUnit('targets', 'code_style', 'markdown')) \ .add_phony_target(FORMAT_MARKDOWN).set_runnables(EnforceMarkdownCodeStyle()) \ .add_phony_target(TEST_FORMAT_MARKDOWN).set_runnables(CheckMarkdownCodeStyle()) \ .build() diff --git a/build_system/code_style/markdown/mdformat.py b/build_system/targets/code_style/markdown/mdformat.py similarity index 94% rename from build_system/code_style/markdown/mdformat.py rename to build_system/targets/code_style/markdown/mdformat.py index ce56745a7..40d8c1654 100644 --- a/build_system/code_style/markdown/mdformat.py +++ b/build_system/targets/code_style/markdown/mdformat.py @@ -3,10 +3,11 @@ Provides classes that allow to run the external program "mdformat". """ -from code_style.modules import CodeModule from core.build_unit import BuildUnit from util.run import Program +from targets.code_style.modules import CodeModule + class MdFormat(Program): """ diff --git a/build_system/code_style/markdown/requirements.txt b/build_system/targets/code_style/markdown/requirements.txt similarity index 100% rename from build_system/code_style/markdown/requirements.txt rename to build_system/targets/code_style/markdown/requirements.txt diff --git a/build_system/code_style/markdown/targets.py b/build_system/targets/code_style/markdown/targets.py similarity index 91% rename from build_system/code_style/markdown/targets.py rename to build_system/targets/code_style/markdown/targets.py index 4e50e5e25..e52735bd4 100644 --- a/build_system/code_style/markdown/targets.py +++ b/build_system/targets/code_style/markdown/targets.py @@ -3,14 +3,15 @@ Implements targets for checking and enforcing code style definitions for Markdown files. """ -from code_style.markdown.mdformat import MdFormat -from code_style.modules import CodeModule from core.build_unit import BuildUnit from core.modules import Module from core.targets import PhonyTarget from util.files import FileType from util.log import Log +from targets.code_style.markdown.mdformat import MdFormat +from targets.code_style.modules import CodeModule + MODULE_FILTER = CodeModule.Filter(FileType.markdown()) diff --git a/build_system/code_style/modules.py b/build_system/targets/code_style/modules.py similarity index 100% rename from build_system/code_style/modules.py rename to build_system/targets/code_style/modules.py diff --git a/build_system/code_style/python/.isort.cfg b/build_system/targets/code_style/python/.isort.cfg similarity index 100% rename from build_system/code_style/python/.isort.cfg rename to build_system/targets/code_style/python/.isort.cfg diff --git a/build_system/code_style/python/.pylintrc b/build_system/targets/code_style/python/.pylintrc similarity index 100% rename from build_system/code_style/python/.pylintrc rename to build_system/targets/code_style/python/.pylintrc diff --git a/build_system/code_style/python/.style.yapf b/build_system/targets/code_style/python/.style.yapf similarity index 100% rename from build_system/code_style/python/.style.yapf rename to build_system/targets/code_style/python/.style.yapf diff --git a/build_system/code_style/python/__init__.py b/build_system/targets/code_style/python/__init__.py similarity index 83% rename from build_system/code_style/python/__init__.py rename to build_system/targets/code_style/python/__init__.py index a559b45de..85ca4ef55 100644 --- a/build_system/code_style/python/__init__.py +++ b/build_system/targets/code_style/python/__init__.py @@ -3,19 +3,20 @@ Defines targets and modules for checking and enforcing code style definitions for Python and Cython files. """ -from code_style.modules import CodeModule -from code_style.python.targets import CheckCythonCodeStyle, CheckPythonCodeStyle, EnforceCythonCodeStyle, \ - EnforcePythonCodeStyle from core.build_unit import BuildUnit from core.targets import PhonyTarget, TargetBuilder from util.files import FileType -from util.paths import Project + +from targets.code_style.modules import CodeModule +from targets.code_style.python.targets import CheckCythonCodeStyle, CheckPythonCodeStyle, EnforceCythonCodeStyle, \ + EnforcePythonCodeStyle +from targets.paths import Project FORMAT_PYTHON = 'format_python' TEST_FORMAT_PYTHON = 'test_format_python' -TARGETS = TargetBuilder(BuildUnit('code_style', 'python')) \ +TARGETS = TargetBuilder(BuildUnit('targets', 'code_style', 'python')) \ .add_phony_target(FORMAT_PYTHON).set_runnables(EnforcePythonCodeStyle(), EnforceCythonCodeStyle()) \ .add_phony_target(TEST_FORMAT_PYTHON).set_runnables(CheckPythonCodeStyle(), CheckCythonCodeStyle()) \ .build() diff --git a/build_system/code_style/python/isort.py b/build_system/targets/code_style/python/isort.py similarity index 94% rename from build_system/code_style/python/isort.py rename to build_system/targets/code_style/python/isort.py index 9e4fe656e..b67bf3bca 100644 --- a/build_system/code_style/python/isort.py +++ b/build_system/targets/code_style/python/isort.py @@ -3,10 +3,11 @@ Provides classes that allow to run the external program "isort". """ -from code_style.modules import CodeModule from core.build_unit import BuildUnit from util.run import Program +from targets.code_style.modules import CodeModule + class ISort(Program): """ diff --git a/build_system/code_style/python/pylint.py b/build_system/targets/code_style/python/pylint.py similarity index 93% rename from build_system/code_style/python/pylint.py rename to build_system/targets/code_style/python/pylint.py index 7c98fda87..ad1ae2bf7 100644 --- a/build_system/code_style/python/pylint.py +++ b/build_system/targets/code_style/python/pylint.py @@ -5,10 +5,11 @@ """ from os import path -from code_style.modules import CodeModule from core.build_unit import BuildUnit from util.run import Program +from targets.code_style.modules import CodeModule + class PyLint(Program): """ diff --git a/build_system/code_style/python/requirements.txt b/build_system/targets/code_style/python/requirements.txt similarity index 100% rename from build_system/code_style/python/requirements.txt rename to build_system/targets/code_style/python/requirements.txt diff --git a/build_system/code_style/python/targets.py b/build_system/targets/code_style/python/targets.py similarity index 92% rename from build_system/code_style/python/targets.py rename to build_system/targets/code_style/python/targets.py index d629c0f54..b10e8fe8b 100644 --- a/build_system/code_style/python/targets.py +++ b/build_system/targets/code_style/python/targets.py @@ -3,16 +3,17 @@ Implements targets for checking and enforcing code style definitions for Python and Cython files. """ -from code_style.modules import CodeModule -from code_style.python.isort import ISort -from code_style.python.pylint import PyLint -from code_style.python.yapf import Yapf from core.build_unit import BuildUnit from core.modules import Module from core.targets import PhonyTarget from util.files import FileType from util.log import Log +from targets.code_style.modules import CodeModule +from targets.code_style.python.isort import ISort +from targets.code_style.python.pylint import PyLint +from targets.code_style.python.yapf import Yapf + PYTHON_MODULE_FILTER = CodeModule.Filter(FileType.python()) CYTHON_MODULE_FILTER = CodeModule.Filter(FileType.cython()) diff --git a/build_system/code_style/python/yapf.py b/build_system/targets/code_style/python/yapf.py similarity index 94% rename from build_system/code_style/python/yapf.py rename to build_system/targets/code_style/python/yapf.py index 81e97cd96..e18b77198 100644 --- a/build_system/code_style/python/yapf.py +++ b/build_system/targets/code_style/python/yapf.py @@ -5,10 +5,11 @@ """ from os import path -from code_style.modules import CodeModule from core.build_unit import BuildUnit from util.run import Program +from targets.code_style.modules import CodeModule + class Yapf(Program): """ diff --git a/build_system/code_style/yaml/.yamlfix.toml b/build_system/targets/code_style/yaml/.yamlfix.toml similarity index 100% rename from build_system/code_style/yaml/.yamlfix.toml rename to build_system/targets/code_style/yaml/.yamlfix.toml diff --git a/build_system/code_style/yaml/__init__.py b/build_system/targets/code_style/yaml/__init__.py similarity index 77% rename from build_system/code_style/yaml/__init__.py rename to build_system/targets/code_style/yaml/__init__.py index 44f5754d0..97a468e64 100644 --- a/build_system/code_style/yaml/__init__.py +++ b/build_system/targets/code_style/yaml/__init__.py @@ -3,18 +3,19 @@ Defines targets and modules for checking and enforcing code style definitions for YAML files. """ -from code_style.modules import CodeModule -from code_style.yaml.targets import CheckYamlCodeStyle, EnforceYamlCodeStyle from core.build_unit import BuildUnit from core.targets import PhonyTarget, TargetBuilder from util.files import FileSearch, FileType -from util.paths import Project + +from targets.code_style.modules import CodeModule +from targets.code_style.yaml.targets import CheckYamlCodeStyle, EnforceYamlCodeStyle +from targets.paths import Project FORMAT_YAML = 'format_yaml' TEST_FORMAT_YAML = 'test_format_yaml' -TARGETS = TargetBuilder(BuildUnit('code_style', 'yaml')) \ +TARGETS = TargetBuilder(BuildUnit('targets', 'code_style', 'yaml')) \ .add_phony_target(FORMAT_YAML).set_runnables(EnforceYamlCodeStyle()) \ .add_phony_target(TEST_FORMAT_YAML).set_runnables(CheckYamlCodeStyle()) \ .build() diff --git a/build_system/code_style/yaml/requirements.txt b/build_system/targets/code_style/yaml/requirements.txt similarity index 100% rename from build_system/code_style/yaml/requirements.txt rename to build_system/targets/code_style/yaml/requirements.txt diff --git a/build_system/code_style/yaml/targets.py b/build_system/targets/code_style/yaml/targets.py similarity index 92% rename from build_system/code_style/yaml/targets.py rename to build_system/targets/code_style/yaml/targets.py index 2ec44620d..1e1670d06 100644 --- a/build_system/code_style/yaml/targets.py +++ b/build_system/targets/code_style/yaml/targets.py @@ -3,14 +3,15 @@ Implements targets for checking and enforcing code style definitions for YAML files. """ -from code_style.modules import CodeModule -from code_style.yaml.yamlfix import YamlFix from core.build_unit import BuildUnit from core.modules import Module from core.targets import PhonyTarget from util.files import FileType from util.log import Log +from targets.code_style.modules import CodeModule +from targets.code_style.yaml.yamlfix import YamlFix + MODULE_FILTER = CodeModule.Filter(FileType.yaml()) diff --git a/build_system/code_style/yaml/yamlfix.py b/build_system/targets/code_style/yaml/yamlfix.py similarity index 95% rename from build_system/code_style/yaml/yamlfix.py rename to build_system/targets/code_style/yaml/yamlfix.py index 546b466b4..6becdfafc 100644 --- a/build_system/code_style/yaml/yamlfix.py +++ b/build_system/targets/code_style/yaml/yamlfix.py @@ -5,10 +5,11 @@ """ from os import path -from code_style.modules import CodeModule from core.build_unit import BuildUnit from util.run import Program +from targets.code_style.modules import CodeModule + class YamlFix(Program): """ diff --git a/build_system/compilation/__init__.py b/build_system/targets/compilation/__init__.py similarity index 70% rename from build_system/compilation/__init__.py rename to build_system/targets/compilation/__init__.py index 31871ed73..c3d6db12f 100644 --- a/build_system/compilation/__init__.py +++ b/build_system/targets/compilation/__init__.py @@ -3,14 +3,15 @@ Defines targets for compiling code. """ -from compilation.cpp import COMPILE_CPP, INSTALL_CPP -from compilation.cython import COMPILE_CYTHON, INSTALL_CYTHON from core.build_unit import BuildUnit from core.targets import TargetBuilder +from targets.compilation.cpp import COMPILE_CPP, INSTALL_CPP +from targets.compilation.cython import COMPILE_CYTHON, INSTALL_CYTHON + INSTALL = 'install' -TARGETS = TargetBuilder(BuildUnit('compilation')) \ +TARGETS = TargetBuilder(BuildUnit('targets', 'compilation')) \ .add_phony_target('compile') \ .depends_on(COMPILE_CPP, COMPILE_CYTHON, clean_dependencies=True) \ .nop() \ diff --git a/build_system/compilation/build_options.py b/build_system/targets/compilation/build_options.py similarity index 100% rename from build_system/compilation/build_options.py rename to build_system/targets/compilation/build_options.py diff --git a/build_system/compilation/cpp/__init__.py b/build_system/targets/compilation/cpp/__init__.py similarity index 77% rename from build_system/compilation/cpp/__init__.py rename to build_system/targets/compilation/cpp/__init__.py index 3989a96a0..a3cc49a2d 100644 --- a/build_system/compilation/cpp/__init__.py +++ b/build_system/targets/compilation/cpp/__init__.py @@ -3,13 +3,14 @@ Defines targets and modules for compiling C++ code. """ -from compilation.cpp.targets import CompileCpp, InstallCpp, SetupCpp -from compilation.modules import CompilationModule from core.build_unit import BuildUnit from core.targets import PhonyTarget, TargetBuilder -from dependencies.python import VENV from util.files import FileType -from util.paths import Project + +from targets.compilation.cpp.targets import CompileCpp, InstallCpp, SetupCpp +from targets.compilation.modules import CompilationModule +from targets.dependencies.python import VENV +from targets.paths import Project SETUP_CPP = 'setup_cpp' @@ -17,7 +18,7 @@ INSTALL_CPP = 'install_cpp' -TARGETS = TargetBuilder(BuildUnit('compilation', 'cpp')) \ +TARGETS = TargetBuilder(BuildUnit('targets', 'compilation', 'cpp')) \ .add_build_target(SETUP_CPP) \ .depends_on(VENV) \ .set_runnables(SetupCpp()) \ diff --git a/build_system/compilation/cpp/targets.py b/build_system/targets/compilation/cpp/targets.py similarity index 91% rename from build_system/compilation/cpp/targets.py rename to build_system/targets/compilation/cpp/targets.py index 9e2d3a83f..3d0aee389 100644 --- a/build_system/compilation/cpp/targets.py +++ b/build_system/targets/compilation/cpp/targets.py @@ -5,15 +5,16 @@ """ from typing import List -from compilation.build_options import BuildOptions, EnvBuildOption -from compilation.meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup -from compilation.modules import CompilationModule from core.build_unit import BuildUnit from core.modules import Module from core.targets import BuildTarget, PhonyTarget from util.files import FileType from util.log import Log +from targets.compilation.build_options import BuildOptions, EnvBuildOption +from targets.compilation.meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup +from targets.compilation.modules import CompilationModule + MODULE_FILTER = CompilationModule.Filter(FileType.cpp()) BUILD_OPTIONS = BuildOptions() \ diff --git a/build_system/compilation/cython/__init__.py b/build_system/targets/compilation/cython/__init__.py similarity index 76% rename from build_system/compilation/cython/__init__.py rename to build_system/targets/compilation/cython/__init__.py index a03bb505a..72c43e858 100644 --- a/build_system/compilation/cython/__init__.py +++ b/build_system/targets/compilation/cython/__init__.py @@ -3,13 +3,14 @@ Defines targets and modules for compiling Cython code. """ -from compilation.cpp import COMPILE_CPP -from compilation.cython.targets import CompileCython, InstallCython, SetupCython -from compilation.modules import CompilationModule from core.build_unit import BuildUnit from core.targets import PhonyTarget, TargetBuilder from util.files import FileType -from util.paths import Project + +from targets.compilation.cpp import COMPILE_CPP +from targets.compilation.cython.targets import CompileCython, InstallCython, SetupCython +from targets.compilation.modules import CompilationModule +from targets.paths import Project SETUP_CYTHON = 'setup_cython' @@ -17,7 +18,7 @@ INSTALL_CYTHON = 'install_cython' -TARGETS = TargetBuilder(BuildUnit('compilation', 'cython')) \ +TARGETS = TargetBuilder(BuildUnit('targets', 'compilation', 'cython')) \ .add_build_target(SETUP_CYTHON) \ .depends_on(COMPILE_CPP) \ .set_runnables(SetupCython()) \ diff --git a/build_system/compilation/cython/requirements.txt b/build_system/targets/compilation/cython/requirements.txt similarity index 100% rename from build_system/compilation/cython/requirements.txt rename to build_system/targets/compilation/cython/requirements.txt diff --git a/build_system/compilation/cython/targets.py b/build_system/targets/compilation/cython/targets.py similarity index 90% rename from build_system/compilation/cython/targets.py rename to build_system/targets/compilation/cython/targets.py index 6730fa371..a69f5bbe8 100644 --- a/build_system/compilation/cython/targets.py +++ b/build_system/targets/compilation/cython/targets.py @@ -5,15 +5,16 @@ """ from typing import List -from compilation.build_options import BuildOptions, EnvBuildOption -from compilation.meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup -from compilation.modules import CompilationModule from core.build_unit import BuildUnit from core.modules import Module from core.targets import BuildTarget, PhonyTarget from util.files import FileType from util.log import Log +from targets.compilation.build_options import BuildOptions, EnvBuildOption +from targets.compilation.meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup +from targets.compilation.modules import CompilationModule + MODULE_FILTER = CompilationModule.Filter(FileType.cython()) BUILD_OPTIONS = BuildOptions() \ diff --git a/build_system/compilation/meson.py b/build_system/targets/compilation/meson.py similarity index 96% rename from build_system/compilation/meson.py rename to build_system/targets/compilation/meson.py index 3e12abd62..93cbf4c2a 100644 --- a/build_system/compilation/meson.py +++ b/build_system/targets/compilation/meson.py @@ -6,12 +6,13 @@ from abc import ABC from typing import List -from compilation.build_options import BuildOptions -from compilation.modules import CompilationModule from core.build_unit import BuildUnit from util.log import Log from util.run import Program +from targets.compilation.build_options import BuildOptions +from targets.compilation.modules import CompilationModule + def build_options_as_meson_arguments(build_options: BuildOptions) -> List[str]: """ diff --git a/build_system/compilation/modules.py b/build_system/targets/compilation/modules.py similarity index 100% rename from build_system/compilation/modules.py rename to build_system/targets/compilation/modules.py diff --git a/build_system/compilation/requirements.txt b/build_system/targets/compilation/requirements.txt similarity index 100% rename from build_system/compilation/requirements.txt rename to build_system/targets/compilation/requirements.txt diff --git a/build_system/targets/dependencies/__init__.py b/build_system/targets/dependencies/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/build_system/dependencies/github/__init__.py b/build_system/targets/dependencies/github/__init__.py similarity index 64% rename from build_system/dependencies/github/__init__.py rename to build_system/targets/dependencies/github/__init__.py index 6e85d9f4b..029b95544 100644 --- a/build_system/dependencies/github/__init__.py +++ b/build_system/targets/dependencies/github/__init__.py @@ -5,11 +5,12 @@ """ from core.build_unit import BuildUnit from core.targets import PhonyTarget, TargetBuilder -from dependencies.github.modules import GithubWorkflowModule -from dependencies.github.targets import CheckGithubActions, UpdateGithubActions -from util.paths import Project -TARGETS = TargetBuilder(BuildUnit('dependencies', 'github')) \ +from targets.dependencies.github.modules import GithubWorkflowModule +from targets.dependencies.github.targets import CheckGithubActions, UpdateGithubActions +from targets.paths import Project + +TARGETS = TargetBuilder(BuildUnit('targets', 'dependencies', 'github')) \ .add_phony_target('check_github_actions').set_runnables(CheckGithubActions()) \ .add_phony_target('update_github_actions').set_runnables(UpdateGithubActions()) \ .build() diff --git a/build_system/dependencies/github/actions.py b/build_system/targets/dependencies/github/actions.py similarity index 98% rename from build_system/dependencies/github/actions.py rename to build_system/targets/dependencies/github/actions.py index 1355a5d4a..9b5e4c17b 100644 --- a/build_system/dependencies/github/actions.py +++ b/build_system/targets/dependencies/github/actions.py @@ -9,12 +9,13 @@ from typing import Dict, List, Optional, Set from core.build_unit import BuildUnit -from dependencies.github.modules import GithubWorkflowModule -from dependencies.github.pygithub import GithubApi -from dependencies.github.pyyaml import YamlFile from util.env import get_env from util.log import Log +from targets.dependencies.github.modules import GithubWorkflowModule +from targets.dependencies.github.pygithub import GithubApi +from targets.dependencies.github.pyyaml import YamlFile + @dataclass class ActionVersion: diff --git a/build_system/dependencies/github/modules.py b/build_system/targets/dependencies/github/modules.py similarity index 100% rename from build_system/dependencies/github/modules.py rename to build_system/targets/dependencies/github/modules.py diff --git a/build_system/dependencies/github/pygithub.py b/build_system/targets/dependencies/github/pygithub.py similarity index 100% rename from build_system/dependencies/github/pygithub.py rename to build_system/targets/dependencies/github/pygithub.py diff --git a/build_system/dependencies/github/pyyaml.py b/build_system/targets/dependencies/github/pyyaml.py similarity index 100% rename from build_system/dependencies/github/pyyaml.py rename to build_system/targets/dependencies/github/pyyaml.py diff --git a/build_system/dependencies/github/requirements.txt b/build_system/targets/dependencies/github/requirements.txt similarity index 100% rename from build_system/dependencies/github/requirements.txt rename to build_system/targets/dependencies/github/requirements.txt diff --git a/build_system/dependencies/github/targets.py b/build_system/targets/dependencies/github/targets.py similarity index 92% rename from build_system/dependencies/github/targets.py rename to build_system/targets/dependencies/github/targets.py index 7f9eb6782..71613a349 100644 --- a/build_system/dependencies/github/targets.py +++ b/build_system/targets/dependencies/github/targets.py @@ -6,11 +6,12 @@ from core.build_unit import BuildUnit from core.modules import Module from core.targets import PhonyTarget -from dependencies.github.actions import WorkflowUpdater -from dependencies.github.modules import GithubWorkflowModule -from dependencies.table import Table from util.log import Log +from targets.dependencies.github.actions import WorkflowUpdater +from targets.dependencies.github.modules import GithubWorkflowModule +from targets.dependencies.table import Table + MODULE_FILTER = GithubWorkflowModule.Filter() diff --git a/build_system/dependencies/python/__init__.py b/build_system/targets/dependencies/python/__init__.py similarity index 78% rename from build_system/dependencies/python/__init__.py rename to build_system/targets/dependencies/python/__init__.py index 165e06289..f3be20ae6 100644 --- a/build_system/dependencies/python/__init__.py +++ b/build_system/targets/dependencies/python/__init__.py @@ -5,13 +5,14 @@ """ from core.build_unit import BuildUnit from core.targets import PhonyTarget, TargetBuilder -from dependencies.python.modules import DependencyType, PythonDependencyModule -from dependencies.python.targets import CheckPythonDependencies, InstallRuntimeDependencies -from util.paths import Project + +from targets.dependencies.python.modules import DependencyType, PythonDependencyModule +from targets.dependencies.python.targets import CheckPythonDependencies, InstallRuntimeDependencies +from targets.paths import Project VENV = 'venv' -TARGETS = TargetBuilder(BuildUnit('dependencies', 'python')) \ +TARGETS = TargetBuilder(BuildUnit('targets', 'dependencies', 'python')) \ .add_phony_target(VENV).set_runnables(InstallRuntimeDependencies()) \ .add_phony_target('check_dependencies').set_runnables(CheckPythonDependencies()) \ .build() diff --git a/build_system/dependencies/python/modules.py b/build_system/targets/dependencies/python/modules.py similarity index 98% rename from build_system/dependencies/python/modules.py rename to build_system/targets/dependencies/python/modules.py index 45efc7d04..c487f60c9 100644 --- a/build_system/dependencies/python/modules.py +++ b/build_system/targets/dependencies/python/modules.py @@ -3,7 +3,7 @@ Provides classes that provide access to Python requirements files that belong to individual modules. """ -from enum import Enum, auto +from enum import Enum from typing import List from core.modules import Module diff --git a/build_system/dependencies/python/pip.py b/build_system/targets/dependencies/python/pip.py similarity index 100% rename from build_system/dependencies/python/pip.py rename to build_system/targets/dependencies/python/pip.py diff --git a/build_system/dependencies/python/targets.py b/build_system/targets/dependencies/python/targets.py similarity index 91% rename from build_system/dependencies/python/targets.py rename to build_system/targets/dependencies/python/targets.py index 20d526ded..bdb7b0d75 100644 --- a/build_system/dependencies/python/targets.py +++ b/build_system/targets/dependencies/python/targets.py @@ -9,11 +9,12 @@ from core.build_unit import BuildUnit from core.modules import Module from core.targets import PhonyTarget -from dependencies.python.modules import DependencyType, PythonDependencyModule -from dependencies.python.pip import PipList -from dependencies.table import Table from util.log import Log +from targets.dependencies.python.modules import DependencyType, PythonDependencyModule +from targets.dependencies.python.pip import PipList +from targets.dependencies.table import Table + class InstallRuntimeDependencies(PhonyTarget.Runnable): """ diff --git a/build_system/dependencies/requirements.txt b/build_system/targets/dependencies/requirements.txt similarity index 100% rename from build_system/dependencies/requirements.txt rename to build_system/targets/dependencies/requirements.txt diff --git a/build_system/dependencies/table.py b/build_system/targets/dependencies/table.py similarity index 100% rename from build_system/dependencies/table.py rename to build_system/targets/dependencies/table.py diff --git a/build_system/packaging/__init__.py b/build_system/targets/packaging/__init__.py similarity index 75% rename from build_system/packaging/__init__.py rename to build_system/targets/packaging/__init__.py index 67a2ac010..727dec43a 100644 --- a/build_system/packaging/__init__.py +++ b/build_system/targets/packaging/__init__.py @@ -5,18 +5,19 @@ """ from os import path -from compilation import INSTALL from core.build_unit import BuildUnit from core.targets import TargetBuilder -from packaging.modules import PythonPackageModule -from packaging.targets import BuildPythonWheels, InstallPythonWheels -from util.paths import Project + +from targets.compilation import INSTALL +from targets.packaging.modules import PythonPackageModule +from targets.packaging.targets import BuildPythonWheels, InstallPythonWheels +from targets.paths import Project BUILD_WHEELS = 'build_wheels' INSTALL_WHEELS = 'install_wheels' -TARGETS = TargetBuilder(BuildUnit('packaging')) \ +TARGETS = TargetBuilder(BuildUnit('targets', 'packaging')) \ .add_build_target(BUILD_WHEELS) \ .depends_on(INSTALL) \ .set_runnables(BuildPythonWheels()) \ diff --git a/build_system/packaging/build.py b/build_system/targets/packaging/build.py similarity index 92% rename from build_system/packaging/build.py rename to build_system/targets/packaging/build.py index c2a755292..bf67ae54e 100644 --- a/build_system/packaging/build.py +++ b/build_system/targets/packaging/build.py @@ -4,9 +4,10 @@ Provides classes that allow to build wheel packages via the external program "build". """ from core.build_unit import BuildUnit -from packaging.modules import PythonPackageModule from util.run import PythonModule +from targets.packaging.modules import PythonPackageModule + class Build(PythonModule): """ diff --git a/build_system/packaging/modules.py b/build_system/targets/packaging/modules.py similarity index 100% rename from build_system/packaging/modules.py rename to build_system/targets/packaging/modules.py diff --git a/build_system/packaging/pip.py b/build_system/targets/packaging/pip.py similarity index 100% rename from build_system/packaging/pip.py rename to build_system/targets/packaging/pip.py diff --git a/build_system/packaging/requirements.txt b/build_system/targets/packaging/requirements.txt similarity index 100% rename from build_system/packaging/requirements.txt rename to build_system/targets/packaging/requirements.txt diff --git a/build_system/packaging/targets.py b/build_system/targets/packaging/targets.py similarity index 92% rename from build_system/packaging/targets.py rename to build_system/targets/packaging/targets.py index f2b0416d0..5375f56b6 100644 --- a/build_system/packaging/targets.py +++ b/build_system/targets/packaging/targets.py @@ -8,12 +8,13 @@ from core.build_unit import BuildUnit from core.modules import Module from core.targets import BuildTarget -from packaging.build import Build -from packaging.modules import PythonPackageModule -from packaging.pip import PipInstallWheel from util.files import DirectorySearch, FileType from util.log import Log -from util.paths import Project + +from targets.packaging.build import Build +from targets.packaging.modules import PythonPackageModule +from targets.packaging.pip import PipInstallWheel +from targets.paths import Project MODULE_FILTER = PythonPackageModule.Filter() diff --git a/build_system/util/paths.py b/build_system/targets/paths.py similarity index 100% rename from build_system/util/paths.py rename to build_system/targets/paths.py diff --git a/build_system/testing/__init__.py b/build_system/targets/testing/__init__.py similarity index 65% rename from build_system/testing/__init__.py rename to build_system/targets/testing/__init__.py index 8fa2ad4de..fd0c49806 100644 --- a/build_system/testing/__init__.py +++ b/build_system/targets/testing/__init__.py @@ -5,10 +5,11 @@ """ from core.build_unit import BuildUnit from core.targets import TargetBuilder -from testing.cpp import TESTS_CPP -from testing.python import TESTS_PYTHON -TARGETS = TargetBuilder(BuildUnit('testing')) \ +from targets.testing.cpp import TESTS_CPP +from targets.testing.python import TESTS_PYTHON + +TARGETS = TargetBuilder(BuildUnit('targets', 'testing')) \ .add_phony_target('tests') \ .depends_on(TESTS_CPP, TESTS_PYTHON) \ .nop() \ diff --git a/build_system/testing/cpp/__init__.py b/build_system/targets/testing/cpp/__init__.py similarity index 66% rename from build_system/testing/cpp/__init__.py rename to build_system/targets/testing/cpp/__init__.py index 650628790..d14319cfc 100644 --- a/build_system/testing/cpp/__init__.py +++ b/build_system/targets/testing/cpp/__init__.py @@ -3,16 +3,17 @@ Defines targets and modules for testing C++ code. """ -from compilation.cpp import COMPILE_CPP from core.build_unit import BuildUnit from core.targets import PhonyTarget, TargetBuilder -from testing.cpp.modules import CppTestModule -from testing.cpp.targets import TestCpp -from util.paths import Project + +from targets.compilation.cpp import COMPILE_CPP +from targets.paths import Project +from targets.testing.cpp.modules import CppTestModule +from targets.testing.cpp.targets import TestCpp TESTS_CPP = 'tests_cpp' -TARGETS = TargetBuilder(BuildUnit('testing', 'cpp')) \ +TARGETS = TargetBuilder(BuildUnit('targets', 'testing', 'cpp')) \ .add_phony_target(TESTS_CPP) \ .depends_on(COMPILE_CPP) \ .set_runnables(TestCpp()) \ diff --git a/build_system/testing/cpp/meson.py b/build_system/targets/testing/cpp/meson.py similarity index 87% rename from build_system/testing/cpp/meson.py rename to build_system/targets/testing/cpp/meson.py index ff6499cc8..8c8a01b9a 100644 --- a/build_system/testing/cpp/meson.py +++ b/build_system/targets/testing/cpp/meson.py @@ -3,9 +3,10 @@ Provides classes that allow to run automated tests via the external program "meson". """ -from compilation.meson import Meson from core.build_unit import BuildUnit -from testing.cpp.modules import CppTestModule + +from targets.compilation.meson import Meson +from targets.testing.cpp.modules import CppTestModule class MesonTest(Meson): diff --git a/build_system/testing/cpp/modules.py b/build_system/targets/testing/cpp/modules.py similarity index 96% rename from build_system/testing/cpp/modules.py rename to build_system/targets/testing/cpp/modules.py index 816b2e35a..b294742b3 100644 --- a/build_system/testing/cpp/modules.py +++ b/build_system/targets/testing/cpp/modules.py @@ -6,7 +6,8 @@ from os import path from core.modules import Module -from testing.modules import TestModule + +from targets.testing.modules import TestModule class CppTestModule(TestModule): diff --git a/build_system/testing/cpp/targets.py b/build_system/targets/testing/cpp/targets.py similarity index 82% rename from build_system/testing/cpp/targets.py rename to build_system/targets/testing/cpp/targets.py index 6bb65591c..0784b7f0b 100644 --- a/build_system/testing/cpp/targets.py +++ b/build_system/targets/testing/cpp/targets.py @@ -6,8 +6,9 @@ from core.build_unit import BuildUnit from core.modules import Module from core.targets import BuildTarget, PhonyTarget -from testing.cpp.meson import MesonTest -from testing.cpp.modules import CppTestModule + +from targets.testing.cpp.meson import MesonTest +from targets.testing.cpp.modules import CppTestModule class TestCpp(PhonyTarget.Runnable): diff --git a/build_system/testing/modules.py b/build_system/targets/testing/modules.py similarity index 100% rename from build_system/testing/modules.py rename to build_system/targets/testing/modules.py diff --git a/build_system/testing/python/__init__.py b/build_system/targets/testing/python/__init__.py similarity index 71% rename from build_system/testing/python/__init__.py rename to build_system/targets/testing/python/__init__.py index ca31332fe..f49e01ef0 100644 --- a/build_system/testing/python/__init__.py +++ b/build_system/targets/testing/python/__init__.py @@ -5,14 +5,15 @@ """ from core.build_unit import BuildUnit from core.targets import PhonyTarget, TargetBuilder -from packaging import INSTALL_WHEELS -from testing.python.modules import PythonTestModule -from testing.python.targets import TestPython -from util.paths import Project + +from targets.packaging import INSTALL_WHEELS +from targets.paths import Project +from targets.testing.python.modules import PythonTestModule +from targets.testing.python.targets import TestPython TESTS_PYTHON = 'tests_python' -TARGETS = TargetBuilder(BuildUnit('testing', 'python')) \ +TARGETS = TargetBuilder(BuildUnit('targets', 'testing', 'python')) \ .add_phony_target(TESTS_PYTHON) \ .depends_on(INSTALL_WHEELS) \ .set_runnables(TestPython()) \ diff --git a/build_system/testing/python/modules.py b/build_system/targets/testing/python/modules.py similarity index 97% rename from build_system/testing/python/modules.py rename to build_system/targets/testing/python/modules.py index d1475ea48..5d28c6955 100644 --- a/build_system/testing/python/modules.py +++ b/build_system/targets/testing/python/modules.py @@ -7,9 +7,10 @@ from typing import List from core.modules import Module -from testing.modules import TestModule from util.files import FileSearch, FileType +from targets.testing.modules import TestModule + class PythonTestModule(TestModule): """ diff --git a/build_system/testing/python/requirements.txt b/build_system/targets/testing/python/requirements.txt similarity index 100% rename from build_system/testing/python/requirements.txt rename to build_system/targets/testing/python/requirements.txt diff --git a/build_system/testing/python/targets.py b/build_system/targets/testing/python/targets.py similarity index 81% rename from build_system/testing/python/targets.py rename to build_system/targets/testing/python/targets.py index 38105ccb7..9290267ce 100644 --- a/build_system/testing/python/targets.py +++ b/build_system/targets/testing/python/targets.py @@ -6,8 +6,9 @@ from core.build_unit import BuildUnit from core.modules import Module from core.targets import PhonyTarget -from testing.python.modules import PythonTestModule -from testing.python.unittest import UnitTest + +from targets.testing.python.modules import PythonTestModule +from targets.testing.python.unittest import UnitTest class TestPython(PhonyTarget.Runnable): diff --git a/build_system/testing/python/unittest.py b/build_system/targets/testing/python/unittest.py similarity index 95% rename from build_system/testing/python/unittest.py rename to build_system/targets/testing/python/unittest.py index e258c3976..b506a5729 100644 --- a/build_system/testing/python/unittest.py +++ b/build_system/targets/testing/python/unittest.py @@ -4,9 +4,10 @@ Provides classes that allow to run automated tests via the external program "unittest". """ from core.build_unit import BuildUnit -from testing.python.modules import PythonTestModule from util.run import PythonModule +from targets.testing.python.modules import PythonTestModule + class UnitTest: """ diff --git a/build_system/versioning/__init__.py b/build_system/targets/versioning/__init__.py similarity index 81% rename from build_system/versioning/__init__.py rename to build_system/targets/versioning/__init__.py index 6a7464090..4051db366 100644 --- a/build_system/versioning/__init__.py +++ b/build_system/targets/versioning/__init__.py @@ -3,10 +3,12 @@ """ from core.build_unit import BuildUnit from core.targets import PhonyTarget, TargetBuilder -from versioning.changelog import print_latest_changelog, update_changelog_bugfix, update_changelog_feature, \ + +from targets.versioning.changelog import print_latest_changelog, update_changelog_bugfix, update_changelog_feature, \ update_changelog_main, validate_changelog_bugfix, validate_changelog_feature, validate_changelog_main -from versioning.versioning import apply_development_version, increment_development_version, increment_major_version, \ - increment_minor_version, increment_patch_version, print_current_version, reset_development_version +from targets.versioning.versioning import apply_development_version, increment_development_version, \ + increment_major_version, increment_minor_version, increment_patch_version, print_current_version, \ + reset_development_version TARGETS = TargetBuilder(BuildUnit('util')) \ .add_phony_target('increment_development_version').set_functions(increment_development_version) \ diff --git a/build_system/versioning/changelog.py b/build_system/targets/versioning/changelog.py similarity index 99% rename from build_system/versioning/changelog.py rename to build_system/targets/versioning/changelog.py index c04b98dce..ae43201dd 100644 --- a/build_system/versioning/changelog.py +++ b/build_system/targets/versioning/changelog.py @@ -13,7 +13,8 @@ from util.io import TextFile from util.log import Log -from versioning.versioning import Version, get_current_version + +from targets.versioning.versioning import Version, get_current_version CHANGESET_FILE_MAIN = '.changelog-main.md' diff --git a/build_system/versioning/versioning.py b/build_system/targets/versioning/versioning.py similarity index 100% rename from build_system/versioning/versioning.py rename to build_system/targets/versioning/versioning.py From a8e5af1a2e8f1fa8a63d0f72d5221dfe22c928e0 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 11 Dec 2024 16:21:57 +0100 Subject: [PATCH 088/114] Only run build targets if output files are missing or input files have changed. --- build_system/core/changes.py | 134 ++++++++++++++++++++++ build_system/core/targets.py | 120 +++++++++---------- build_system/targets/packaging/targets.py | 2 +- 3 files changed, 195 insertions(+), 61 deletions(-) create mode 100644 build_system/core/changes.py diff --git a/build_system/core/changes.py b/build_system/core/changes.py new file mode 100644 index 000000000..13aba4d14 --- /dev/null +++ b/build_system/core/changes.py @@ -0,0 +1,134 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes for detecting changes in files. +""" +import json + +from functools import cached_property +from os import path +from typing import Dict, List, Set + +from core.modules import Module +from util.io import TextFile + + +class JsonFile(TextFile): + """ + Allows to read and write the content of a JSON file. + """ + + def __init__(self, file: str, accept_missing: bool = False): + """ + :param file: The path to the JSON file + :param accept_missing: True, if no errors should be raised if the text file is missing, False otherwise + """ + super().__init__(file, accept_missing) + + @cached_property + def json(self) -> Dict: + """ + The content of the JSON file as a dictionary. + """ + lines = self.lines + + if lines: + return json.loads('\n'.join(lines)) + + return {} + + def write_json(self, dictionary: Dict): + """ + Writes a given dictionary to the JSON file. + + :param dictionary: The dictionary to be written + """ + self.write_lines(json.dumps(dictionary, indent=4)) + + def write_lines(self, *lines: str): + super().write_lines(*lines) + del self.json + + +class ChangeDetection: + """ + Allows to detect changes in tracked files. + """ + + class CacheFile(JsonFile): + """ + A JSON file that stores checksums for tracked files. + """ + + @staticmethod + def __checksum(file: str) -> str: + return str(path.getmtime(file)) + + def __init__(self, file: str): + """ + :param file: The path to the JSON file + """ + super().__init__(file, accept_missing=True) + + def update(self, module_name: str, files: Set[str]): + """ + Updates the checksums of given files. + + :param module_name: The name of the module, the files belong to + :param files: A set that contains the paths of the files to be updated + """ + cache = self.json + module_cache = cache.setdefault(module_name, {}) + + for invalid_key in [file for file in module_cache.keys() if file not in files]: + del module_cache[invalid_key] + + for file in files: + module_cache[file] = self.__checksum(file) + + if module_cache: + cache[module_name] = module_cache + else: + del cache[module_name] + + if cache: + self.write_json(cache) + else: + self.delete() + + def has_changed(self, module_name: str, file: str) -> bool: + """ + Returns whether a file has changed according to the cache or not. + + :param module_name: The name of the module, the file belongs to + :param file: The file to be checked + :return: True, if the file has changed, False otherwise + """ + module_cache = self.json.get(module_name, {}) + return file not in module_cache or module_cache[file] != self.__checksum(file) + + def __init__(self, cache_file: str): + """ + :param cache_file: The path to the file that should be used for tracking files + """ + self.cache_file = ChangeDetection.CacheFile(cache_file) + + def track_files(self, module: Module, *files: str): + """ + Updates the cache to keep track of given files. + + :param module: The module, the files belong to + :param files: The files to be tracked + """ + self.cache_file.update(str(module), set(files)) + + def get_changed_files(self, module: Module, *files: str) -> List[str]: + """ + Filters given files and returns only those that have changed. + + :param module: The module, the files belong to + :param files: The files to be filtered + :return: A list that contains the files that have changed + """ + module_name = str(module) + return [file for file in files if self.cache_file.has_changed(module_name, file)] diff --git a/build_system/core/targets.py b/build_system/core/targets.py index 048fd55b5..2629ccbfb 100644 --- a/build_system/core/targets.py +++ b/build_system/core/targets.py @@ -7,9 +7,10 @@ from dataclasses import dataclass from functools import reduce from os import path -from typing import Any, Callable, Dict, Iterable, List, Optional +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple from core.build_unit import BuildUnit +from core.changes import ChangeDetection from core.modules import Module, ModuleRegistry from util.format import format_iterable from util.io import delete_files @@ -253,64 +254,6 @@ def get_clean_files(self, module: Module) -> List[str]: """ return self.get_output_files(module) - def output_files_exist(self, module: Module) -> bool: - """ - Returns whether all output files produced by the target exist or not. - - :param module: The module, the target should be applied to - :return: True, if all output files exist, False otherwise - """ - output_files = self.get_output_files(module) - - if output_files: - missing_files = [output_file for output_file in output_files if not path.exists(output_file)] - - if missing_files: - Log.verbose( - 'Target needs to be applied to module %s, because the following output files do not exist:', - str(module)) - - for missing_file in missing_files: - Log.verbose(' - %s', missing_file) - - Log.verbose('') - return False - - Log.verbose('All output files of module %s already exist.', str(module)) - return True - - return False - - def input_files_have_changed(self, module: Module) -> bool: - """ - Returns whether any input files required by the target have changed since the target was run for the last - time. - - :param module: The module, the target should be applied to - :return: True, if any input files have changed, False otherwise - """ - input_files = self.get_input_files(module) - - if input_files: - # TODO check timestamps or hashes - changed_files = input_files - - if changed_files: - Log.verbose( - 'Target needs to be applied to module %s, because the following input files have changed:\n', - str(module)) - - for input_file in input_files: - Log.verbose(' - %s', input_file) - - Log.verbose('') - return True - - Log.verbose('No input files of module %s have changed.', str(module)) - return False - - return True - class Builder(Target.Builder): """ A builder that allows to configure and create build targets. @@ -337,6 +280,58 @@ def set_runnables(self, *runnables: 'BuildTarget.Runnable') -> Any: def _build(self, build_unit: BuildUnit) -> Target: return BuildTarget(self.target_name, self.dependencies, self.runnables, build_unit) + def __get_missing_output_files(self, runnable: Runnable, module: Module) -> Tuple[List[str], List[str]]: + output_files = runnable.get_output_files(module) + missing_output_files = [output_file for output_file in output_files if not path.exists(output_file)] + + if output_files: + if missing_output_files: + Log.verbose( + 'Target "%s" must be applied to module "%s", because the following output files do not exist:\n', + self.name, str(module)) + + for missing_output_file in missing_output_files: + Log.verbose(' - %s', missing_output_file) + + Log.verbose('') + else: + Log.verbose('Target "%s" must not be applied to module "%s", because all output files already exist:\n', + self.name, str(module)) + + for output_file in output_files: + Log.verbose(' - %s', output_file) + + Log.verbose('') + + return output_files, missing_output_files + + def __get_changed_input_files(self, runnable: Runnable, module: Module) -> Tuple[List[str], List[str]]: + input_files = runnable.get_input_files(module) + changed_input_files = [ + input_file for input_file in input_files if self.change_detection.get_changed_files(module, *input_files) + ] + + if input_files: + if changed_input_files: + Log.verbose( + 'Target "%s" must be applied to module "%s", because the following input files have changed:\n', + self.name, str(module)) + + for changed_input_file in changed_input_files: + Log.verbose(' - %s', changed_input_file) + + Log.verbose('') + else: + Log.verbose('Target "%s" must not be applied to module "%s", because no input files have changed:\n', + self.name, str(module)) + + for input_file in input_files: + Log.verbose(' - %s', input_file) + + Log.verbose('') + + return input_files, changed_input_files + def __init__(self, name: str, dependencies: List[Target.Dependency], runnables: List[Runnable], build_unit: BuildUnit): """ @@ -348,14 +343,19 @@ def __init__(self, name: str, dependencies: List[Target.Dependency], runnables: super().__init__(name, dependencies) self.runnables = runnables self.build_unit = build_unit + self.change_detection = ChangeDetection(path.join('build_system', 'build', self.name + '.json')) def run(self, module_registry: ModuleRegistry): for runnable in self.runnables: modules = module_registry.lookup(runnable.module_filter) for module in modules: - if not runnable.output_files_exist(module) or runnable.input_files_have_changed(module): + output_files, missing_output_files = self.__get_missing_output_files(runnable, module) + input_files, changed_input_files = self.__get_changed_input_files(runnable, module) + + if (not output_files and not input_files) or missing_output_files or changed_input_files: runnable.run(self.build_unit, module) + self.change_detection.track_files(module, *input_files) def clean(self, module_registry: ModuleRegistry): for runnable in self.runnables: diff --git a/build_system/targets/packaging/targets.py b/build_system/targets/packaging/targets.py index 5375f56b6..5f4c758f7 100644 --- a/build_system/targets/packaging/targets.py +++ b/build_system/targets/packaging/targets.py @@ -39,7 +39,7 @@ def get_input_files(self, module: Module) -> List[str]: return file_search.list(module.root_directory) def get_output_files(self, module: Module) -> List[str]: - return module.wheel_directory + return [module.wheel_directory] def get_clean_files(self, module: Module) -> List[str]: clean_files = [] From cfaab758fa1cc92586f96d6c8b88cd06bd467e96 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Wed, 11 Dec 2024 16:33:04 +0100 Subject: [PATCH 089/114] Add function "for_file" to class BuildUnit. --- build_system/core/build_unit.py | 29 +++++++++++++++---- build_system/core/targets.py | 4 +-- build_system/main.py | 4 +-- build_system/targets/code_style/__init__.py | 2 +- .../targets/code_style/cpp/__init__.py | 2 +- .../targets/code_style/markdown/__init__.py | 2 +- .../targets/code_style/python/__init__.py | 2 +- .../targets/code_style/yaml/__init__.py | 2 +- build_system/targets/compilation/__init__.py | 2 +- .../targets/compilation/cpp/__init__.py | 2 +- .../targets/compilation/cython/__init__.py | 2 +- .../targets/dependencies/github/__init__.py | 2 +- .../targets/dependencies/python/__init__.py | 2 +- build_system/targets/packaging/__init__.py | 2 +- build_system/targets/paths.py | 5 ++-- build_system/targets/testing/__init__.py | 2 +- build_system/targets/testing/cpp/__init__.py | 2 +- .../targets/testing/python/__init__.py | 2 +- build_system/targets/versioning/__init__.py | 2 +- build_system/util/pip.py | 2 +- build_system/util/run.py | 2 +- 21 files changed, 48 insertions(+), 28 deletions(-) diff --git a/build_system/core/build_unit.py b/build_system/core/build_unit.py index afd7dfa5a..926f31722 100644 --- a/build_system/core/build_unit.py +++ b/build_system/core/build_unit.py @@ -12,13 +12,32 @@ class BuildUnit: An independent unit of the build system that may come with its own built-time dependencies. """ - ROOT_DIRECTORY = 'build_system' + BUILD_SYSTEM_DIRECTORY = 'build_system' - def __init__(self, *subdirectories: str): + BUILD_DIRECTORY_NAME = 'build' + + def __init__(self, root_directory: str = BUILD_SYSTEM_DIRECTORY): + """ + :param root_directory: The root directory of this unit + """ + self.root_directory = root_directory + + @staticmethod + def for_file(file) -> 'BuildUnit': + """ + Creates and returns a `BuildUnit` for a given file. + + :param file: The file for which a `BuildUnit` should be created + :return: The `BuildUnit` that has been created + """ + return BuildUnit(path.relpath(path.dirname(file), path.dirname(BuildUnit.BUILD_SYSTEM_DIRECTORY))) + + @property + def build_directory(self) -> str: """ - :param subdirectories: The subdirectories within the build system that lead to the root directory of this unit + The path to the build directory of this unit. """ - self.root_directory = path.join(self.ROOT_DIRECTORY, *subdirectories) + return path.join(self.root_directory, self.BUILD_DIRECTORY_NAME) def find_requirements_files(self) -> List[str]: """ @@ -27,7 +46,7 @@ def find_requirements_files(self) -> List[str]: requirements_files = [] current_directory = self.root_directory - while path.basename(current_directory) != self.ROOT_DIRECTORY: + while path.basename(current_directory) != self.BUILD_SYSTEM_DIRECTORY: requirements_file = path.join(current_directory, 'requirements.txt') if path.isfile(requirements_file): diff --git a/build_system/core/targets.py b/build_system/core/targets.py index 2629ccbfb..920443354 100644 --- a/build_system/core/targets.py +++ b/build_system/core/targets.py @@ -343,7 +343,7 @@ def __init__(self, name: str, dependencies: List[Target.Dependency], runnables: super().__init__(name, dependencies) self.runnables = runnables self.build_unit = build_unit - self.change_detection = ChangeDetection(path.join('build_system', 'build', self.name + '.json')) + self.change_detection = ChangeDetection(path.join(BuildUnit().build_directory, self.name + '.json')) def run(self, module_registry: ModuleRegistry): for runnable in self.runnables: @@ -484,7 +484,7 @@ class TargetBuilder: A builder that allows to configure and create multiple targets. """ - def __init__(self, build_unit: BuildUnit = BuildUnit()): + def __init__(self, build_unit: BuildUnit): """ :param build_unit: The build unit, the targets belong to """ diff --git a/build_system/main.py b/build_system/main.py index 462c66401..993b07c9b 100644 --- a/build_system/main.py +++ b/build_system/main.py @@ -7,10 +7,10 @@ from argparse import ArgumentParser from importlib.util import module_from_spec, spec_from_file_location -from os import path from types import ModuleType from typing import List +from core.build_unit import BuildUnit from core.modules import Module, ModuleRegistry from core.targets import DependencyGraph, Target, TargetRegistry from util.files import FileSearch @@ -35,7 +35,7 @@ def __find_init_files() -> List[str]: return FileSearch() \ .set_recursive(True) \ .filter_by_name('__init__.py') \ - .list(path.dirname(__file__)) + .list(BuildUnit().root_directory) def __import_source_file(source_file: str) -> ModuleType: diff --git a/build_system/targets/code_style/__init__.py b/build_system/targets/code_style/__init__.py index 3222f189c..2939cd87c 100644 --- a/build_system/targets/code_style/__init__.py +++ b/build_system/targets/code_style/__init__.py @@ -11,7 +11,7 @@ from targets.code_style.python import FORMAT_PYTHON, TEST_FORMAT_PYTHON from targets.code_style.yaml import FORMAT_YAML, TEST_FORMAT_YAML -TARGETS = TargetBuilder(BuildUnit('targets', 'code_style')) \ +TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ .add_phony_target('format') \ .depends_on(FORMAT_PYTHON, FORMAT_CPP, FORMAT_MARKDOWN, FORMAT_YAML) \ .nop() \ diff --git a/build_system/targets/code_style/cpp/__init__.py b/build_system/targets/code_style/cpp/__init__.py index ac877fcea..e75a92b16 100644 --- a/build_system/targets/code_style/cpp/__init__.py +++ b/build_system/targets/code_style/cpp/__init__.py @@ -15,7 +15,7 @@ TEST_FORMAT_CPP = 'test_format_cpp' -TARGETS = TargetBuilder(BuildUnit('targets', 'code_style', 'cpp')) \ +TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ .add_phony_target(FORMAT_CPP).set_runnables(EnforceCppCodeStyle()) \ .add_phony_target(TEST_FORMAT_CPP).set_runnables(CheckCppCodeStyle()) \ .build() diff --git a/build_system/targets/code_style/markdown/__init__.py b/build_system/targets/code_style/markdown/__init__.py index ff50f2395..b9717b265 100644 --- a/build_system/targets/code_style/markdown/__init__.py +++ b/build_system/targets/code_style/markdown/__init__.py @@ -15,7 +15,7 @@ TEST_FORMAT_MARKDOWN = 'test_format_md' -TARGETS = TargetBuilder(BuildUnit('targets', 'code_style', 'markdown')) \ +TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ .add_phony_target(FORMAT_MARKDOWN).set_runnables(EnforceMarkdownCodeStyle()) \ .add_phony_target(TEST_FORMAT_MARKDOWN).set_runnables(CheckMarkdownCodeStyle()) \ .build() diff --git a/build_system/targets/code_style/python/__init__.py b/build_system/targets/code_style/python/__init__.py index 85ca4ef55..e30a72280 100644 --- a/build_system/targets/code_style/python/__init__.py +++ b/build_system/targets/code_style/python/__init__.py @@ -16,7 +16,7 @@ TEST_FORMAT_PYTHON = 'test_format_python' -TARGETS = TargetBuilder(BuildUnit('targets', 'code_style', 'python')) \ +TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ .add_phony_target(FORMAT_PYTHON).set_runnables(EnforcePythonCodeStyle(), EnforceCythonCodeStyle()) \ .add_phony_target(TEST_FORMAT_PYTHON).set_runnables(CheckPythonCodeStyle(), CheckCythonCodeStyle()) \ .build() diff --git a/build_system/targets/code_style/yaml/__init__.py b/build_system/targets/code_style/yaml/__init__.py index 97a468e64..c24cfc3dc 100644 --- a/build_system/targets/code_style/yaml/__init__.py +++ b/build_system/targets/code_style/yaml/__init__.py @@ -15,7 +15,7 @@ TEST_FORMAT_YAML = 'test_format_yaml' -TARGETS = TargetBuilder(BuildUnit('targets', 'code_style', 'yaml')) \ +TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ .add_phony_target(FORMAT_YAML).set_runnables(EnforceYamlCodeStyle()) \ .add_phony_target(TEST_FORMAT_YAML).set_runnables(CheckYamlCodeStyle()) \ .build() diff --git a/build_system/targets/compilation/__init__.py b/build_system/targets/compilation/__init__.py index c3d6db12f..e74159069 100644 --- a/build_system/targets/compilation/__init__.py +++ b/build_system/targets/compilation/__init__.py @@ -11,7 +11,7 @@ INSTALL = 'install' -TARGETS = TargetBuilder(BuildUnit('targets', 'compilation')) \ +TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ .add_phony_target('compile') \ .depends_on(COMPILE_CPP, COMPILE_CYTHON, clean_dependencies=True) \ .nop() \ diff --git a/build_system/targets/compilation/cpp/__init__.py b/build_system/targets/compilation/cpp/__init__.py index a3cc49a2d..c7d5f9a0a 100644 --- a/build_system/targets/compilation/cpp/__init__.py +++ b/build_system/targets/compilation/cpp/__init__.py @@ -18,7 +18,7 @@ INSTALL_CPP = 'install_cpp' -TARGETS = TargetBuilder(BuildUnit('targets', 'compilation', 'cpp')) \ +TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ .add_build_target(SETUP_CPP) \ .depends_on(VENV) \ .set_runnables(SetupCpp()) \ diff --git a/build_system/targets/compilation/cython/__init__.py b/build_system/targets/compilation/cython/__init__.py index 72c43e858..be7145746 100644 --- a/build_system/targets/compilation/cython/__init__.py +++ b/build_system/targets/compilation/cython/__init__.py @@ -18,7 +18,7 @@ INSTALL_CYTHON = 'install_cython' -TARGETS = TargetBuilder(BuildUnit('targets', 'compilation', 'cython')) \ +TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ .add_build_target(SETUP_CYTHON) \ .depends_on(COMPILE_CPP) \ .set_runnables(SetupCython()) \ diff --git a/build_system/targets/dependencies/github/__init__.py b/build_system/targets/dependencies/github/__init__.py index 029b95544..32d11d829 100644 --- a/build_system/targets/dependencies/github/__init__.py +++ b/build_system/targets/dependencies/github/__init__.py @@ -10,7 +10,7 @@ from targets.dependencies.github.targets import CheckGithubActions, UpdateGithubActions from targets.paths import Project -TARGETS = TargetBuilder(BuildUnit('targets', 'dependencies', 'github')) \ +TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ .add_phony_target('check_github_actions').set_runnables(CheckGithubActions()) \ .add_phony_target('update_github_actions').set_runnables(UpdateGithubActions()) \ .build() diff --git a/build_system/targets/dependencies/python/__init__.py b/build_system/targets/dependencies/python/__init__.py index f3be20ae6..bc8ce0bc3 100644 --- a/build_system/targets/dependencies/python/__init__.py +++ b/build_system/targets/dependencies/python/__init__.py @@ -12,7 +12,7 @@ VENV = 'venv' -TARGETS = TargetBuilder(BuildUnit('targets', 'dependencies', 'python')) \ +TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ .add_phony_target(VENV).set_runnables(InstallRuntimeDependencies()) \ .add_phony_target('check_dependencies').set_runnables(CheckPythonDependencies()) \ .build() diff --git a/build_system/targets/packaging/__init__.py b/build_system/targets/packaging/__init__.py index 727dec43a..c7a16eed3 100644 --- a/build_system/targets/packaging/__init__.py +++ b/build_system/targets/packaging/__init__.py @@ -17,7 +17,7 @@ INSTALL_WHEELS = 'install_wheels' -TARGETS = TargetBuilder(BuildUnit('targets', 'packaging')) \ +TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ .add_build_target(BUILD_WHEELS) \ .depends_on(INSTALL) \ .set_runnables(BuildPythonWheels()) \ diff --git a/build_system/targets/paths.py b/build_system/targets/paths.py index 7f3c75a29..84203fe11 100644 --- a/build_system/targets/paths.py +++ b/build_system/targets/paths.py @@ -3,6 +3,7 @@ Provides paths within the project that are important for the build system. """ +from core.build_unit import BuildUnit from util.files import FileSearch @@ -25,9 +26,9 @@ class BuildSystem: build_directory_name: The name of the build system's build directory """ - root_directory = 'build_system' + root_directory = BuildUnit.BUILD_SYSTEM_DIRECTORY - build_directory_name = 'build' + build_directory_name = BuildUnit.BUILD_DIRECTORY_NAME @staticmethod def file_search() -> FileSearch: diff --git a/build_system/targets/testing/__init__.py b/build_system/targets/testing/__init__.py index fd0c49806..e1a567b3e 100644 --- a/build_system/targets/testing/__init__.py +++ b/build_system/targets/testing/__init__.py @@ -9,7 +9,7 @@ from targets.testing.cpp import TESTS_CPP from targets.testing.python import TESTS_PYTHON -TARGETS = TargetBuilder(BuildUnit('targets', 'testing')) \ +TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ .add_phony_target('tests') \ .depends_on(TESTS_CPP, TESTS_PYTHON) \ .nop() \ diff --git a/build_system/targets/testing/cpp/__init__.py b/build_system/targets/testing/cpp/__init__.py index d14319cfc..f06278511 100644 --- a/build_system/targets/testing/cpp/__init__.py +++ b/build_system/targets/testing/cpp/__init__.py @@ -13,7 +13,7 @@ TESTS_CPP = 'tests_cpp' -TARGETS = TargetBuilder(BuildUnit('targets', 'testing', 'cpp')) \ +TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ .add_phony_target(TESTS_CPP) \ .depends_on(COMPILE_CPP) \ .set_runnables(TestCpp()) \ diff --git a/build_system/targets/testing/python/__init__.py b/build_system/targets/testing/python/__init__.py index f49e01ef0..1af719d68 100644 --- a/build_system/targets/testing/python/__init__.py +++ b/build_system/targets/testing/python/__init__.py @@ -13,7 +13,7 @@ TESTS_PYTHON = 'tests_python' -TARGETS = TargetBuilder(BuildUnit('targets', 'testing', 'python')) \ +TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ .add_phony_target(TESTS_PYTHON) \ .depends_on(INSTALL_WHEELS) \ .set_runnables(TestPython()) \ diff --git a/build_system/targets/versioning/__init__.py b/build_system/targets/versioning/__init__.py index 4051db366..1e3122f77 100644 --- a/build_system/targets/versioning/__init__.py +++ b/build_system/targets/versioning/__init__.py @@ -10,7 +10,7 @@ increment_major_version, increment_minor_version, increment_patch_version, print_current_version, \ reset_development_version -TARGETS = TargetBuilder(BuildUnit('util')) \ +TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ .add_phony_target('increment_development_version').set_functions(increment_development_version) \ .add_phony_target('reset_development_version').set_functions(reset_development_version) \ .add_phony_target('apply_development_version').set_functions(apply_development_version) \ diff --git a/build_system/util/pip.py b/build_system/util/pip.py index b33741348..72063c889 100644 --- a/build_system/util/pip.py +++ b/build_system/util/pip.py @@ -230,7 +230,7 @@ def __init__(self, *requirements_files: str): self.requirements = RequirementsFiles(*requirements_files) @staticmethod - def for_build_unit(build_unit: BuildUnit = BuildUnit('util')): + def for_build_unit(build_unit: BuildUnit = BuildUnit.for_file(__file__)): """ Creates and returns a new `Pip` instance for installing packages for a specific build unit. diff --git a/build_system/util/run.py b/build_system/util/run.py index 19ca0f45c..1dc5db405 100644 --- a/build_system/util/run.py +++ b/build_system/util/run.py @@ -20,7 +20,7 @@ class RunOptions(Command.RunOptions): Allows to customize options for running an external program. """ - def __init__(self, build_unit: BuildUnit = BuildUnit('util')): + def __init__(self, build_unit: BuildUnit = BuildUnit.for_file(__file__)): """ :param build_unit: The build unit from which the program should be run """ From e43f7969f8478f8d850d2a79d36e97e67ba08956 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 12 Dec 2024 00:53:52 +0100 Subject: [PATCH 090/114] Remove obsolete functions from class Target.Builder. --- build_system/core/targets.py | 118 +++++------------------------------ 1 file changed, 15 insertions(+), 103 deletions(-) diff --git a/build_system/core/targets.py b/build_system/core/targets.py index 920443354..a169ee869 100644 --- a/build_system/core/targets.py +++ b/build_system/core/targets.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from functools import reduce from os import path -from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple from core.build_unit import BuildUnit from core.changes import ChangeDetection @@ -46,17 +46,7 @@ class Builder(ABC): An abstract base class for all builders that allow to configure and create targets. """ - def __anonymous_target_name(self) -> str: - return 'anonymous-dependency-' + str(len(self.dependencies) + 1) + '-of-' + self.target_name - - def __init__(self, parent_builder: Any, target_name: str): - """ - :param parent_builder: The builder, this builder has been created from - :param target_name: The name of the target that is configured via the builder - """ - self.parent_builder = parent_builder - self.target_name = target_name - self.child_builders = [] + def __init__(self): self.dependency_names = set() self.dependencies = [] @@ -77,97 +67,15 @@ def depends_on(self, *target_names: str, clean_dependencies: bool = False) -> 'T return self - def depends_on_build_target(self, clean_dependency: bool = True) -> 'BuildTarget.Builder': - """ - Creates and returns a `BuildTarget.Builder` that allows to configure a build target, this target should - depend on. - - :param clean_dependency: True, if output files of the dependency should also be cleaned when cleaning the - output files of this target, False otherwise - :return: The `BuildTarget.Builder` that has been created - """ - target_name = self.__anonymous_target_name() - target_builder = BuildTarget.Builder(self, target_name) - self.child_builders.append(target_builder) - self.depends_on(target_name, clean_dependencies=clean_dependency) - return target_builder - - def depends_on_build_targets(self, - iterable: Iterable[Any], - target_configurator: Callable[[Any, 'BuildTarget.Builder'], None], - clean_dependencies: bool = True) -> 'Target.Builder': - """ - Configures multiple build targets, one for each value in a given `Iterable`, this target should depend on. - - :param iterable: An `Iterable` that provides access to the values for which dependencies should - be created - :param target_configurator: A function that accepts one value in the `Iterable` at a time, as well as a - `BuildTarget.Builder` for configuring the corresponding dependency - :param clean_dependencies: True, if output files of the dependencies should also be cleaned when cleaning - the output files of this target, False otherwise - :return: The `Target.Builder` itself - """ - for value in iterable: - target_builder = self.depends_on_build_target(clean_dependency=clean_dependencies) - target_configurator(value, target_builder) - - return self - - def depends_on_phony_target(self, clean_dependency: bool = True) -> 'PhonyTarget.Builder': - """ - Creates and returns a `PhonyTarget.Builder` that allows to configure a phony target, this target should - depend on. - - :param clean_dependency: True, if output files of the dependency should also be cleaned when cleaning the - output files of this target, False otherwise - :return: The `PhonyTarget.Builder` that has been created - """ - target_name = self.__anonymous_target_name() - target_builder = PhonyTarget.Builder(self, target_name) - self.child_builders.append(target_builder) - self.depends_on(target_name, clean_dependencies=clean_dependency) - return target_builder - - def depends_on_phony_targets(self, - iterable: Iterable[Any], - target_configurator: Callable[[Any, 'PhonyTarget.Builder'], None], - clean_dependencies: bool = True) -> 'Target.Builder': - """ - Configures multiple phony targets, one for each value in a given `Iterable`, this target should depend on. - - :param iterable: An `Iterable` that provides access to the values for which dependencies should - be created - :param target_configurator: A function that accepts one value in the `Iterable` at a time, as well as a - `BuildTarget.Builder` for configuring the corresponding dependency - :param clean_dependencies: True, if output files of the dependencies should also be cleaned when cleaning - the output files of this target, False otherwise - :return: The `Target.Builder` itself - """ - for value in iterable: - target_builder = self.depends_on_phony_target(clean_dependency=clean_dependencies) - target_configurator(value, target_builder) - - return self - @abstractmethod - def _build(self, build_unit: BuildUnit) -> 'Target': + def build(self, build_unit: BuildUnit) -> 'Target': """ - Must be implemented by subclasses in order to create the target that has been configured via the builder. + Creates and returns the target that has been configured via the builder. :param build_unit: The build unit, the target belongs to :return: The target that has been created """ - def build(self, build_unit: BuildUnit) -> List['Target']: - """ - Creates and returns all targets that have been configured via the builder. - - :param build_unit: The build unit, the target belongs to - :return: The targets that have been created - """ - return [self._build(build_unit)] + reduce(lambda aggr, builder: aggr + builder.build(build_unit), - self.child_builders, []) - def __init__(self, name: str, dependencies: List['Target.Dependency']): """ :param name: The name of the target @@ -259,12 +167,14 @@ class Builder(Target.Builder): A builder that allows to configure and create build targets. """ - def __init__(self, parent_builder: Any, target_name: str): + def __init__(self, parent_builder: 'TargetBuilder', target_name: str): """ :param parent_builder: The builder, this builder has been created from :param target_name: The name of the target that is configured via the builder """ - super().__init__(parent_builder, target_name) + super().__init__() + self.parent_builder = parent_builder + self.target_name = target_name self.runnables = [] def set_runnables(self, *runnables: 'BuildTarget.Runnable') -> Any: @@ -277,7 +187,7 @@ def set_runnables(self, *runnables: 'BuildTarget.Runnable') -> Any: self.runnables = list(runnables) return self.parent_builder - def _build(self, build_unit: BuildUnit) -> Target: + def build(self, build_unit: BuildUnit) -> Target: return BuildTarget(self.target_name, self.dependencies, self.runnables, build_unit) def __get_missing_output_files(self, runnable: Runnable, module: Module) -> Tuple[List[str], List[str]]: @@ -408,12 +318,14 @@ class Builder(Target.Builder): A builder that allows to configure and create phony targets. """ - def __init__(self, parent_builder: Any, target_name: str): + def __init__(self, parent_builder: 'TargetBuilder', target_name: str): """ :param parent_builder: The builder, this builder has been created from :param target_name: The name of the target that is configured via the builder """ - super().__init__(parent_builder, target_name) + super().__init__() + self.parent_builder = parent_builder + self.target_name = target_name self.functions = [] self.runnables = [] @@ -445,7 +357,7 @@ def set_runnables(self, *runnables: 'PhonyTarget.Runnable') -> Any: self.runnables = list(runnables) return self.parent_builder - def _build(self, build_unit: BuildUnit) -> Target: + def build(self, build_unit: BuildUnit) -> Target: def action(module_registry: ModuleRegistry): for function in self.functions: @@ -519,7 +431,7 @@ def build(self) -> List[Target]: :return: A list that stores the targets that have been created """ - return reduce(lambda aggr, builder: aggr + builder.build(self.build_unit), self.target_builders, []) + return [builder.build(self.build_unit) for builder in self.target_builders] class DependencyGraph: From 7340add086eb19603db0d85f940c43770f4a12dd Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 12 Dec 2024 00:41:57 +0100 Subject: [PATCH 091/114] Add type hints to function arguments. --- build_system/util/env.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build_system/util/env.py b/build_system/util/env.py index 54ba70c11..614a8a21a 100644 --- a/build_system/util/env.py +++ b/build_system/util/env.py @@ -3,12 +3,12 @@ Provides utility functions for accessing environment variables. """ -from typing import List, Optional +from typing import Dict, List, Optional from util.log import Log -def get_env(env, name: str, default: Optional[str] = None) -> Optional[str]: +def get_env(env: Dict, name: str, default: Optional[str] = None) -> Optional[str]: """ Returns the value of the environment variable with a given name. @@ -20,7 +20,7 @@ def get_env(env, name: str, default: Optional[str] = None) -> Optional[str]: return env.get(name, default) -def get_env_bool(env, name: str, default: bool = False) -> bool: +def get_env_bool(env: Dict, name: str, default: bool = False) -> bool: """ Returns the value of the environment variable with a given name as a boolean value. @@ -33,7 +33,7 @@ def get_env_bool(env, name: str, default: bool = False) -> bool: return bool(value) if value else default -def get_env_array(env, name: str, default: Optional[List[str]] = None) -> List[str]: +def get_env_array(env: Dict, name: str, default: Optional[List[str]] = None) -> List[str]: """ Returns the value of the environment variable with a given name as a comma-separated list. @@ -50,7 +50,7 @@ def get_env_array(env, name: str, default: Optional[List[str]] = None) -> List[s return default if default else [] -def set_env(env, name: str, value: str): +def set_env(env: Dict, name: str, value: str): """ Sets the value of the environment variable with a given name. From 7b171178c86c9872975a6be4f703a0c57cecf46d Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 12 Dec 2024 00:42:10 +0100 Subject: [PATCH 092/114] Add utility function "create_directories". --- build_system/util/io.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/build_system/util/io.py b/build_system/util/io.py index 2d6d3f6c3..c2d928027 100644 --- a/build_system/util/io.py +++ b/build_system/util/io.py @@ -4,7 +4,7 @@ Provides utility functions for reading and writing files. """ from functools import cached_property -from os import path, remove +from os import makedirs, path, remove from shutil import rmtree from typing import List @@ -48,6 +48,18 @@ def delete_files(*files: str, accept_missing: bool = True): remove(file) +def create_directories(*directories: str): + """ + Creates one or several directories, if they do not already exist. + + :param directories: The directories to be created + """ + for directory in directories: + if not path.isdir(directory): + Log.verbose('Creating directory "%s"...', directory) + makedirs(directory) + + class TextFile: """ Allows to read and write the content of a text file. From 89db992f38caa737a98dde59c78c532ba4864b46 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 12 Dec 2024 00:52:45 +0100 Subject: [PATCH 093/114] Dynamically register targets and modules for generating C++ API documentations. --- .../targets/documentation/__init__.py | 0 .../targets/documentation/cpp}/Doxyfile | 0 .../targets/documentation/cpp/__init__.py | 29 +++++++++ .../documentation/cpp/breathe_apidoc.py | 30 +++++++++ .../targets/documentation/cpp/doxygen.py | 44 +++++++++++++ .../targets/documentation/cpp/modules.py | 63 +++++++++++++++++++ .../documentation/cpp/requirements.txt | 1 + .../targets/documentation/cpp/targets.py | 39 ++++++++++++ build_system/targets/paths.py | 4 ++ doc/requirements.txt | 1 - 10 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 build_system/targets/documentation/__init__.py rename {doc => build_system/targets/documentation/cpp}/Doxyfile (100%) create mode 100644 build_system/targets/documentation/cpp/__init__.py create mode 100644 build_system/targets/documentation/cpp/breathe_apidoc.py create mode 100644 build_system/targets/documentation/cpp/doxygen.py create mode 100644 build_system/targets/documentation/cpp/modules.py create mode 100644 build_system/targets/documentation/cpp/requirements.txt create mode 100644 build_system/targets/documentation/cpp/targets.py diff --git a/build_system/targets/documentation/__init__.py b/build_system/targets/documentation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/doc/Doxyfile b/build_system/targets/documentation/cpp/Doxyfile similarity index 100% rename from doc/Doxyfile rename to build_system/targets/documentation/cpp/Doxyfile diff --git a/build_system/targets/documentation/cpp/__init__.py b/build_system/targets/documentation/cpp/__init__.py new file mode 100644 index 000000000..fa59209a6 --- /dev/null +++ b/build_system/targets/documentation/cpp/__init__.py @@ -0,0 +1,29 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Defines targets and modules for generating API documentations for C++ code. +""" +from os import path + +from core.build_unit import BuildUnit +from core.targets import TargetBuilder + +from targets.documentation.cpp.modules import CppApidocModule +from targets.documentation.cpp.targets import ApidocCpp +from targets.paths import Project + +TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ + .add_build_target('apidoc_cpp') \ + .set_runnables(ApidocCpp()) \ + .build() + +MODULES = [ + CppApidocModule( + root_directory=path.dirname(meson_file), + output_directory=path.join(Project.Documentation.apidoc_directory, 'cpp', + path.basename(path.dirname(meson_file))), + project_name=path.basename(path.dirname(meson_file)), + include_directory_name='include', + ) for meson_file in Project.Cpp.file_search().filter_by_name('meson.build').list(Project.Cpp.root_directory) + if path.isdir(path.join(path.dirname(meson_file), 'include')) +] diff --git a/build_system/targets/documentation/cpp/breathe_apidoc.py b/build_system/targets/documentation/cpp/breathe_apidoc.py new file mode 100644 index 000000000..7e54cd5a5 --- /dev/null +++ b/build_system/targets/documentation/cpp/breathe_apidoc.py @@ -0,0 +1,30 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that allow to run the external program "breathe-apidoc". +""" +from os import path + +from core.build_unit import BuildUnit +from util.run import Program + +from targets.documentation.cpp.modules import CppApidocModule + + +class BreatheApidoc(Program): + """ + Allows to run the external program "breathe-apidoc". + """ + + def __init__(self, build_unit: BuildUnit, module: CppApidocModule): + """ + :param build_unit: The build unit from which the program should be run + :param module: The module, the program should be applied to + """ + super().__init__('breathe-apidoc', '--members', '--project', module.project_name, '-g', 'file', '-o', + module.output_directory, path.join(module.output_directory, 'xml')) + self.module = module + self.print_arguments(True) + self.install_program(False) + self.add_dependencies('breathe') + self.set_build_unit(build_unit) diff --git a/build_system/targets/documentation/cpp/doxygen.py b/build_system/targets/documentation/cpp/doxygen.py new file mode 100644 index 000000000..4561a274b --- /dev/null +++ b/build_system/targets/documentation/cpp/doxygen.py @@ -0,0 +1,44 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that allow to run the external program "doxygen". +""" +from os import environ, path +from typing import Dict + +from core.build_unit import BuildUnit +from util.env import set_env +from util.io import create_directories +from util.run import Program + +from targets.documentation.cpp.modules import CppApidocModule + + +class Doxygen(Program): + """ + Allows to run the external program "doxygen". + """ + + @staticmethod + def __create_environment(module: CppApidocModule) -> Dict: + env = environ.copy() + set_env(env, 'DOXYGEN_PROJECT_NAME', 'libmlrl' + module.project_name) + set_env(env, 'DOXYGEN_INPUT_DIR', module.include_directory) + set_env(env, 'DOXYGEN_OUTPUT_DIR', module.output_directory) + set_env(env, 'DOXYGEN_PREDEFINED', 'MLRL' + module.project_name.upper() + '_API=') + return env + + def __init__(self, build_unit: BuildUnit, module: CppApidocModule): + """ + :param build_unit: The build unit from which the program should be run + :param module: The module, the program should be applied to + """ + super().__init__('doxygen', path.join(build_unit.root_directory, 'Doxyfile')) + self.module = module + self.print_arguments(True) + self.install_program(False) + self.use_environment(self.__create_environment(module)) + self.set_build_unit(build_unit) + + def _before(self): + create_directories(self.module.output_directory) diff --git a/build_system/targets/documentation/cpp/modules.py b/build_system/targets/documentation/cpp/modules.py new file mode 100644 index 000000000..a6f6a2e0c --- /dev/null +++ b/build_system/targets/documentation/cpp/modules.py @@ -0,0 +1,63 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that provide access to C++ code for which an API documentation can be generated. +""" +from os import path +from typing import List + +from core.modules import Module +from util.files import FileSearch, FileType + + +class CppApidocModule(Module): + """ + A module that contains C++ code for which an API documentation can be generated. + """ + + class Filter(Module.Filter): + """ + A filter that matches code modules. + """ + + def matches(self, module: Module) -> bool: + return isinstance(module, CppApidocModule) + + def __init__(self, + root_directory: str, + output_directory: str, + project_name: str, + include_directory_name: str, + header_file_search: FileSearch = FileSearch().set_recursive(True)): + """ + :param root_directory: The path to the module's root directory + :param output_directory: The path to the directory where the API documentation should be stored + :param project_name: The name of the C++ project to be documented + :param include_directory_name: The name of the directory that contains the header files to be included in the + API documentation + :param header_file_search: The `FileSearch` that should be used to search for the header files to be + included in the API documentation + """ + self.root_directory = root_directory + self.output_directory = output_directory + self.project_name = project_name + self.include_directory_name = include_directory_name + self.header_file_search = header_file_search + + @property + def include_directory(self) -> str: + """ + The path to the directory that contains the header files to be included in the API documentation. + """ + return path.join(self.root_directory, self.include_directory_name) + + def find_header_files(self) -> List[str]: + """ + Finds and returns the header files to be included in the API documentation. + + :return: A list that contains the header files that have been found + """ + return self.header_file_search.filter_by_file_type(FileType.cpp()).list(self.include_directory) + + def __str__(self) -> str: + return 'CppApidocModule {root_directory="' + self.root_directory + '"}' diff --git a/build_system/targets/documentation/cpp/requirements.txt b/build_system/targets/documentation/cpp/requirements.txt new file mode 100644 index 000000000..7e3a3f719 --- /dev/null +++ b/build_system/targets/documentation/cpp/requirements.txt @@ -0,0 +1 @@ +breathe >= 4.35, < 4.36 diff --git a/build_system/targets/documentation/cpp/targets.py b/build_system/targets/documentation/cpp/targets.py new file mode 100644 index 000000000..4970623ad --- /dev/null +++ b/build_system/targets/documentation/cpp/targets.py @@ -0,0 +1,39 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Implements targets for generating API documentations for C++ code. +""" +from typing import List + +from core.build_unit import BuildUnit +from core.modules import Module +from core.targets import BuildTarget +from util.log import Log + +from targets.documentation.cpp.breathe_apidoc import BreatheApidoc +from targets.documentation.cpp.doxygen import Doxygen +from targets.documentation.cpp.modules import CppApidocModule + + +class ApidocCpp(BuildTarget.Runnable): + """ + Generates API documentations for C++ code. + """ + + def __init__(self): + super().__init__(CppApidocModule.Filter()) + + def run(self, build_unit: BuildUnit, module: Module): + Log.info('Generating C++ API documentation for directory "%s"...', module.root_directory) + Doxygen(build_unit, module).run() + BreatheApidoc(build_unit, module).run() + + def get_output_files(self, module: Module) -> List[str]: + return [module.output_directory] + + def get_input_files(self, module: Module) -> List[str]: + return module.find_header_files() + + def get_clean_files(self, module: Module) -> List[str]: + Log.info('Removing C++ API documentation for directory "%s"...', module.root_directory) + return super().get_clean_files(module) diff --git a/build_system/targets/paths.py b/build_system/targets/paths.py index 84203fe11..e32015cf6 100644 --- a/build_system/targets/paths.py +++ b/build_system/targets/paths.py @@ -3,6 +3,8 @@ Provides paths within the project that are important for the build system. """ +from os import path + from core.build_unit import BuildUnit from util.files import FileSearch @@ -111,6 +113,8 @@ class Documentation: root_directory = 'doc' + apidoc_directory = path.join(root_directory, 'developer_guide', 'api') + @staticmethod def file_search() -> FileSearch: """ diff --git a/doc/requirements.txt b/doc/requirements.txt index d7d7050e5..21d52227e 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,4 +1,3 @@ -breathe >= 4.35, < 4.36 furo == 2024.8.6 myst-parser >= 4.0, < 4.1 sphinx >= 7.4, < 7.5 From d8177be8a6be97ab198664be01cda72d104dd527 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 12 Dec 2024 01:38:24 +0100 Subject: [PATCH 094/114] Add constructor argument "suffixes" to class FileType. --- build_system/util/files.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/build_system/util/files.py b/build_system/util/files.py index cf8e334fe..95e0cda71 100644 --- a/build_system/util/files.py +++ b/build_system/util/files.py @@ -384,13 +384,20 @@ class FileType: Represents different types of files. """ - def __init__(self, name: str, file_search_decorator: Callable[[FileSearch], None]): + def __init__(self, + name: str, + suffixes: Set[str], + file_search_decorator: Optional[Callable[[FileSearch], None]] = None): """ :param name: The name of the file type - :param file_search_decorator: A function that adds a filter for this file type to a `FileSearch` + :param suffixes: The suffixes that correspond to this file type (without leading dot) + :param file_search_decorator: A function that adds a filter for this file type to a `FileSearch` or None, if a + filter should automatically be created """ self.name = name - self.file_search_decorator = file_search_decorator + self.suffixes = suffixes + self.file_search_decorator = file_search_decorator if file_search_decorator else lambda file_search: file_search.filter_by_suffix( + *suffixes) @staticmethod def python() -> 'FileType': @@ -399,7 +406,7 @@ def python() -> 'FileType': :return: The `FileType` that has been created """ - return FileType('Python', lambda file_search: file_search.filter_by_suffix('py')) + return FileType(name='Python', suffixes={'py'}) @staticmethod def cpp() -> 'FileType': @@ -408,7 +415,7 @@ def cpp() -> 'FileType': :return: The `FileType` that has been created """ - return FileType('C++', lambda file_search: file_search.filter_by_suffix('cpp', 'hpp')) + return FileType(name='C++', suffixes={'cpp', 'hpp'}) @staticmethod def cython() -> 'FileType': @@ -417,7 +424,7 @@ def cython() -> 'FileType': :return: The `FileType` that has been created """ - return FileType('Cython', lambda file_search: file_search.filter_by_suffix('pyx', 'pxd')) + return FileType(name='Cython', suffixes={'pyx', 'pxd'}) @staticmethod def markdown() -> 'FileType': @@ -426,7 +433,7 @@ def markdown() -> 'FileType': :return: The `FileType` that has been created """ - return FileType('Markdown', lambda file_search: file_search.filter_by_suffix('md')) + return FileType(name='Markdown', suffixes={'md'}) @staticmethod def yaml() -> 'FileType': @@ -435,7 +442,7 @@ def yaml() -> 'FileType': :return: The `FileType` that has been created """ - return FileType('YAML', lambda file_search: file_search.filter_by_suffix('yaml', 'yml')) + return FileType(name='YAML', suffixes={'yaml', 'yml'}) @staticmethod def extension_module() -> 'FileType': @@ -445,8 +452,9 @@ def extension_module() -> 'FileType': :return: The `FileType` that has been created """ return FileType( - 'Extension module', - lambda file_search: file_search \ + name='Extension module', + suffixes={'so', 'pyd', 'lib'}, + file_search_decorator=lambda file_search: file_search \ .filter_by_substrings(not_starts_with='lib', ends_with='.so') \ .filter_by_substrings(ends_with='.pyd') \ .filter_by_substrings(not_starts_with='mlrl', ends_with='.lib'), @@ -460,8 +468,9 @@ def shared_library() -> 'FileType': :return: The `FileType` that has been created """ return FileType( - 'Shared library', - lambda file_search: file_search \ + name='Shared library', + suffixes={'so', 'dylib', 'lib', 'dll'}, + file_search_decorator=lambda file_search: file_search \ .filter_by_substrings(starts_with='lib', contains='.so') \ .filter_by_substrings(ends_with='.dylib') \ .filter_by_substrings(starts_with='mlrl', ends_with='.lib') \ From 809317b54596136d009be6f85574721e1220956b Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 12 Dec 2024 01:53:12 +0100 Subject: [PATCH 095/114] Dynamically register targets and modules for generating Python API documentations. --- .../targets/documentation/python/__init__.py | 29 +++++++++ .../targets/documentation/python/modules.py | 60 +++++++++++++++++++ .../documentation/python/requirements.txt | 1 + .../documentation/python/sphinx_apidoc.py | 41 +++++++++++++ .../targets/documentation/python/targets.py | 37 ++++++++++++ 5 files changed, 168 insertions(+) create mode 100644 build_system/targets/documentation/python/__init__.py create mode 100644 build_system/targets/documentation/python/modules.py create mode 100644 build_system/targets/documentation/python/requirements.txt create mode 100644 build_system/targets/documentation/python/sphinx_apidoc.py create mode 100644 build_system/targets/documentation/python/targets.py diff --git a/build_system/targets/documentation/python/__init__.py b/build_system/targets/documentation/python/__init__.py new file mode 100644 index 000000000..bd435a418 --- /dev/null +++ b/build_system/targets/documentation/python/__init__.py @@ -0,0 +1,29 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Defines targets and modules for generating API documentations for C++ code. +""" +from os import path + +from core.build_unit import BuildUnit +from core.targets import TargetBuilder + +from targets.documentation.python.modules import PythonApidocModule +from targets.documentation.python.targets import ApidocPython +from targets.packaging import INSTALL_WHEELS +from targets.paths import Project + +TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ + .add_build_target('apidoc_python') \ + .depends_on(INSTALL_WHEELS) \ + .set_runnables(ApidocPython()) \ + .build() + +MODULES = [ + PythonApidocModule(root_directory=path.dirname(setup_file), + output_directory=path.join(Project.Documentation.apidoc_directory, 'python', + path.basename(path.dirname(setup_file))), + source_directory_name='mlrl', + source_file_search=Project.Python.file_search()) + for setup_file in Project.Python.file_search().filter_by_name('setup.py').list(Project.Python.root_directory) +] diff --git a/build_system/targets/documentation/python/modules.py b/build_system/targets/documentation/python/modules.py new file mode 100644 index 000000000..5648ad418 --- /dev/null +++ b/build_system/targets/documentation/python/modules.py @@ -0,0 +1,60 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that provide access to Python code for which an API documentation can be generated. +""" +from os import path +from typing import List + +from core.modules import Module +from util.files import FileSearch, FileType + + +class PythonApidocModule(Module): + """ + A module that contains Python code for which an API documentation can be generated. + """ + + class Filter(Module.Filter): + """ + A filter that matches code modules. + """ + + def matches(self, module: Module) -> bool: + return isinstance(module, PythonApidocModule) + + def __init__(self, + root_directory: str, + output_directory: str, + source_directory_name: str, + source_file_search: FileSearch = FileSearch().set_recursive(True)): + """ + :param root_directory: The path to the module's root directory + :param output_directory: The path to the directory where the API documentation should be stored + :param source_directory_name: The name of the directory that contains the Python source files to be included + in the API documentation + :param source_file_search: The `FileSearch` that should be used to search for the header files to be included + in the API documentation + """ + self.root_directory = root_directory + self.output_directory = output_directory + self.source_directory_name = source_directory_name + self.source_file_search = source_file_search + + @property + def source_directory(self) -> str: + """ + The path to the directory that contains the Python source files to be included in the API documentation. + """ + return path.join(self.root_directory, self.source_directory_name) + + def find_source_files(self) -> List[str]: + """ + Finds and returns the Python source files to be included in the API documentation. + + :return: A list that contains the source files that have been found + """ + return self.source_file_search.filter_by_file_type(FileType.python()).list(self.source_directory) + + def __str__(self) -> str: + return 'PythonApidocModule {root_directory="' + self.root_directory + '"}' diff --git a/build_system/targets/documentation/python/requirements.txt b/build_system/targets/documentation/python/requirements.txt new file mode 100644 index 000000000..d35e56e84 --- /dev/null +++ b/build_system/targets/documentation/python/requirements.txt @@ -0,0 +1 @@ +sphinx >= 7.4, < 7.5 diff --git a/build_system/targets/documentation/python/sphinx_apidoc.py b/build_system/targets/documentation/python/sphinx_apidoc.py new file mode 100644 index 000000000..2827b09f9 --- /dev/null +++ b/build_system/targets/documentation/python/sphinx_apidoc.py @@ -0,0 +1,41 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that allow to run the external program "sphinx-apidoc". +""" +from os import path + +from core.build_unit import BuildUnit +from util.files import FileType +from util.io import create_directories, delete_files +from util.run import Program + +from targets.documentation.python.modules import PythonApidocModule + + +class SphinxApidoc(Program): + """ + Allows to run the external program "sphinx-apidoc". + """ + + def __init__(self, build_unit: BuildUnit, module: PythonApidocModule): + """ + :param build_unit: The build unit from which the program should be run + :param module: The module, the program should be applied to + """ + super().__init__('sphinx-apidoc', '--separate', '--module-first', '--no-toc', '-o', module.output_directory, + module.source_directory, + *['*.' + suffix + '*' for suffix in FileType.extension_module().suffixes], + *['*.' + suffix + '*' for suffix in FileType.shared_library().suffixes]) + self.module = module + self.print_arguments(True) + self.install_program(False) + self.add_dependencies('sphinx') + self.set_build_unit(build_unit) + + def _before(self): + create_directories(self.module.output_directory) + + def _after(self): + root_rst_file = path.join(self.module.output_directory, path.basename(self.module.source_directory) + '.rst') + delete_files(root_rst_file, accept_missing=False) diff --git a/build_system/targets/documentation/python/targets.py b/build_system/targets/documentation/python/targets.py new file mode 100644 index 000000000..5db3c086a --- /dev/null +++ b/build_system/targets/documentation/python/targets.py @@ -0,0 +1,37 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Implements targets for generating API documentations for Python code. +""" +from typing import List + +from core.build_unit import BuildUnit +from core.modules import Module +from core.targets import BuildTarget +from util.log import Log + +from targets.documentation.python.modules import PythonApidocModule +from targets.documentation.python.sphinx_apidoc import SphinxApidoc + + +class ApidocPython(BuildTarget.Runnable): + """ + Generates API documentations for Python code. + """ + + def __init__(self): + super().__init__(PythonApidocModule.Filter()) + + def run(self, build_unit: BuildUnit, module: Module): + Log.info('Generating Python API documentation for directory "%s"...', module.root_directory) + SphinxApidoc(build_unit, module).run() + + def get_output_files(self, module: Module) -> List[str]: + return [module.output_directory] + + def get_input_files(self, module: Module) -> List[str]: + return module.find_source_files() + + def get_clean_files(self, module: Module) -> List[str]: + Log.info('Removing Python API documentation for directory "%s"...', module.root_directory) + return super().get_clean_files(module) From e1c52ff63f11e964368fb5995b6f6445ce44c6b6 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 12 Dec 2024 03:07:22 +0100 Subject: [PATCH 096/114] Allow to exclude files by name when using the class FileSearch. --- build_system/util/files.py | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/build_system/util/files.py b/build_system/util/files.py index 95e0cda71..d4802492f 100644 --- a/build_system/util/files.py +++ b/build_system/util/files.py @@ -164,6 +164,7 @@ class FileSearch: def __init__(self): self.hidden = False self.symlinks = True + self.excludes = [] self.filters = [] self.directory_search = DirectorySearch() @@ -345,6 +346,29 @@ def filter_by_file_type(self, *file_types: 'FileType') -> 'FileSearch': return self + def exclude(self, *excludes: Filter) -> 'FileSearch': + """ + Adds one or several filters that should be used for excluding files. + + :param excludes: The filters to be set + :return: The `FileSearch` itself + """ + self.excludes.extend(excludes) + return self + + def exclude_by_name(self, *names: str) -> 'FileSearch': + """ + Adds one or several filters that should be used for excluding files by their names. + + :param names: The names of the files to be excluded + :return: The `FileSearch` itself + """ + + def filter_file(excluded_name: str, _: str, file_name: str): + return file_name == excluded_name + + return self.exclude(*[partial(filter_file, name) for name in names]) + def list(self, *directories: str) -> List[str]: """ Lists all files that can be found in given directories. @@ -357,14 +381,17 @@ def list(self, *directories: str) -> List[str]: def filter_file(file: str) -> bool: if path.isfile(file) and (self.symlinks or not path.islink(file)): - if not self.filters: - return True - parent = path.dirname(file) file_name = path.basename(file) - if reduce(lambda aggr, file_filter: aggr or file_filter(parent, file_name), self.filters, False): - return True + if not self.filters: + match = True + else: + match = reduce(lambda aggr, file_filter: aggr or file_filter(parent, file_name), self.filters, + False) + + exclude = reduce(lambda aggr, file_filter: aggr or file_filter(parent, file_name), self.excludes, False) + return match and not exclude return False From 6e80455f03ebc6c4163287c0721f843726d38f35 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 12 Dec 2024 03:33:54 +0100 Subject: [PATCH 097/114] Catch exceptions. --- build_system/core/changes.py | 6 +++++- build_system/targets/dependencies/github/actions.py | 12 ++++++++++-- build_system/targets/dependencies/github/pyyaml.py | 6 +++++- build_system/targets/versioning/changelog.py | 12 ++++++++++-- build_system/targets/versioning/versioning.py | 12 ++++++++++-- build_system/util/io.py | 5 ++++- 6 files changed, 44 insertions(+), 9 deletions(-) diff --git a/build_system/core/changes.py b/build_system/core/changes.py index 13aba4d14..8d9884df6 100644 --- a/build_system/core/changes.py +++ b/build_system/core/changes.py @@ -47,7 +47,11 @@ def write_json(self, dictionary: Dict): def write_lines(self, *lines: str): super().write_lines(*lines) - del self.json + + try: + del self.json + except AttributeError: + pass class ChangeDetection: diff --git a/build_system/targets/dependencies/github/actions.py b/build_system/targets/dependencies/github/actions.py index 9b5e4c17b..61ca1c1ab 100644 --- a/build_system/targets/dependencies/github/actions.py +++ b/build_system/targets/dependencies/github/actions.py @@ -179,8 +179,16 @@ def update_actions(self, *updated_actions: Action): def write_lines(self, *lines: str): super().write_lines(lines) - del self.uses_clauses - del self.actions + + try: + del self.uses_clauses + except AttributeError: + pass + + try: + del self.actions + except AttributeError: + pass def __eq__(self, other: 'Workflow') -> bool: return self.file == other.file diff --git a/build_system/targets/dependencies/github/pyyaml.py b/build_system/targets/dependencies/github/pyyaml.py index cfdf262d4..0acd4e194 100644 --- a/build_system/targets/dependencies/github/pyyaml.py +++ b/build_system/targets/dependencies/github/pyyaml.py @@ -37,4 +37,8 @@ def yaml_dict(self) -> Dict: def write_lines(self, *lines: str): super().write_lines(lines) - del self.yaml_dict + + try: + del self.yaml_dict + except AttributeError: + pass diff --git a/build_system/targets/versioning/changelog.py b/build_system/targets/versioning/changelog.py index ae43201dd..f31f3a40b 100644 --- a/build_system/targets/versioning/changelog.py +++ b/build_system/targets/versioning/changelog.py @@ -179,8 +179,16 @@ def validate(self): def write_lines(self, *lines: str): super().write_lines(lines) - del self.parsed_lines - del self.changesets + + try: + del self.parsed_lines + except AttributeError: + pass + + try: + del self.changesets + except AttributeError: + pass class ReleaseType(Enum): diff --git a/build_system/targets/versioning/versioning.py b/build_system/targets/versioning/versioning.py index 4561996be..ab3e7f967 100644 --- a/build_system/targets/versioning/versioning.py +++ b/build_system/targets/versioning/versioning.py @@ -95,7 +95,11 @@ def update(self, version: Version): def write_lines(self, *lines: str): super().write_lines(lines) - del self.version + + try: + del self.version + except AttributeError: + pass class DevelopmentVersionFile(TextFile): @@ -118,7 +122,11 @@ def update(self, development_version: int): def write_lines(self, *lines: str): super().write_lines(lines) - del self.development_version + + try: + del self.development_version + except AttributeError: + pass def __get_version_file() -> VersionFile: diff --git a/build_system/util/io.py b/build_system/util/io.py index c2d928027..78c9b2cd7 100644 --- a/build_system/util/io.py +++ b/build_system/util/io.py @@ -93,7 +93,10 @@ def write_lines(self, *lines: str): with write_file(self.file) as file: file.writelines(lines) - del self.lines + try: + del self.lines + except AttributeError: + pass def clear(self): """ From 4495ad6b2fce8282cc93f8231514ad6b6841d422 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 12 Dec 2024 03:39:53 +0100 Subject: [PATCH 098/114] Fix filtering files by name when using a FileSearch. --- build_system/util/files.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_system/util/files.py b/build_system/util/files.py index d4802492f..276a9f297 100644 --- a/build_system/util/files.py +++ b/build_system/util/files.py @@ -285,8 +285,8 @@ def filter_by_name(self, *names: str) -> 'FileSearch': :return: The `FileSearch` itself """ - def filter_file(filtered_names: Set[str], _: str, file_name: str): - return file_name in filtered_names + def filter_file(filtered_name: str, _: str, file_name: str): + return file_name == filtered_name return self.add_filters(*[partial(filter_file, name) for name in names]) From 9cb6fdd8ba43c751daeb6e3e583fb15807ed61c3 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 12 Dec 2024 23:41:02 +0100 Subject: [PATCH 099/114] Add function "run_all" to class BuildTarget.Runnable. --- build_system/core/targets.py | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/build_system/core/targets.py b/build_system/core/targets.py index a169ee869..8e412da58 100644 --- a/build_system/core/targets.py +++ b/build_system/core/targets.py @@ -124,15 +124,24 @@ def __init__(self, module_filter: Module.Filter): """ self.module_filter = module_filter - @abstractmethod + def run_all(self, build_unit: BuildUnit, modules: List[Module]): + """ + May be overridden by subclasses in order to apply the target to all modules that match the filter. + + :param build_unit: The build unit, the target belongs to + :param modules: A list that contains the modules, the target should be applied to + """ + raise NotImplementedError('Class ' + type(self).__name__ + ' does not implement the "run_all" method') + def run(self, build_unit: BuildUnit, module: Module): """ - may be overridden by subclasses in order to apply the target to an individual module that matches the + May be overridden by subclasses in order to apply the target to an individual module that matches the filter. :param build_unit: The build unit, the target belongs to :param module: The module, the target should be applied to """ + raise NotImplementedError('Class ' + type(self).__name__ + ' does not implement the "run" method') def get_input_files(self, module: Module) -> List[str]: """ @@ -258,14 +267,29 @@ def __init__(self, name: str, dependencies: List[Target.Dependency], runnables: def run(self, module_registry: ModuleRegistry): for runnable in self.runnables: modules = module_registry.lookup(runnable.module_filter) + modules_to_be_run = [] + input_files_per_module = [] for module in modules: output_files, missing_output_files = self.__get_missing_output_files(runnable, module) input_files, changed_input_files = self.__get_changed_input_files(runnable, module) if (not output_files and not input_files) or missing_output_files or changed_input_files: - runnable.run(self.build_unit, module) - self.change_detection.track_files(module, *input_files) + modules_to_be_run.append(module) + input_files_per_module.append(input_files) + + try: + runnable.run_all(self.build_unit, modules_to_be_run) + except NotImplementedError: + try: + for module in modules_to_be_run: + runnable.run(self.build_unit, module) + except NotImplementedError as error: + raise RuntimeError('Class ' + type(runnable).__name__ + + ' must implement either the "run_all" or "run" method') from error + + for i, module in enumerate(modules_to_be_run): + self.change_detection.track_files(module, *input_files_per_module[i]) def clean(self, module_registry: ModuleRegistry): for runnable in self.runnables: @@ -294,12 +318,12 @@ def __init__(self, module_filter: Module.Filter): """ self.module_filter = module_filter - def run_all(self, build_unit: BuildUnit, module: List[Module]): + def run_all(self, build_unit: BuildUnit, modules: List[Module]): """ May be overridden by subclasses in order to apply the target to all modules that match the filter. :param build_unit: The build unit, the target belongs to - :param module: A list that contains the modules, the target should be applied to + :param modules: A list that contains the modules, the target should be applied to """ raise NotImplementedError('Class ' + type(self).__name__ + ' does not implement the "run_all" method') From 7b465e906c139f814c4f1d9051e49c913560fa35 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Fri, 13 Dec 2024 00:47:40 +0100 Subject: [PATCH 100/114] Dynamically register targets for generating index files referencing API documentations. --- .../targets/documentation/cpp/__init__.py | 11 ++- .../targets/documentation/cpp/modules.py | 12 ++- .../targets/documentation/cpp/targets.py | 14 +++- build_system/targets/documentation/modules.py | 33 ++++++++ .../targets/documentation/python/__init__.py | 11 ++- .../targets/documentation/python/modules.py | 13 +++- .../targets/documentation/python/targets.py | 14 +++- build_system/targets/documentation/targets.py | 77 +++++++++++++++++++ 8 files changed, 173 insertions(+), 12 deletions(-) create mode 100644 build_system/targets/documentation/modules.py create mode 100644 build_system/targets/documentation/targets.py diff --git a/build_system/targets/documentation/cpp/__init__.py b/build_system/targets/documentation/cpp/__init__.py index fa59209a6..4ad6c586f 100644 --- a/build_system/targets/documentation/cpp/__init__.py +++ b/build_system/targets/documentation/cpp/__init__.py @@ -9,12 +9,19 @@ from core.targets import TargetBuilder from targets.documentation.cpp.modules import CppApidocModule -from targets.documentation.cpp.targets import ApidocCpp +from targets.documentation.cpp.targets import ApidocCpp, ApidocIndexCpp from targets.paths import Project +APIDOC_CPP = 'apidoc_cpp' + +APIDOC_CPP_INDEX = 'apidoc_cpp_index' + TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ - .add_build_target('apidoc_cpp') \ + .add_build_target(APIDOC_CPP) \ .set_runnables(ApidocCpp()) \ + .add_build_target(APIDOC_CPP_INDEX) \ + .depends_on(APIDOC_CPP) \ + .set_runnables(ApidocIndexCpp()) \ .build() MODULES = [ diff --git a/build_system/targets/documentation/cpp/modules.py b/build_system/targets/documentation/cpp/modules.py index a6f6a2e0c..afdffd683 100644 --- a/build_system/targets/documentation/cpp/modules.py +++ b/build_system/targets/documentation/cpp/modules.py @@ -9,13 +9,15 @@ from core.modules import Module from util.files import FileSearch, FileType +from targets.documentation.modules import ApidocModule -class CppApidocModule(Module): + +class CppApidocModule(ApidocModule): """ A module that contains C++ code for which an API documentation can be generated. """ - class Filter(Module.Filter): + class Filter(ApidocModule.Filter): """ A filter that matches code modules. """ @@ -38,8 +40,8 @@ def __init__(self, :param header_file_search: The `FileSearch` that should be used to search for the header files to be included in the API documentation """ + super().__init__(output_directory) self.root_directory = root_directory - self.output_directory = output_directory self.project_name = project_name self.include_directory_name = include_directory_name self.header_file_search = header_file_search @@ -59,5 +61,9 @@ def find_header_files(self) -> List[str]: """ return self.header_file_search.filter_by_file_type(FileType.cpp()).list(self.include_directory) + def create_reference(self) -> str: + return 'Library libmlrl' + self.project_name + ' <' + path.join(path.basename(self.output_directory), + 'filelist.rst') + '>' + def __str__(self) -> str: return 'CppApidocModule {root_directory="' + self.root_directory + '"}' diff --git a/build_system/targets/documentation/cpp/targets.py b/build_system/targets/documentation/cpp/targets.py index 4970623ad..44ca0cc69 100644 --- a/build_system/targets/documentation/cpp/targets.py +++ b/build_system/targets/documentation/cpp/targets.py @@ -13,6 +13,9 @@ from targets.documentation.cpp.breathe_apidoc import BreatheApidoc from targets.documentation.cpp.doxygen import Doxygen from targets.documentation.cpp.modules import CppApidocModule +from targets.documentation.targets import ApidocIndex + +MODULE_FILTER = CppApidocModule.Filter() class ApidocCpp(BuildTarget.Runnable): @@ -21,7 +24,7 @@ class ApidocCpp(BuildTarget.Runnable): """ def __init__(self): - super().__init__(CppApidocModule.Filter()) + super().__init__(MODULE_FILTER) def run(self, build_unit: BuildUnit, module: Module): Log.info('Generating C++ API documentation for directory "%s"...', module.root_directory) @@ -37,3 +40,12 @@ def get_input_files(self, module: Module) -> List[str]: def get_clean_files(self, module: Module) -> List[str]: Log.info('Removing C++ API documentation for directory "%s"...', module.root_directory) return super().get_clean_files(module) + + +class ApidocIndexCpp(ApidocIndex): + """ + Generates index files referencing API documentations for C++ code. + """ + + def __init__(self): + super().__init__(MODULE_FILTER) diff --git a/build_system/targets/documentation/modules.py b/build_system/targets/documentation/modules.py new file mode 100644 index 000000000..0dc757bfa --- /dev/null +++ b/build_system/targets/documentation/modules.py @@ -0,0 +1,33 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that provide access to a Sphinx documentation. +""" +from abc import ABC, abstractmethod + +from core.modules import Module + + +class ApidocModule(Module, ABC): + """ + An abstract base class for all modules that contain code for which an API documentation can be generated. + """ + + class Filter(Module.Filter, ABC): + """ + A filter that matches apidoc modules. + """ + + def __init__(self, output_directory: str): + """ + :param output_directory: The path to the directory where the API documentation should be stored + """ + self.output_directory = output_directory + + @abstractmethod + def create_reference(self) -> str: + """ + Must be implemented by subclasses in order to create a reference to API documentation. + + :return: The reference that has been created + """ diff --git a/build_system/targets/documentation/python/__init__.py b/build_system/targets/documentation/python/__init__.py index bd435a418..ec1df9e11 100644 --- a/build_system/targets/documentation/python/__init__.py +++ b/build_system/targets/documentation/python/__init__.py @@ -9,14 +9,21 @@ from core.targets import TargetBuilder from targets.documentation.python.modules import PythonApidocModule -from targets.documentation.python.targets import ApidocPython +from targets.documentation.python.targets import ApidocIndexPython, ApidocPython from targets.packaging import INSTALL_WHEELS from targets.paths import Project +APIDOC_PYTHON = 'apidoc_python' + +APIDOC_PYTHON_INDEX = 'apidoc_python_index' + TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ - .add_build_target('apidoc_python') \ + .add_build_target(APIDOC_PYTHON) \ .depends_on(INSTALL_WHEELS) \ .set_runnables(ApidocPython()) \ + .add_build_target(APIDOC_PYTHON_INDEX) \ + .depends_on(APIDOC_PYTHON) \ + .set_runnables(ApidocIndexPython()) \ .build() MODULES = [ diff --git a/build_system/targets/documentation/python/modules.py b/build_system/targets/documentation/python/modules.py index 5648ad418..3e0c9edbc 100644 --- a/build_system/targets/documentation/python/modules.py +++ b/build_system/targets/documentation/python/modules.py @@ -9,13 +9,15 @@ from core.modules import Module from util.files import FileSearch, FileType +from targets.documentation.modules import ApidocModule -class PythonApidocModule(Module): + +class PythonApidocModule(ApidocModule): """ A module that contains Python code for which an API documentation can be generated. """ - class Filter(Module.Filter): + class Filter(ApidocModule.Filter): """ A filter that matches code modules. """ @@ -36,8 +38,8 @@ def __init__(self, :param source_file_search: The `FileSearch` that should be used to search for the header files to be included in the API documentation """ + super().__init__(output_directory) self.root_directory = root_directory - self.output_directory = output_directory self.source_directory_name = source_directory_name self.source_file_search = source_file_search @@ -56,5 +58,10 @@ def find_source_files(self) -> List[str]: """ return self.source_file_search.filter_by_file_type(FileType.python()).list(self.source_directory) + def create_reference(self) -> str: + project_name = path.basename(self.output_directory) + return 'Package mlrl-' + path.basename(self.output_directory) + ' <' + path.join( + project_name, self.source_directory_name + '.' + project_name + '.rst') + '>' + def __str__(self) -> str: return 'PythonApidocModule {root_directory="' + self.root_directory + '"}' diff --git a/build_system/targets/documentation/python/targets.py b/build_system/targets/documentation/python/targets.py index 5db3c086a..8cc20ae38 100644 --- a/build_system/targets/documentation/python/targets.py +++ b/build_system/targets/documentation/python/targets.py @@ -12,6 +12,9 @@ from targets.documentation.python.modules import PythonApidocModule from targets.documentation.python.sphinx_apidoc import SphinxApidoc +from targets.documentation.targets import ApidocIndex + +MODULE_FILTER = PythonApidocModule.Filter() class ApidocPython(BuildTarget.Runnable): @@ -20,7 +23,7 @@ class ApidocPython(BuildTarget.Runnable): """ def __init__(self): - super().__init__(PythonApidocModule.Filter()) + super().__init__(MODULE_FILTER) def run(self, build_unit: BuildUnit, module: Module): Log.info('Generating Python API documentation for directory "%s"...', module.root_directory) @@ -35,3 +38,12 @@ def get_input_files(self, module: Module) -> List[str]: def get_clean_files(self, module: Module) -> List[str]: Log.info('Removing Python API documentation for directory "%s"...', module.root_directory) return super().get_clean_files(module) + + +class ApidocIndexPython(ApidocIndex): + """ + Generates index files referencing API documentations for Python code. + """ + + def __init__(self): + super().__init__(MODULE_FILTER) diff --git a/build_system/targets/documentation/targets.py b/build_system/targets/documentation/targets.py new file mode 100644 index 000000000..05918be04 --- /dev/null +++ b/build_system/targets/documentation/targets.py @@ -0,0 +1,77 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Implements targets for generating documentations. +""" +from abc import ABC +from os import path +from typing import Dict, List, Optional + +from core.build_unit import BuildUnit +from core.modules import Module +from core.targets import BuildTarget +from util.io import TextFile +from util.log import Log + +from targets.documentation.modules import ApidocModule + + +class ApidocIndex(BuildTarget.Runnable, ABC): + """ + An abstract base class for all targets that generate index files referencing API documentations. + """ + + @staticmethod + def __get_template(module: ApidocModule) -> Optional[str]: + parent_directory = path.dirname(module.output_directory) + template = path.join(parent_directory, 'index.md.template') + return template if path.isfile(template) else None + + @staticmethod + def __get_templates_and_modules(modules: List[ApidocModule]) -> Dict[str, List[ApidocModule]]: + modules_by_template = {} + + for module in modules: + template = ApidocIndex.__get_template(module) + + if template: + modules_in_directory = modules_by_template.setdefault(template, []) + modules_in_directory.append(module) + + return modules_by_template + + @staticmethod + def __index_file(template: str) -> str: + return path.join(path.dirname(template), 'index.md') + + def __init__(self, module_filter: ApidocModule.Filter): + """ + :param module_filter: A filter that matches the modules, the target should be applied to + """ + super().__init__(module_filter) + + def run_all(self, _: BuildUnit, modules: List[Module]): + for template, modules_in_directory in self.__get_templates_and_modules(modules).items(): + Log.info('Generating index file referencing API documentations from template "%s"...', template) + references = [module.create_reference() + '\n' for module in modules_in_directory] + new_lines = [] + + for line in TextFile(template).lines: + if line.strip() == '%s': + new_lines.extend(references) + else: + new_lines.append(line) + + TextFile(self.__index_file(template), accept_missing=True).write_lines(*new_lines) + + def get_input_files(self, module: Module) -> List[str]: + template = self.__get_template(module) + return [template] if template else [] + + def get_output_files(self, module: Module) -> List[str]: + template = self.__get_template(module) + return [self.__index_file(template)] if template else [] + + def get_clean_files(self, module: Module) -> List[str]: + Log.info('Removing index file referencing API documentation in directory "%s"', module.output_directory) + return super().get_clean_files(module) From c720b9e14eab3546003d25aa3f99cdac0d38e1aa Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Fri, 13 Dec 2024 00:59:36 +0100 Subject: [PATCH 101/114] Dynamically register targets and modules for generating Sphinx documentations. --- build_system/documentation.py | 165 ------------------ .../targets/documentation/__init__.py | 37 ++++ build_system/targets/documentation/modules.py | 41 +++++ .../targets/documentation}/requirements.txt | 0 .../targets/documentation/sphinx_build.py | 38 ++++ build_system/targets/documentation/targets.py | 26 ++- build_system/targets/paths.py | 4 +- 7 files changed, 144 insertions(+), 167 deletions(-) delete mode 100644 build_system/documentation.py rename {doc => build_system/targets/documentation}/requirements.txt (100%) create mode 100644 build_system/targets/documentation/sphinx_build.py diff --git a/build_system/documentation.py b/build_system/documentation.py deleted file mode 100644 index d8e82c010..000000000 --- a/build_system/documentation.py +++ /dev/null @@ -1,165 +0,0 @@ -""" -Author: Michael Rapp (michael.rapp.ml@gmail.com) - -Provides utility functions for generating the documentation. -""" -from os import environ, makedirs, path, remove -from typing import List - -from modules_old import CPP_MODULE, DOC_MODULE, PYTHON_MODULE -from util.env import set_env -from util.io import read_file, write_file -from util.log import Log -from util.run import Program - - -def __doxygen(project_name: str, input_dir: str, output_dir: str): - makedirs(output_dir, exist_ok=True) - env = environ.copy() - set_env(env, 'DOXYGEN_PROJECT_NAME', 'libmlrl' + project_name) - set_env(env, 'DOXYGEN_INPUT_DIR', input_dir) - set_env(env, 'DOXYGEN_OUTPUT_DIR', output_dir) - set_env(env, 'DOXYGEN_PREDEFINED', 'MLRL' + project_name.upper() + '_API=') - Program('doxygen', DOC_MODULE.doxygen_config_file) \ - .print_arguments(False) \ - .install_program(False) \ - .use_environment(env) \ - .run() - - -def __breathe_apidoc(source_dir: str, output_dir: str, project: str): - Program('breathe-apidoc', '--members', '--project', project, '-g', 'file', '-o', output_dir, source_dir) \ - .set_build_unit(DOC_MODULE) \ - .print_arguments(True) \ - .add_dependencies('breathe') \ - .install_program(False) \ - .run() - - -def __sphinx_apidoc(source_dir: str, output_dir: str): - Program('sphinx-apidoc', '--separate', '--module-first', '--no-toc', '-o', output_dir, source_dir, '*.so*') \ - .set_build_unit(DOC_MODULE) \ - .print_arguments(True) \ - .add_dependencies('sphinx') \ - .install_program(False) \ - .run() - - root_rst_file = path.join(output_dir, 'mlrl.rst') - - if path.isfile(root_rst_file): - remove(root_rst_file) - - -def __sphinx_build(source_dir: str, output_dir: str): - Program('sphinx-build', '--jobs', 'auto', source_dir, output_dir) \ - .set_build_unit(DOC_MODULE) \ - .print_arguments(True) \ - .add_dependencies('furo', 'myst-parser', 'sphinxext-opengraph', 'sphinx-inline-tabs', 'sphinx-copybutton', - 'sphinx-favicon',) \ - .install_program(False) \ - .run() - - -def __read_tocfile_template(directory: str) -> List[str]: - with read_file(path.join(directory, 'index.md.template')) as file: - return file.readlines() - - -def __write_tocfile(directory: str, tocfile_entries: List[str]): - tocfile_template = __read_tocfile_template(directory) - tocfile = [] - - for line in tocfile_template: - if line.strip() == '%s': - tocfile.extend(tocfile_entries) - else: - tocfile.append(line) - - with write_file(path.join(directory, 'index.md')) as file: - file.writelines(tocfile) - - -# pylint: disable=unused-argument -def apidoc_cpp(env, target, source): - """ - Builds the API documentation for a single C++ subproject. - - :param env: The scons environment - :param target: The path of the files that belong to the API documentation, if it has already been built, or the - path of the directory, where the API documentation should be stored - :param source: The paths of the source files from which the API documentation should be built - """ - if target: - apidoc_subproject = DOC_MODULE.find_cpp_apidoc_subproject(target[0].path) - - if apidoc_subproject: - subproject_name = apidoc_subproject.name - Log.info('Generating C++ API documentation for subproject "%s"...', subproject_name) - include_dir = path.join(apidoc_subproject.source_subproject.root_dir, 'include') - build_dir = apidoc_subproject.build_dir - __doxygen(project_name=subproject_name, input_dir=include_dir, output_dir=build_dir) - __breathe_apidoc(source_dir=path.join(build_dir, 'xml'), output_dir=build_dir, project=subproject_name) - - -def apidoc_cpp_tocfile(**_): - """ - Generates a tocfile referencing the C++ API documentation for all existing subprojects. - """ - Log.info('Generating tocfile referencing the C++ API documentation for all subprojects...') - tocfile_entries = [] - - for subproject in CPP_MODULE.find_subprojects(): - apidoc_subproject = DOC_MODULE.get_cpp_apidoc_subproject(subproject) - root_file = apidoc_subproject.root_file - - if path.isfile(root_file): - tocfile_entries.append('Library libmlrl' + apidoc_subproject.name + ' <' - + path.relpath(root_file, DOC_MODULE.apidoc_dir_cpp) + '>\n') - - __write_tocfile(DOC_MODULE.apidoc_dir_cpp, tocfile_entries) - - -# pylint: disable=unused-argument -def apidoc_python(env, target, source): - """ - Builds the API documentation for a single Python subproject. - - :param env: The scons environment - :param target: The path of the files that belong to the API documentation, if it has already been built, or the - path of the directory, where the API documentation should be stored - :param source: The paths of the source files from which the API documentation should be built - """ - if target: - apidoc_subproject = DOC_MODULE.find_python_apidoc_subproject(target[0].path) - - if apidoc_subproject: - Log.info('Generating Python API documentation for subproject "%s"...', apidoc_subproject.name) - build_dir = apidoc_subproject.build_dir - makedirs(build_dir, exist_ok=True) - __sphinx_apidoc(source_dir=apidoc_subproject.source_subproject.source_dir, output_dir=build_dir) - - -def apidoc_python_tocfile(**_): - """ - Generates a tocfile referencing the Python API documentation for all existing subprojects. - """ - Log.info('Generating tocfile referencing the Python API documentation for all subprojects...') - tocfile_entries = [] - - for subproject in PYTHON_MODULE.find_subprojects(): - apidoc_subproject = DOC_MODULE.get_python_apidoc_subproject(subproject) - root_file = apidoc_subproject.root_file - - if path.isfile(root_file): - tocfile_entries.append('Package mlrl-' + apidoc_subproject.name + ' <' - + path.relpath(root_file, DOC_MODULE.apidoc_dir_python) + '>\n') - - __write_tocfile(DOC_MODULE.apidoc_dir_python, tocfile_entries) - - -def doc(**_): - """ - Builds the documentation. - """ - Log.info('Generating documentation...') - __sphinx_build(source_dir=DOC_MODULE.root_dir, output_dir=DOC_MODULE.build_dir) diff --git a/build_system/targets/documentation/__init__.py b/build_system/targets/documentation/__init__.py index e69de29bb..9e7f81fb2 100644 --- a/build_system/targets/documentation/__init__.py +++ b/build_system/targets/documentation/__init__.py @@ -0,0 +1,37 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Defines targets and modules for generating documentations. +""" +from os import path + +from core.build_unit import BuildUnit +from core.targets import TargetBuilder + +from targets.documentation.cpp import APIDOC_CPP, APIDOC_CPP_INDEX +from targets.documentation.modules import SphinxModule +from targets.documentation.python import APIDOC_PYTHON, APIDOC_PYTHON_INDEX +from targets.documentation.targets import BuildDocumentation +from targets.paths import Project + +APIDOC_INDEX = 'apidoc_index' + +TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ + .add_phony_target('apidoc') \ + .depends_on(APIDOC_CPP, APIDOC_PYTHON, clean_dependencies=True) \ + .nop() \ + .add_phony_target(APIDOC_INDEX) \ + .depends_on(APIDOC_CPP_INDEX, APIDOC_PYTHON_INDEX, clean_dependencies=True) \ + .nop() \ + .add_build_target('doc') \ + .depends_on(APIDOC_INDEX, clean_dependencies=True) \ + .set_runnables(BuildDocumentation()) \ + .build() + +MODULES = [ + SphinxModule( + root_directory=Project.Documentation.root_directory, + output_directory=path.join(Project.Documentation.root_directory, Project.Documentation.build_directory_name), + source_file_search=Project.Documentation.file_search(), + ), +] diff --git a/build_system/targets/documentation/modules.py b/build_system/targets/documentation/modules.py index 0dc757bfa..d4407c1c9 100644 --- a/build_system/targets/documentation/modules.py +++ b/build_system/targets/documentation/modules.py @@ -4,8 +4,10 @@ Provides classes that provide access to a Sphinx documentation. """ from abc import ABC, abstractmethod +from typing import List from core.modules import Module +from util.files import FileSearch class ApidocModule(Module, ABC): @@ -31,3 +33,42 @@ def create_reference(self) -> str: :return: The reference that has been created """ + + +class SphinxModule(Module): + """ + A module that contains a Sphinx documentation. + """ + + class Filter(Module.Filter): + """ + A filter that matches sphinx modules. + """ + + def matches(self, module: Module) -> bool: + return isinstance(module, SphinxModule) + + def __init__(self, + root_directory: str, + output_directory: str, + source_file_search: FileSearch = FileSearch().set_recursive(True)): + """ + :param root_directory: The path to the module's root directory + :param output_directory: The path to the directory where the documentation should be stored + :param source_file_search: The `FileSearch` that should be used to search for the source files of the + documentation + """ + self.root_directory = root_directory + self.output_directory = output_directory + self.source_file_search = source_file_search + + def find_source_files(self) -> List[str]: + """ + Finds and returns all source files of the documentation. + + :return: A list that contains the source files that have been found + """ + return self.source_file_search.list(self.root_directory) + + def __str__(self) -> str: + return 'SphinxModule {root_directory="' + self.root_directory + '"}' diff --git a/doc/requirements.txt b/build_system/targets/documentation/requirements.txt similarity index 100% rename from doc/requirements.txt rename to build_system/targets/documentation/requirements.txt diff --git a/build_system/targets/documentation/sphinx_build.py b/build_system/targets/documentation/sphinx_build.py new file mode 100644 index 000000000..238488637 --- /dev/null +++ b/build_system/targets/documentation/sphinx_build.py @@ -0,0 +1,38 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides classes that allow to run the external program "sphinx-build". +""" +from os import path + +from core.build_unit import BuildUnit +from util.run import Program + +from targets.documentation.cpp.modules import CppApidocModule + + +class SphinxBuild(Program): + """ + Allows to run the external program "sphinx-build". + """ + + def __init__(self, build_unit: BuildUnit, module: CppApidocModule): + """ + :param build_unit: The build unit from which the program should be run + :param module: The module, the program should be applied to + """ + super().__init__('sphinx-build', '--jobs', 'auto', module.root_directory, + path.join(module.output_directory, 'html')) + self.module = module + self.print_arguments(True) + self.install_program(False) + self.add_dependencies( + 'furo', + 'myst-parser', + 'sphinx', + 'sphinx-copybutton', + 'sphinx-favicon', + 'sphinx-inline-tabs', + 'sphinxext-opengraph', + ) + self.set_build_unit(build_unit) diff --git a/build_system/targets/documentation/targets.py b/build_system/targets/documentation/targets.py index 05918be04..41c6921ad 100644 --- a/build_system/targets/documentation/targets.py +++ b/build_system/targets/documentation/targets.py @@ -13,7 +13,8 @@ from util.io import TextFile from util.log import Log -from targets.documentation.modules import ApidocModule +from targets.documentation.modules import ApidocModule, SphinxModule +from targets.documentation.sphinx_build import SphinxBuild class ApidocIndex(BuildTarget.Runnable, ABC): @@ -75,3 +76,26 @@ def get_output_files(self, module: Module) -> List[str]: def get_clean_files(self, module: Module) -> List[str]: Log.info('Removing index file referencing API documentation in directory "%s"', module.output_directory) return super().get_clean_files(module) + + +class BuildDocumentation(BuildTarget.Runnable): + """ + Generates documentations. + """ + + def __init__(self): + super().__init__(SphinxModule.Filter()) + + def run(self, build_unit: BuildUnit, module: Module): + Log.info('Generating documentation for directory "%s"...', module.root_directory) + SphinxBuild(build_unit, module).run() + + def get_input_files(self, module: Module) -> List[str]: + return module.find_source_files() + + def get_output_files(self, module: Module) -> List[str]: + return [module.output_directory] + + def get_clean_files(self, module: Module) -> List[str]: + Log.info('Removing documentation generated for directory "%s"...', module.root_directory) + return super().get_clean_files(module) diff --git a/build_system/targets/paths.py b/build_system/targets/paths.py index e32015cf6..4da0f9aa3 100644 --- a/build_system/targets/paths.py +++ b/build_system/targets/paths.py @@ -115,6 +115,8 @@ class Documentation: apidoc_directory = path.join(root_directory, 'developer_guide', 'api') + build_directory_name = '_build' + @staticmethod def file_search() -> FileSearch: """ @@ -124,7 +126,7 @@ def file_search() -> FileSearch: """ return FileSearch() \ .set_recursive(True) \ - .exclude_subdirectories_by_name('_build') + .exclude_subdirectories_by_name(Project.Documentation.build_directory_name) class Github: """ From 42f12f43be0a90091c021ae5309f50c37ff20412 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Fri, 13 Dec 2024 01:22:42 +0100 Subject: [PATCH 102/114] Define default target. --- build_system/main.py | 29 ++++++++++++++++++++++++++--- build_system/targets/__init__.py | 8 ++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/build_system/main.py b/build_system/main.py index 993b07c9b..6978aa2e1 100644 --- a/build_system/main.py +++ b/build_system/main.py @@ -8,7 +8,7 @@ from argparse import ArgumentParser from importlib.util import module_from_spec, spec_from_file_location from types import ModuleType -from typing import List +from typing import List, Optional from core.build_unit import BuildUnit from core.modules import Module, ModuleRegistry @@ -97,8 +97,30 @@ def __register_targets(init_files: List[str]) -> TargetRegistry: return target_registry -def __create_dependency_graph(target_registry: TargetRegistry, args) -> DependencyGraph: +def __find_default_target(init_files: List[str]) -> Optional[str]: + Log.verbose('Searching for default target...') + default_targets = [] + + for init_file in init_files: + default_target = getattr(__import_source_file(init_file), 'DEFAULT_TARGET', None) + + if default_target and isinstance(default_target, str): + Log.verbose('Found default target "%s" defined in file "%s"', default_target, init_file) + default_targets.append(default_target) + + if len(default_targets) > 1: + raise RuntimeError('Only one default target may be specified, but found: ' + format_iterable(default_targets)) + + if default_targets: + return default_targets[0] + + Log.verbose('Found no default target.') + return None + + +def __create_dependency_graph(target_registry: TargetRegistry, args, default_target: Optional[str]) -> DependencyGraph: targets = args.targets + targets = targets if targets else ([default_target] if default_target else []) clean = args.clean Log.verbose('Creating dependency graph for %s targets [%s]...', 'cleaning' if clean else 'running', format_iterable(targets)) @@ -123,7 +145,8 @@ def main(): init_files = __find_init_files() module_registry = __register_modules(init_files) target_registry = __register_targets(init_files) - dependency_graph = __create_dependency_graph(target_registry, args) + default_target = __find_default_target(init_files) + dependency_graph = __create_dependency_graph(target_registry, args, default_target=default_target) __execute_dependency_graph(dependency_graph, module_registry) diff --git a/build_system/targets/__init__.py b/build_system/targets/__init__.py index e69de29bb..18b8fd261 100644 --- a/build_system/targets/__init__.py +++ b/build_system/targets/__init__.py @@ -0,0 +1,8 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Defines the build sytem's default target. +""" +from targets.packaging import INSTALL_WHEELS + +DEFAULT_TARGET = INSTALL_WHEELS From a309d56a6ef1d264ec7b98d8e542a6345c181b76 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Fri, 13 Dec 2024 01:36:38 +0100 Subject: [PATCH 103/114] Edit comments. --- build_system/targets/code_style/modules.py | 6 +++--- build_system/targets/compilation/modules.py | 6 +++--- build_system/targets/dependencies/github/modules.py | 6 +++--- build_system/targets/dependencies/python/modules.py | 6 +++--- build_system/targets/documentation/cpp/modules.py | 6 +++--- build_system/targets/documentation/modules.py | 9 +++++---- build_system/targets/documentation/python/modules.py | 6 +++--- build_system/targets/packaging/modules.py | 6 +++--- build_system/targets/testing/cpp/modules.py | 6 +++--- build_system/targets/testing/modules.py | 4 ++-- build_system/targets/testing/python/modules.py | 6 +++--- 11 files changed, 34 insertions(+), 33 deletions(-) diff --git a/build_system/targets/code_style/modules.py b/build_system/targets/code_style/modules.py index 04eb37830..9c922d493 100644 --- a/build_system/targets/code_style/modules.py +++ b/build_system/targets/code_style/modules.py @@ -1,7 +1,7 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Provides classes that provide access to directories and files that belong to individual modules. +Implements modules that provide access to source code. """ from typing import List @@ -11,12 +11,12 @@ class CodeModule(Module): """ - A module that contains source code. + A module that provides access to source code. """ class Filter(Module.Filter): """ - A filter that matches code modules. + A filter that matches modules of type `CodeModule`. """ def __init__(self, *file_types: FileType): diff --git a/build_system/targets/compilation/modules.py b/build_system/targets/compilation/modules.py index 772942496..7ba5097a4 100644 --- a/build_system/targets/compilation/modules.py +++ b/build_system/targets/compilation/modules.py @@ -1,7 +1,7 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Provides classes that provide access to directories and files that belong to individual modules. +Implements modules that provide access to source code that must be compiled. """ from os import path from typing import List, Optional @@ -12,12 +12,12 @@ class CompilationModule(Module): """ - A module that contains source code that must be compiled. + A module that provides access to source code that must be compiled. """ class Filter(Module.Filter): """ - A filter that matches modules that contain source code that must be compiled. + A filter that matches modules of type `CompilationModule`. """ def __init__(self, *file_types: FileType): diff --git a/build_system/targets/dependencies/github/modules.py b/build_system/targets/dependencies/github/modules.py index 4ac3cbe3d..5f4b31d6f 100644 --- a/build_system/targets/dependencies/github/modules.py +++ b/build_system/targets/dependencies/github/modules.py @@ -1,7 +1,7 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Provides classes that provide access to GitHub workflows tha belong to individual modules. +Implements modules that provide access to GitHub workflows. """ from typing import List @@ -11,12 +11,12 @@ class GithubWorkflowModule(Module): """ - A module that contains GitHub workflows. + A module that provides access to GitHub workflows. """ class Filter(Module.Filter): """ - A filter that matches modules that contain GitHub workflows. + A filter that matches modules of type `GithubWorkflowModule`. """ def matches(self, module: Module) -> bool: diff --git a/build_system/targets/dependencies/python/modules.py b/build_system/targets/dependencies/python/modules.py index c487f60c9..c75cf18c7 100644 --- a/build_system/targets/dependencies/python/modules.py +++ b/build_system/targets/dependencies/python/modules.py @@ -1,7 +1,7 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Provides classes that provide access to Python requirements files that belong to individual modules. +Implements modules that provide access to Python requirements files. """ from enum import Enum from typing import List @@ -20,12 +20,12 @@ class DependencyType(Enum): class PythonDependencyModule(Module): """ - A module that contains source code that comes with Python dependencies. + A module that provides access to Python requirements files. """ class Filter(Module.Filter): """ - A filter that matches modules that contain source that comes with Python dependencies. + A filter that matches modules of type `PythonDependencyModule`. """ def __init__(self, *dependency_types: DependencyType): diff --git a/build_system/targets/documentation/cpp/modules.py b/build_system/targets/documentation/cpp/modules.py index afdffd683..69d0bb5b6 100644 --- a/build_system/targets/documentation/cpp/modules.py +++ b/build_system/targets/documentation/cpp/modules.py @@ -1,7 +1,7 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Provides classes that provide access to C++ code for which an API documentation can be generated. +Implements modules that provide access to C++ code for which an API documentation can be generated. """ from os import path from typing import List @@ -14,12 +14,12 @@ class CppApidocModule(ApidocModule): """ - A module that contains C++ code for which an API documentation can be generated. + A module that provides access to C++ code for which an API documentation can be generated. """ class Filter(ApidocModule.Filter): """ - A filter that matches code modules. + A filter that matches modules of type `CppApidocModule`. """ def matches(self, module: Module) -> bool: diff --git a/build_system/targets/documentation/modules.py b/build_system/targets/documentation/modules.py index d4407c1c9..00c24d9ae 100644 --- a/build_system/targets/documentation/modules.py +++ b/build_system/targets/documentation/modules.py @@ -12,12 +12,13 @@ class ApidocModule(Module, ABC): """ - An abstract base class for all modules that contain code for which an API documentation can be generated. + An abstract base class for all modules that provide access to source code for which an API documentation can be + generated. """ class Filter(Module.Filter, ABC): """ - A filter that matches apidoc modules. + A filter that matches modules of type `ApidocModule`. """ def __init__(self, output_directory: str): @@ -37,12 +38,12 @@ def create_reference(self) -> str: class SphinxModule(Module): """ - A module that contains a Sphinx documentation. + A module that provides access to a Sphinx documentation. """ class Filter(Module.Filter): """ - A filter that matches sphinx modules. + A filter that matches modules of type `SphinxModule`. """ def matches(self, module: Module) -> bool: diff --git a/build_system/targets/documentation/python/modules.py b/build_system/targets/documentation/python/modules.py index 3e0c9edbc..72a71c57b 100644 --- a/build_system/targets/documentation/python/modules.py +++ b/build_system/targets/documentation/python/modules.py @@ -1,7 +1,7 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Provides classes that provide access to Python code for which an API documentation can be generated. +Implements modules that provide access to Python code for which an API documentation can be generated. """ from os import path from typing import List @@ -14,12 +14,12 @@ class PythonApidocModule(ApidocModule): """ - A module that contains Python code for which an API documentation can be generated. + A module that provides access to Python code for which an API documentation can be generated. """ class Filter(ApidocModule.Filter): """ - A filter that matches code modules. + A filter that matches modules of type `PythonApidocModule`. """ def matches(self, module: Module) -> bool: diff --git a/build_system/targets/packaging/modules.py b/build_system/targets/packaging/modules.py index c4bbfd2f3..e3015a098 100644 --- a/build_system/targets/packaging/modules.py +++ b/build_system/targets/packaging/modules.py @@ -1,7 +1,7 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Provides classes that provide access to Python code that can be built as wheel packages. +Implements modules that provide access to Python code that can be built as wheel packages. """ from os import path from typing import List @@ -12,12 +12,12 @@ class PythonPackageModule(Module): """ - A module that contains Python code that can be built as wheel packages. + A module that provides access to Python code that can be built as wheel packages. """ class Filter(Module.Filter): """ - A filter that matches modules that contain Python code that can be built as wheel packages. + A filter that matches modules of type `PythonPackageModule`. """ def matches(self, module: Module) -> bool: diff --git a/build_system/targets/testing/cpp/modules.py b/build_system/targets/testing/cpp/modules.py index b294742b3..104ea8133 100644 --- a/build_system/targets/testing/cpp/modules.py +++ b/build_system/targets/testing/cpp/modules.py @@ -1,7 +1,7 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Provides classes that provide access to automated tests for C++ code that belong to individual modules. +Implements modules that provide access to automated tests for C++ code. """ from os import path @@ -12,12 +12,12 @@ class CppTestModule(TestModule): """ - A module that contains automated tests for C++ code. + A module that provides access to automated tests for C++ code. """ class Filter(Module.Filter): """ - A filter that matches modules that contain automated tests for C++ code. + A filter that matches modules of type `CppTestModule`. """ def matches(self, module: Module) -> bool: diff --git a/build_system/targets/testing/modules.py b/build_system/targets/testing/modules.py index ca0b283f1..7e1a4d82a 100644 --- a/build_system/targets/testing/modules.py +++ b/build_system/targets/testing/modules.py @@ -1,7 +1,7 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Provides classes that provide access to automated tests that belong to individual modules. +Implements modules that provide access to automated tests. """ from abc import ABC from os import environ @@ -12,7 +12,7 @@ class TestModule(Module, ABC): """ - An abstract base class for all modules that contain automated tests. + An abstract base class for all modules that provide access to automated tests. """ @property diff --git a/build_system/targets/testing/python/modules.py b/build_system/targets/testing/python/modules.py index 5d28c6955..965f1ad3e 100644 --- a/build_system/targets/testing/python/modules.py +++ b/build_system/targets/testing/python/modules.py @@ -1,7 +1,7 @@ """ Author: Michael Rapp (michael.rapp.ml@gmail.com) -Provides classes that provide access to automated tests for Python code that belong to individual modules. +Implements modules that provide access to automated tests for Python code. """ from os import path from typing import List @@ -14,12 +14,12 @@ class PythonTestModule(TestModule): """ - A module that contains automated tests for Python code. + A module that provides access to automated tests for Python code. """ class Filter(Module.Filter): """ - A filter that matches modules that contain automated tests for Python code. + A filter that matches modules of type `PythonTestModule`. """ def matches(self, module: Module) -> bool: From b06b0798a983f2f570eae50755f205dcf9ea7bb8 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Fri, 13 Dec 2024 02:00:44 +0100 Subject: [PATCH 104/114] Fix pylint errors. --- build_system/core/changes.py | 7 --- build_system/core/targets.py | 13 ++++- .../targets/dependencies/python/modules.py | 3 +- .../targets/dependencies/python/pip.py | 7 --- .../targets/documentation/python/modules.py | 4 +- build_system/targets/packaging/pip.py | 7 --- build_system/targets/testing/cpp/targets.py | 2 +- build_system/targets/versioning/changelog.py | 10 +++- build_system/targets/versioning/versioning.py | 21 ++++++- build_system/util/files.py | 56 +++++++++++++++---- 10 files changed, 87 insertions(+), 43 deletions(-) diff --git a/build_system/core/changes.py b/build_system/core/changes.py index 8d9884df6..528f0b70e 100644 --- a/build_system/core/changes.py +++ b/build_system/core/changes.py @@ -18,13 +18,6 @@ class JsonFile(TextFile): Allows to read and write the content of a JSON file. """ - def __init__(self, file: str, accept_missing: bool = False): - """ - :param file: The path to the JSON file - :param accept_missing: True, if no errors should be raised if the text file is missing, False otherwise - """ - super().__init__(file, accept_missing) - @cached_property def json(self) -> Dict: """ diff --git a/build_system/core/targets.py b/build_system/core/targets.py index 8e412da58..4446ef67f 100644 --- a/build_system/core/targets.py +++ b/build_system/core/targets.py @@ -39,7 +39,7 @@ def __str__(self) -> str: return self.target_name def __eq__(self, other) -> bool: - return type(self) == type(other) and self.target_name == other.target_name + return isinstance(other, type(self)) and self.target_name == other.target_name class Builder(ABC): """ @@ -143,6 +143,7 @@ def run(self, build_unit: BuildUnit, module: Module): """ raise NotImplementedError('Class ' + type(self).__name__ + ' does not implement the "run" method') + # pylint: disable=unused-argument def get_input_files(self, module: Module) -> List[str]: """ May be overridden by subclasses in order to return the input files required by the target. @@ -152,6 +153,7 @@ def get_input_files(self, module: Module) -> List[str]: """ return [] + # pylint: disable=unused-argument def get_output_files(self, module: Module) -> List[str]: """ May be overridden by subclasses in order to return the output files produced by the target. @@ -513,6 +515,8 @@ def from_dependency(targets_by_name: Dict[str, Target], dependency: Target.Depen def execute(self, module_registry: ModuleRegistry): """ Must be implemented by subclasses in order to execute the node. + + :param module_registry: A `ModuleRegistry` that may be used for looking up modules """ @abstractmethod @@ -527,7 +531,7 @@ def __str__(self) -> str: return '[' + self.target.name + ']' def __eq__(self, other) -> bool: - return type(self) == type(other) and self.target == other.target + return isinstance(other, type(self)) and self.target == other.target class RunNode(Node): """ @@ -606,6 +610,11 @@ def copy(self) -> 'DependencyGraph.Sequence': return copy def execute(self, module_registry: ModuleRegistry): + """ + Executes all nodes in the sequence. + + :param module_registry: A `ModuleRegistry` that may be used for looking up modules + """ current_node = self.first while current_node: diff --git a/build_system/targets/dependencies/python/modules.py b/build_system/targets/dependencies/python/modules.py index c75cf18c7..f5bba96a2 100644 --- a/build_system/targets/dependencies/python/modules.py +++ b/build_system/targets/dependencies/python/modules.py @@ -61,4 +61,5 @@ def find_requirements_files(self) -> List[str]: return self.requirements_file_search.filter_by_name('requirements.txt').list(self.root_directory) def __str__(self) -> str: - return 'PythonDependencyModule {dependency_type="' + self.dependency_type.value + '", root_directory="' + self.root_directory + '"}' + return ('PythonDependencyModule {dependency_type="' + self.dependency_type.value + '", root_directory="' + + self.root_directory + '"}') diff --git a/build_system/targets/dependencies/python/pip.py b/build_system/targets/dependencies/python/pip.py index b2667dac5..0d1a2db6d 100644 --- a/build_system/targets/dependencies/python/pip.py +++ b/build_system/targets/dependencies/python/pip.py @@ -45,13 +45,6 @@ def __init__(self, outdated: bool = False): super().__init__('list') self.add_conditional_arguments(outdated, '--outdated') - def __init__(self, *requirements_files: str): - """ - :param requirements_files: The paths to the requirements files that specify the versions of the packages to be - installed - """ - super().__init__(*requirements_files) - def install_all_packages(self): """ Installs all dependencies in the requirements file. diff --git a/build_system/targets/documentation/python/modules.py b/build_system/targets/documentation/python/modules.py index 72a71c57b..9c81fbbbe 100644 --- a/build_system/targets/documentation/python/modules.py +++ b/build_system/targets/documentation/python/modules.py @@ -35,8 +35,8 @@ def __init__(self, :param output_directory: The path to the directory where the API documentation should be stored :param source_directory_name: The name of the directory that contains the Python source files to be included in the API documentation - :param source_file_search: The `FileSearch` that should be used to search for the header files to be included - in the API documentation + :param source_file_search: The `FileSearch` that should be used to search for the header files to be + included in the API documentation """ super().__init__(output_directory) self.root_directory = root_directory diff --git a/build_system/targets/packaging/pip.py b/build_system/targets/packaging/pip.py index 7809ce7b1..ac4456743 100644 --- a/build_system/targets/packaging/pip.py +++ b/build_system/targets/packaging/pip.py @@ -22,13 +22,6 @@ def __init__(self, *wheels: str): """ super().__init__('install', '--force-reinstall', '--no-deps', *wheels) - def __init__(self, *requirements_files: str): - """ - :param requirements_files: The paths to the requirements files that specify the versions of the packages to be - installed - """ - super().__init__(*requirements_files) - def install_wheels(self, *wheels: str): """ Installs several wheel packages. diff --git a/build_system/targets/testing/cpp/targets.py b/build_system/targets/testing/cpp/targets.py index 0784b7f0b..1a572260e 100644 --- a/build_system/targets/testing/cpp/targets.py +++ b/build_system/targets/testing/cpp/targets.py @@ -5,7 +5,7 @@ """ from core.build_unit import BuildUnit from core.modules import Module -from core.targets import BuildTarget, PhonyTarget +from core.targets import PhonyTarget from targets.testing.cpp.meson import MesonTest from targets.testing.cpp.modules import CppTestModule diff --git a/build_system/targets/versioning/changelog.py b/build_system/targets/versioning/changelog.py index f31f3a40b..39026e6a4 100644 --- a/build_system/targets/versioning/changelog.py +++ b/build_system/targets/versioning/changelog.py @@ -3,8 +3,6 @@ Provides actions for validating and updating the project's changelog. """ -import sys - from dataclasses import dataclass, field from datetime import date from enum import Enum, auto @@ -141,6 +139,9 @@ def __validate_line(self, current_line: Optional[Line], previous_line: Optional[ @cached_property def parsed_lines(self) -> List[Line]: + """ + The lines in the changelog as `Line` objects. + """ parsed_lines = [] for i, line in enumerate(self.lines): @@ -153,6 +154,9 @@ def parsed_lines(self) -> List[Line]: @cached_property def changesets(self) -> List[Changeset]: + """ + A list that contains all changesets in the changelog. + """ changesets = [] for line in self.parsed_lines: @@ -170,7 +174,7 @@ def validate(self): """ previous_line = None - for i, current_line in enumerate(self.parsed_lines): + for current_line in self.parsed_lines: if current_line.line_type != LineType.BLANK: self.__validate_line(current_line=current_line, previous_line=previous_line) previous_line = current_line diff --git a/build_system/targets/versioning/versioning.py b/build_system/targets/versioning/versioning.py index ab3e7f967..58c364d86 100644 --- a/build_system/targets/versioning/versioning.py +++ b/build_system/targets/versioning/versioning.py @@ -82,6 +82,9 @@ def __init__(self): @cached_property def version(self) -> Version: + """ + The version that is stored in the file. + """ lines = self.lines if len(lines) != 1: @@ -90,6 +93,11 @@ def version(self) -> Version: return Version.parse(lines[0]) def update(self, version: Version): + """ + Updates the version that is stored in the file. + + :param version: The version to be stored + """ self.write_lines(str(version)) Log.info('Updated version to "%s"', str(version)) @@ -107,8 +115,14 @@ class DevelopmentVersionFile(TextFile): The file that stores the project's development version. """ + def __init__(self): + super().__init__('.version-dev') + @cached_property def development_version(self) -> int: + """ + The development version that is stored in the file. + """ lines = self.lines if len(lines) != 1: @@ -117,6 +131,11 @@ def development_version(self) -> int: return Version.parse_version_number(lines[0]) def update(self, development_version: int): + """ + Updates the development version that is stored in the file. + + :param development_version: The development version to be stored + """ self.write_lines(str(development_version)) Log.info('Updated development version to "%s"', str(development_version)) @@ -162,7 +181,7 @@ def increment_development_version(): Increments the development version. """ version_file = __get_development_version_file() - version_file.update(version_file.dev + 1) + version_file.update(version_file.development_version + 1) def reset_development_version(): diff --git a/build_system/util/files.py b/build_system/util/files.py index 276a9f297..fd66cbf4f 100644 --- a/build_system/util/files.py +++ b/build_system/util/files.py @@ -3,7 +3,6 @@ Provides classes for listing files and directories. """ -from abc import abstractmethod from functools import partial, reduce from glob import glob from os import path @@ -55,6 +54,37 @@ def filter_directory(filtered_names: Set[str], _: str, directory_name: str): return self.add_filters(*[partial(filter_directory, name) for name in names]) + def filter_by_substrings(self, + starts_with: Optional[str] = None, + not_starts_with: Optional[str] = None, + ends_with: Optional[str] = None, + not_ends_with: Optional[str] = None, + contains: Optional[str] = None, + not_contains: Optional[str] = None) -> 'DirectorySearch': + """ + Adds a filter that matches subdirectories based on whether their name contains specific substrings. + + :param starts_with: A substring, names must start with or None, if no restrictions should be imposed + :param not_starts_with: A substring, names must not start with or None, if no restrictions should be imposed + :param ends_with: A substring, names must end with or None, if no restrictions should be imposed + :param not_ends_with: A substring, names must not end with or None, if no restrictions should be imposed + :param contains: A substring, names must contain or None, if no restrictions should be imposed + :param not_contains: A substring, names must not contain or None, if no restrictions should be imposed + :return: The `DirectorySearch` itself + """ + + def filter_directory(start: Optional[str], not_start: Optional[str], end: Optional[str], not_end: Optional[str], + substring: Optional[str], not_substring: Optional[str], _: str, directory_name: str): + return (not start or directory_name.startswith(start)) \ + and (not not_start or not directory_name.startswith(not_start)) \ + and (not end or directory_name.endswith(end)) \ + and (not not_end or directory_name.endswith(not_end)) \ + and (not substring or directory_name.find(substring) >= 0) \ + and (not not_substring or directory_name.find(not_substring) < 0) + + return self.add_filters( + partial(filter_directory, starts_with, not_starts_with, ends_with, not_ends_with, contains, not_contains)) + def exclude(self, *excludes: Filter) -> 'DirectorySearch': """ Adds one or several filters that should be used for excluding subdirectories. @@ -86,8 +116,8 @@ def exclude_by_substrings(self, contains: Optional[str] = None, not_contains: Optional[str] = None) -> 'DirectorySearch': """ - Adds a filter that that should be used for excluding subdirectories based on whether their name contains - specific substrings. + Adds a filter that should be used for excluding subdirectories based on whether their name contains specific + substrings. :param starts_with: A substring, names must start with or None, if no restrictions should be imposed :param not_starts_with: A substring, names must not start with or None, if no restrictions should be imposed @@ -99,13 +129,13 @@ def exclude_by_substrings(self, """ def filter_directory(start: Optional[str], not_start: Optional[str], end: Optional[str], not_end: Optional[str], - substring: Optional[str], not_substring: Optional[str], _: str, file_name: str): - return (not start or file_name.startswith(start)) \ - and (not not_start or not file_name.startswith(not_start)) \ - and (not end or file_name.endswith(end)) \ - and (not not_end or file_name.endswith(not_end)) \ - and (not substring or file_name.find(substring) >= 0) \ - and (not not_substring or file_name.find(not_substring) < 0) + substring: Optional[str], not_substring: Optional[str], _: str, directory_name: str): + return (not start or directory_name.startswith(start)) \ + and (not not_start or not directory_name.startswith(not_start)) \ + and (not end or directory_name.endswith(end)) \ + and (not not_end or directory_name.endswith(not_end)) \ + and (not substring or directory_name.find(substring) >= 0) \ + and (not not_substring or directory_name.find(not_substring) < 0) return self.exclude( partial(filter_directory, starts_with, not_starts_with, ends_with, not_ends_with, contains, not_contains)) @@ -423,8 +453,10 @@ def __init__(self, """ self.name = name self.suffixes = suffixes - self.file_search_decorator = file_search_decorator if file_search_decorator else lambda file_search: file_search.filter_by_suffix( - *suffixes) + self.file_search_decorator = file_search_decorator + + if not self.file_search_decorator: + self.file_search_decorator = lambda file_search: file_search.filter_by_suffix(*suffixes) @staticmethod def python() -> 'FileType': From 3f4c0ed5f8e2579d285502ef7a67e0f0225d4018 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Fri, 13 Dec 2024 02:07:56 +0100 Subject: [PATCH 105/114] Format template files. --- doc/developer_guide/api/cpp/index.md.template | 5 +++-- doc/developer_guide/api/python/index.md.template | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/developer_guide/api/cpp/index.md.template b/doc/developer_guide/api/cpp/index.md.template index 72136801f..6be8b2d61 100644 --- a/doc/developer_guide/api/cpp/index.md.template +++ b/doc/developer_guide/api/cpp/index.md.template @@ -5,7 +5,8 @@ For those who are interested in modifying the project's source code or want to use it in their own C++ code, the API documentation of the following C++ libraries provides valuable insights into the classes, functions, etc., that are available (see {ref}`project-structure`): ```{toctree} -:maxdepth: 1 - +--- +maxdepth: 1 +--- %s ``` diff --git a/doc/developer_guide/api/python/index.md.template b/doc/developer_guide/api/python/index.md.template index f6781a709..4c15c6097 100644 --- a/doc/developer_guide/api/python/index.md.template +++ b/doc/developer_guide/api/python/index.md.template @@ -5,7 +5,8 @@ For those who want to use the algorithms provided by this project in their own Python code, we provide a documentation of all classes, functions, etc., that can be used. Currently, the project includes the following Python packages (see {ref}`project-structure`): ```{toctree} -:maxdepth: 1 - +--- +maxdepth: 1 +--- %s ``` From 6204aad6d6c302aadf5722af8ca474a7cc92fa4d Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Fri, 13 Dec 2024 02:26:37 +0100 Subject: [PATCH 106/114] Make filters for running workflow steps more fine-grained. --- .github/workflows/test_build.yml | 11 ++++++++++- .github/workflows/test_changelog.yml | 6 +++++- .github/workflows/test_doc.yml | 8 +++++++- .github/workflows/test_format.yml | 18 +++++++++++++----- .github/workflows/test_publish.yml | 6 +++++- 5 files changed, 40 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index 7bd3fd7db..be2f77372 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -35,9 +35,16 @@ jobs: - '.github/workflows/test_build.yml' - 'build' - 'build.bat' - - 'build_system/**' + - 'build_system/main.py' + - 'build_system/core/**' + - 'build_system/util/**' + - 'build_system/targets/paths.py' + - 'build_system/targets/compilation/*' + - 'build_system/targets/testing/*' cpp: &cpp - *build_files + - 'build_system/targets/compilation/cpp/*' + - 'build_system/targets/testing/cpp/*' - 'cpp/**/include/**' - 'cpp/**/src/**' - '**/*.pxd' @@ -48,6 +55,8 @@ jobs: - 'cpp/**/test/**' python: &python - *build_files + - 'build_system/targets/compilation/cython/*' + - 'build_system/targets/testing/python/*' - 'python/requirements.txt' - 'python/**/mlrl/**' python_tests: &python_tests diff --git a/.github/workflows/test_changelog.yml b/.github/workflows/test_changelog.yml index 869fe882c..92221b3b4 100644 --- a/.github/workflows/test_changelog.yml +++ b/.github/workflows/test_changelog.yml @@ -33,7 +33,11 @@ jobs: - '.github/workflows/test_changelog.yml' - 'build' - 'build.bat' - - 'build_system/**' + - 'build_system/main.py' + - 'build_system/core/**' + - 'build_system/util/**' + - 'build_system/targets/paths.py' + - 'build_system/targets/versioning/*' bugfix: - *build_files - '.changelog-bugfix.md' diff --git a/.github/workflows/test_doc.yml b/.github/workflows/test_doc.yml index 021a893e5..509159edf 100644 --- a/.github/workflows/test_doc.yml +++ b/.github/workflows/test_doc.yml @@ -34,15 +34,21 @@ jobs: - '.github/workflows/test_doc.yml' - 'build' - 'build.bat' - - 'build_system/**' + - 'build_system/main.py' + - 'build_system/core/**' + - 'build_system/util/**' + - 'build_system/targets/paths.py' cpp: &cpp - *build_files + - 'build_system/targets/documentation/cpp/*' - 'cpp/**/include/**' python: &python - *build_files + - 'build_system/targets/documentation/python/*' - 'python/**/mlrl/**' doc: &doc - *build_files + - 'build_system/targets/documentation/*' - 'doc/**' any: - *cpp diff --git a/.github/workflows/test_format.yml b/.github/workflows/test_format.yml index bae279776..1360870ee 100644 --- a/.github/workflows/test_format.yml +++ b/.github/workflows/test_format.yml @@ -33,23 +33,31 @@ jobs: - '.github/workflows/test_format.yml' - 'build' - 'build.bat' - - 'build_system/**' + - 'build_system/main.py' + - 'build_system/core/**' + - 'build_system/util/**' + - 'build_system/targets/paths.py' cpp: - *build_files + - 'build_system/targets/code_style/*' + - 'build_system/targets/code_style/cpp/*' + - '.cpplint.cfg' - '**/*.hpp' - '**/*.cpp' - - '.clang-format' python: - *build_files + - 'build_system/targets/code_style/*' + - 'build_system/targets/code_style/python/*' - '**/*.py' - - '.isort.cfg' - - '.pylintrc' - - '.style.yapf' md: - *build_files + - 'build_system/targets/code_style/*' + - 'build_system/targets/code_style/markdown/*' - '**/*.md' yaml: - *build_files + - 'build_system/targets/code_style/*' + - 'build_system/targets/code_style/yaml/*' - '**/*.y*ml' - name: Check C++ code style if: steps.filter.outputs.cpp == 'true' diff --git a/.github/workflows/test_publish.yml b/.github/workflows/test_publish.yml index 2689105e5..1dec1fbc8 100644 --- a/.github/workflows/test_publish.yml +++ b/.github/workflows/test_publish.yml @@ -36,7 +36,11 @@ jobs: - '.github/workflows/template_publish_pure.yml' - 'build' - 'build.bat' - - 'build_system/**' + - 'build_system/main.py' + - 'build_system/core/**' + - 'build_system/util/**' + - 'build_system/targets/paths.py' + - 'build_system/targets/packaging/*' - name: Read Python version uses: juliangruber/read-file-action@v1 id: python_version From a9861641b1f261d5968211b58d4aac0c2823e206 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Fri, 13 Dec 2024 02:04:31 +0100 Subject: [PATCH 107/114] Update changelog. --- .changelog-bugfix.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.changelog-bugfix.md b/.changelog-bugfix.md index 48f5d4c02..d846aebe2 100644 --- a/.changelog-bugfix.md +++ b/.changelog-bugfix.md @@ -1,5 +1,6 @@ # Quality-of-Life Improvements +- The build system now uses a lightweight custom implementation instead of [SCons](https://scons.org/) and is better modularized to avoid unnecessary runs of Continuous Integration jobs when only certain parts of it are modified. - Releases are now automated via Continuous Integration, including the update of the project's changelog. - The presentation of algorithmic parameters in the documentation has been improved. - Outdated GitHub Actions can now be printed via the build target `check_github_actions`. Alternatively, the build target `update_github_actions` may be used to update them automatically. From 291b16b536e4a574031418cfb1ea1b4fcbc2f5f5 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Fri, 13 Dec 2024 02:34:41 +0100 Subject: [PATCH 108/114] Automatically create build directory if it does not exist. --- build_system/core/changes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_system/core/changes.py b/build_system/core/changes.py index 528f0b70e..4590a8507 100644 --- a/build_system/core/changes.py +++ b/build_system/core/changes.py @@ -10,7 +10,7 @@ from typing import Dict, List, Set from core.modules import Module -from util.io import TextFile +from util.io import TextFile, create_directories class JsonFile(TextFile): @@ -66,6 +66,7 @@ def __init__(self, file: str): :param file: The path to the JSON file """ super().__init__(file, accept_missing=True) + create_directories(path.dirname(file)) def update(self, module_name: str, files: Set[str]): """ From 268120c9ac90a65be0e653f9c0589be52f3bd99d Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Fri, 13 Dec 2024 02:43:03 +0100 Subject: [PATCH 109/114] Fix search for test directories. --- build_system/targets/testing/python/modules.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_system/targets/testing/python/modules.py b/build_system/targets/testing/python/modules.py index 965f1ad3e..d3a55dc23 100644 --- a/build_system/targets/testing/python/modules.py +++ b/build_system/targets/testing/python/modules.py @@ -51,11 +51,12 @@ def find_test_directories(self) -> List[str]: :return: A list that contains the paths of the directories that have been found """ - return self.test_file_search \ + test_files = self.test_file_search \ .exclude_subdirectories_by_name(self.build_directory_name) \ .filter_by_substrings(starts_with='test_') \ .filter_by_file_type(FileType.python()) \ .list(self.root_directory) + return list({path.dirname(test_file) for test_file in test_files}) def __str__(self) -> str: return 'PythonTestModule {root_directory="' + self.root_directory + '"}' From c07ece2b07985b08d6f2dd759482603a363039df Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 14 Dec 2024 13:40:47 +0100 Subject: [PATCH 110/114] Apply "mdformat" to files with suffix ".md.template". --- build_system/util/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_system/util/files.py b/build_system/util/files.py index fd66cbf4f..662c752f2 100644 --- a/build_system/util/files.py +++ b/build_system/util/files.py @@ -492,7 +492,7 @@ def markdown() -> 'FileType': :return: The `FileType` that has been created """ - return FileType(name='Markdown', suffixes={'md'}) + return FileType(name='Markdown', suffixes={'md', 'md.template'}) @staticmethod def yaml() -> 'FileType': From 56b3f17952f542e9b33c65a9985ec73748c177c4 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 14 Dec 2024 13:47:38 +0100 Subject: [PATCH 111/114] Fix installation of external programs on Windows. --- build_system/util/cmd.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/build_system/util/cmd.py b/build_system/util/cmd.py index 539dce827..9f73c8d4a 100644 --- a/build_system/util/cmd.py +++ b/build_system/util/cmd.py @@ -45,6 +45,23 @@ class RunOptions: Allows to customize options for running command line programs. """ + @staticmethod + def __in_virtual_environment() -> bool: + return sys.prefix != sys.base_prefix + + def __get_executable(self, command: 'Command') -> str: + if self.__in_virtual_environment(): + # On Windows, we use the relative path to the command's executable within the virtual environment, if + # such an executable exists. This circumvents situations where the PATH environment variable has not + # been updated after activating the virtual environment. This can prevent the executables from being + # found or can lead to the wrong executable, from outside the virtual environment, being executed. + executable = path.join(sys.prefix, 'Scripts', command.command + '.exe') + + if path.isfile(executable): + return executable + + return command.command + def __init__(self): self.print_command = True self.exit_on_error = True @@ -61,7 +78,7 @@ def run(self, command: 'Command', capture_output: bool) -> CompletedProcess: if self.print_command: Log.info('Running external command "%s"...', command.print_options.format(command)) - output = subprocess.run([command.command] + command.arguments, + output = subprocess.run([self.__get_executable(command)] + command.arguments, check=False, text=capture_output, capture_output=capture_output, @@ -82,10 +99,6 @@ def run(self, command: 'Command', capture_output: bool) -> CompletedProcess: return output - @staticmethod - def __in_virtual_environment() -> bool: - return sys.prefix != sys.base_prefix - def __init__(self, command: str, *arguments: str, @@ -99,17 +112,6 @@ def __init__(self, program """ self.command = command - - if self.__in_virtual_environment(): - # On Windows, we use the relative path to the command's executable within the virtual environment, if such - # an executable exists. This circumvents situations where the PATH environment variable has not been updated - # after activating the virtual environment. This can prevent the executables from being found or can lead to - # the wrong executable, from outside the virtual environment, being executed. - executable = path.join(sys.prefix, 'Scripts', command + '.exe') - - if path.isfile(executable): - self.command = executable - self.arguments = list(arguments) self.print_options = print_options self.run_options = run_options From fe37092842306d712a972dfc463f714abdd78080 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 14 Dec 2024 16:17:13 +0100 Subject: [PATCH 112/114] Remove build directories when cleaning target "compile". --- build_system/targets/compilation/cpp/__init__.py | 2 +- build_system/targets/compilation/cython/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build_system/targets/compilation/cpp/__init__.py b/build_system/targets/compilation/cpp/__init__.py index c7d5f9a0a..6f931f796 100644 --- a/build_system/targets/compilation/cpp/__init__.py +++ b/build_system/targets/compilation/cpp/__init__.py @@ -23,7 +23,7 @@ .depends_on(VENV) \ .set_runnables(SetupCpp()) \ .add_phony_target(COMPILE_CPP) \ - .depends_on(SETUP_CPP) \ + .depends_on(SETUP_CPP, clean_dependencies=True) \ .set_runnables(CompileCpp()) \ .add_build_target(INSTALL_CPP) \ .depends_on(COMPILE_CPP) \ diff --git a/build_system/targets/compilation/cython/__init__.py b/build_system/targets/compilation/cython/__init__.py index be7145746..1d3bce4ae 100644 --- a/build_system/targets/compilation/cython/__init__.py +++ b/build_system/targets/compilation/cython/__init__.py @@ -23,7 +23,7 @@ .depends_on(COMPILE_CPP) \ .set_runnables(SetupCython()) \ .add_phony_target(COMPILE_CYTHON) \ - .depends_on(SETUP_CYTHON) \ + .depends_on(SETUP_CYTHON, clean_dependencies=True) \ .set_runnables(CompileCython()) \ .add_build_target(INSTALL_CYTHON) \ .depends_on(COMPILE_CYTHON) \ From 65f8dbab441f932af5f13434e8cc9c5937285fc3 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 14 Dec 2024 16:18:34 +0100 Subject: [PATCH 113/114] Specify build options when setting up meson for compiling Cython code. --- build_system/targets/compilation/cython/targets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_system/targets/compilation/cython/targets.py b/build_system/targets/compilation/cython/targets.py index a69f5bbe8..d981b7a7c 100644 --- a/build_system/targets/compilation/cython/targets.py +++ b/build_system/targets/compilation/cython/targets.py @@ -30,7 +30,7 @@ def __init__(self): super().__init__(MODULE_FILTER) def run(self, build_unit: BuildUnit, module: Module): - MesonSetup(build_unit, module) \ + MesonSetup(build_unit, module, build_options=BUILD_OPTIONS) \ .add_dependencies('cython') \ .run() @@ -52,7 +52,7 @@ def __init__(self): def run(self, build_unit: BuildUnit, module: Module): Log.info('Compiling Cython code in directory "%s"...', module.root_directory) - MesonConfigure(build_unit, module, build_options=BUILD_OPTIONS) + MesonConfigure(build_unit, module, build_options=BUILD_OPTIONS).run() MesonCompile(build_unit, module).run() From dbe5442e70712ba2285b7627a14f216841fb2855 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 14 Dec 2024 17:58:17 +0100 Subject: [PATCH 114/114] Clean all targets if the default target should be cleaned. --- build_system/core/targets.py | 56 ++++++++++++++++++++++++------------ build_system/main.py | 11 +++++-- 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/build_system/core/targets.py b/build_system/core/targets.py index 4446ef67f..51fe019e0 100644 --- a/build_system/core/targets.py +++ b/build_system/core/targets.py @@ -5,6 +5,7 @@ """ from abc import ABC, abstractmethod from dataclasses import dataclass +from enum import Enum, auto from functools import reduce from os import path from typing import Any, Callable, Dict, List, Optional, Tuple @@ -465,6 +466,14 @@ class DependencyGraph: A graph that determines the execution order of targets based on the dependencies between them. """ + class Type(Enum): + """ + All available types of dependency graphs. + """ + RUN = auto() + CLEAN = auto() + CLEAN_ALL = auto() + @dataclass class Node(ABC): """ @@ -480,34 +489,39 @@ class Node(ABC): child: Optional['DependencyGraph.Node'] = None @staticmethod - def from_name(targets_by_name: Dict[str, Target], target_name: str, clean: bool) -> 'DependencyGraph.Node': + def from_name(targets_by_name: Dict[str, Target], target_name: str, + graph_type: 'DependencyGraph.Type') -> 'DependencyGraph.Node': """ Creates and returns a new node of a dependency graph corresponding to the target with a specific name. :param targets_by_name: A dictionary that stores all available targets by their names :param target_name: The name of the target, the node should correspond to - :param clean: True, if the target should be cleaned, False otherwise + :param graph_type: The type of the dependency graph :return: The node that has been created """ target = targets_by_name[target_name] - return DependencyGraph.CleanNode(target) if clean else DependencyGraph.RunNode(target) + return DependencyGraph.RunNode( + target) if graph_type == DependencyGraph.Type.RUN else DependencyGraph.CleanNode(target) @staticmethod def from_dependency(targets_by_name: Dict[str, Target], dependency: Target.Dependency, - clean: bool) -> Optional['DependencyGraph.Node']: + graph_type: 'DependencyGraph.Type') -> Optional['DependencyGraph.Node']: """ Creates and returns a new node of a dependency graph corresponding to the target referred to by a `Target.Dependency`. :param targets_by_name: A dictionary that stores all available targets by their names :param dependency: The dependency referring to the target, the node should correspond to - :param clean: True, if the target should be cleaned, False otherwise + :param graph_type: The type of the dependency graph :return: The node that has been created or None, if the dependency does not require a node to be created """ - if not clean or dependency.clean_dependency: + if graph_type == DependencyGraph.Type.RUN \ + or graph_type == DependencyGraph.Type.CLEAN_ALL \ + or dependency.clean_dependency: target = targets_by_name[dependency.target_name] - return DependencyGraph.CleanNode(target) if clean else DependencyGraph.RunNode(target) + return DependencyGraph.RunNode( + target) if graph_type == DependencyGraph.Type.RUN else DependencyGraph.CleanNode(target) return None @@ -635,18 +649,19 @@ def __str__(self) -> str: return result @staticmethod - def __expand_sequence(targets_by_name: Dict[str, Target], sequence: Sequence, clean: bool) -> List[Sequence]: + def __expand_sequence(targets_by_name: Dict[str, Target], sequence: Sequence, + graph_type: 'DependencyGraph.Type') -> List[Sequence]: sequences = [] dependencies = sequence.first.target.dependencies if dependencies: for dependency in dependencies: - new_node = DependencyGraph.Node.from_dependency(targets_by_name, dependency, clean=clean) + new_node = DependencyGraph.Node.from_dependency(targets_by_name, dependency, graph_type) if new_node: new_sequence = sequence.copy() new_sequence.prepend(new_node) - sequences.extend(DependencyGraph.__expand_sequence(targets_by_name, new_sequence, clean=clean)) + sequences.extend(DependencyGraph.__expand_sequence(targets_by_name, new_sequence, graph_type)) else: sequences.append(sequence) else: @@ -655,10 +670,11 @@ def __expand_sequence(targets_by_name: Dict[str, Target], sequence: Sequence, cl return sequences @staticmethod - def __create_sequence(targets_by_name: Dict[str, Target], target_name: str, clean: bool) -> List[Sequence]: - node = DependencyGraph.Node.from_name(targets_by_name, target_name, clean=clean) + def __create_sequence(targets_by_name: Dict[str, Target], target_name: str, + graph_type: 'DependencyGraph.Type') -> List[Sequence]: + node = DependencyGraph.Node.from_name(targets_by_name, target_name, graph_type) sequence = DependencyGraph.Sequence.from_node(node) - return DependencyGraph.__expand_sequence(targets_by_name, sequence, clean=clean) + return DependencyGraph.__expand_sequence(targets_by_name, sequence, graph_type) @staticmethod def __find_in_parents(node: Node, parent: Optional[Node]) -> Optional[Node]: @@ -711,14 +727,14 @@ def __merge_multiple_sequences(sequences: List[Sequence]) -> Sequence: return sequences[0] - def __init__(self, targets_by_name: Dict[str, Target], *target_names: str, clean: bool): + def __init__(self, targets_by_name: Dict[str, Target], *target_names: str, graph_type: 'DependencyGraph.Type'): """ :param targets_by_name: A dictionary that stores all available targets by their names :param target_names: The names of the targets to be included in the graph - :param clean: True, if the targets should be cleaned, False otherwise + :param graph_type: The type of the dependency graph """ self.sequence = self.__merge_multiple_sequences( - reduce(lambda aggr, target_name: aggr + self.__create_sequence(targets_by_name, target_name, clean=clean), + reduce(lambda aggr, target_name: aggr + self.__create_sequence(targets_by_name, target_name, graph_type), target_names, [])) def execute(self, module_registry: ModuleRegistry): @@ -755,12 +771,14 @@ def register(self, target: Target): self.targets_by_name[target.name] = target - def create_dependency_graph(self, *target_names: str, clean: bool = False) -> DependencyGraph: + def create_dependency_graph(self, + *target_names: str, + graph_type: 'DependencyGraph.Type' = DependencyGraph.Type.RUN) -> DependencyGraph: """ Creates and returns a `DependencyGraph` for each of the given targets. :param target_names: The names of the targets for which graphs should be created - :param clean: True, if the targets should be cleaned, False otherwise + :param graph_type: The type of the dependency graph :return: A list that contains the graphs that have been created """ if not target_names: @@ -771,4 +789,4 @@ def create_dependency_graph(self, *target_names: str, clean: bool = False) -> De if invalid_targets: Log.error('The following targets are invalid: %s', format_iterable(invalid_targets)) - return DependencyGraph(self.targets_by_name, *target_names, clean=clean) + return DependencyGraph(self.targets_by_name, *target_names, graph_type=graph_type) diff --git a/build_system/main.py b/build_system/main.py index 6978aa2e1..362498b0e 100644 --- a/build_system/main.py +++ b/build_system/main.py @@ -120,11 +120,18 @@ def __find_default_target(init_files: List[str]) -> Optional[str]: def __create_dependency_graph(target_registry: TargetRegistry, args, default_target: Optional[str]) -> DependencyGraph: targets = args.targets - targets = targets if targets else ([default_target] if default_target else []) clean = args.clean + graph_type = DependencyGraph.Type.CLEAN if clean else DependencyGraph.Type.RUN + + if not targets and default_target: + targets = [default_target] + + if clean: + graph_type = DependencyGraph.Type.CLEAN_ALL + Log.verbose('Creating dependency graph for %s targets [%s]...', 'cleaning' if clean else 'running', format_iterable(targets)) - dependency_graph = target_registry.create_dependency_graph(*targets, clean=clean) + dependency_graph = target_registry.create_dependency_graph(*targets, graph_type=graph_type) Log.verbose('Successfully created dependency graph:\n\n%s\n', str(dependency_graph)) return dependency_graph