diff --git a/.changelog-bugfix.md b/.changelog-bugfix.md index 184b179e7..050566532 100644 --- a/.changelog-bugfix.md +++ b/.changelog-bugfix.md @@ -4,5 +4,6 @@ - Releases are now automated via Continuous Integration, including the update of the project's changelog. - The presentation of algorithmic parameters in the documentation has been improved. - Outdated GitHub Actions can now be printed via the build target `check_github_actions`. Alternatively, the build target `update_github_actions` may be used to update them automatically. +- The build target `update_dependencies` can now be used to update Python dependencies. The build targets `update_build_dependencies` and `update_runtime_dependencies` only update build-time and runtime dependencies, respectively. - Continuous Integration is now used to automatically update outdated GitHub Actions on a regular schedule. - Continuous Integration is now used to periodically update the Doxygen configuration file used for generating API documentations for C++ code. diff --git a/build_system/targets/dependencies/python/__init__.py b/build_system/targets/dependencies/python/__init__.py index bc8ce0bc3..b452eeeb9 100644 --- a/build_system/targets/dependencies/python/__init__.py +++ b/build_system/targets/dependencies/python/__init__.py @@ -7,14 +7,20 @@ from core.targets import PhonyTarget, TargetBuilder from targets.dependencies.python.modules import DependencyType, PythonDependencyModule -from targets.dependencies.python.targets import CheckPythonDependencies, InstallRuntimeDependencies +from targets.dependencies.python.targets import CheckPythonDependencies, InstallRuntimeDependencies, \ + UpdatePythonDependencies from targets.paths import Project VENV = 'venv' TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \ .add_phony_target(VENV).set_runnables(InstallRuntimeDependencies()) \ + .add_phony_target('check_runtime_dependencies').set_runnables(CheckPythonDependencies(DependencyType.RUNTIME)) \ + .add_phony_target('check_build_dependencies').set_runnables(CheckPythonDependencies(DependencyType.BUILD_TIME)) \ .add_phony_target('check_dependencies').set_runnables(CheckPythonDependencies()) \ + .add_phony_target('update_runtime_dependencies').set_runnables(UpdatePythonDependencies(DependencyType.RUNTIME)) \ + .add_phony_target('update_build_dependencies').set_runnables(UpdatePythonDependencies(DependencyType.BUILD_TIME)) \ + .add_phony_target('update_dependencies').set_runnables(UpdatePythonDependencies()) \ .build() MODULES = [ diff --git a/build_system/targets/dependencies/python/pip.py b/build_system/targets/dependencies/python/pip.py index 0d1a2db6d..bc319ae95 100644 --- a/build_system/targets/dependencies/python/pip.py +++ b/build_system/targets/dependencies/python/pip.py @@ -3,29 +3,33 @@ Provides classes for listing installed Python dependencies via pip. """ -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import Set -from util.pip import Package, Pip, Requirement +from util.pip import Package, Pip, RequirementsFile, RequirementVersion @dataclass class Dependency: """ - Provides information about a dependency. + Provides information about an outdated dependency. Attributes: - installed: The version of the dependency that is currently installed - latest: The latest version of the dependency + requirements_file: The path to the requirements file that defines the dependency + package: The package, the dependency corresponds to + outdated: The outdated version of the dependency + latest: The latest version of the dependency """ - installed: Requirement - latest: Requirement + requirements_file: str + package: Package + outdated: RequirementVersion + latest: RequirementVersion def __eq__(self, other: 'Dependency') -> bool: - return self.installed == other.installed + return self.requirements_file == other.requirements_file and self.package == other.package def __hash__(self) -> int: - return hash(self.installed) + return hash((self.requirements_file, self.package)) class PipList(Pip): @@ -55,6 +59,8 @@ def install_all_packages(self): def list_outdated_dependencies(self) -> Set[Dependency]: """ Returns all outdated Python dependencies that are currently installed. + + :return: A set that contains all outdated dependencies """ stdout = PipList.ListCommand(outdated=True).print_command(False).capture_output() stdout_lines = stdout.strip().split('\n') @@ -77,13 +83,57 @@ def list_outdated_dependencies(self) -> Set[Dependency]: + line) package = Package(parts[0]) - requirement = self.requirements.lookup_requirement(package, accept_missing=True) - - if requirement and requirement.version: - installed_version = parts[1] - latest_version = parts[2] - outdated_dependencies.add( - Dependency(installed=Requirement(package, version=installed_version), - latest=Requirement(package, version=latest_version))) + requirements_by_file = self.requirements.lookup_requirement_by_file(package, accept_missing=True) + + for requirements_file, requirement in requirements_by_file.items(): + if requirement.version: + installed_version = parts[1] + latest_version = parts[2] + outdated_dependencies.add( + Dependency(requirements_file=requirements_file, + package=package, + outdated=RequirementVersion.parse(installed_version), + latest=RequirementVersion.parse(latest_version))) return outdated_dependencies + + def update_outdated_dependencies(self) -> Set[Dependency]: + """ + Updates all outdated Python dependencies that are currently installed. + + :return: A set that contains all dependencies that have been updated + """ + updated_dependencies = set() + separator = '.' + + for outdated_dependency in self.list_outdated_dependencies(): + latest_version = outdated_dependency.latest + latest_version_parts = [int(part) for part in latest_version.min_version.split(separator)] + requirements_file = RequirementsFile(outdated_dependency.requirements_file) + package = outdated_dependency.package + outdated_requirement = requirements_file.lookup_requirement(package) + outdated_version = outdated_requirement.version + updated_version = latest_version + + if outdated_version.is_range(): + min_version_parts = [int(part) for part in outdated_version.min_version.split(separator)] + max_version_parts = [int(part) for part in outdated_version.max_version.split(separator)] + num_version_numbers = min(len(min_version_parts), len(max_version_parts), len(latest_version_parts)) + + for i in range(num_version_numbers): + min_version_parts[i] = latest_version_parts[i] + max_version_parts[i] = latest_version_parts[i] + + max_version_parts[num_version_numbers - 1] += 1 + updated_version = RequirementVersion( + min_version=separator.join([str(part) for part in min_version_parts[:num_version_numbers]]), + max_version=separator.join([str(part) for part in max_version_parts[:num_version_numbers]])) + + requirements_file.update(replace(outdated_requirement, version=updated_version)) + updated_dependencies.add( + Dependency(requirements_file=outdated_dependency.requirements_file, + package=package, + outdated=outdated_version, + latest=updated_version)) + + return updated_dependencies diff --git a/build_system/targets/dependencies/python/targets.py b/build_system/targets/dependencies/python/targets.py index bdb7b0d75..06c5312d5 100644 --- a/build_system/targets/dependencies/python/targets.py +++ b/build_system/targets/dependencies/python/targets.py @@ -4,7 +4,7 @@ Implements targets for installing runtime requirements that are required by the project's source code. """ from functools import reduce -from typing import List +from typing import List, Optional from core.build_unit import BuildUnit from core.modules import Module @@ -34,25 +34,69 @@ class CheckPythonDependencies(PhonyTarget.Runnable): Installs all Python dependencies used by the project and checks for outdated ones. """ - def __init__(self): - super().__init__(PythonDependencyModule.Filter()) + def __init__(self, dependency_type: Optional[DependencyType] = None): + """ + :param dependency_type: The type of the Python dependencies to be checked or None, if all dependencies should be + updated + """ + super().__init__( + PythonDependencyModule.Filter(dependency_type) if dependency_type else PythonDependencyModule.Filter()) + self.dependency_type = dependency_type def run_all(self, build_unit: BuildUnit, modules: List[Module]): requirements_files = reduce(lambda aggr, module: aggr + module.find_requirements_files(), modules, []) pip = PipList(*requirements_files) - Log.info('Installing all dependencies...') + Log.info('Installing %s dependencies...', + ('all build-time' if self.dependency_type == DependencyType.BUILD_TIME else 'all runtime') + if self.dependency_type else 'all') pip.install_all_packages() Log.info('Checking for outdated dependencies...') outdated_dependencies = pip.list_outdated_dependencies() if outdated_dependencies: - table = Table(build_unit, 'Dependency', 'Installed version', 'Latest version') + table = Table(build_unit, 'Dependency', 'Requirements file', 'Installed version', 'Latest version') for outdated_dependency in outdated_dependencies: - table.add_row(str(outdated_dependency.installed.package), outdated_dependency.installed.version, - outdated_dependency.latest.version) + table.add_row(str(outdated_dependency.package), outdated_dependency.requirements_file, + outdated_dependency.outdated.min_version, outdated_dependency.latest.min_version) table.sort_rows(0, 1) Log.info('The following dependencies are outdated:\n\n%s', str(table)) else: Log.info('All dependencies are up-to-date!') + + +class UpdatePythonDependencies(PhonyTarget.Runnable): + """ + Installs all Python dependencies of a specific type and updates outdated ones. + """ + + def __init__(self, dependency_type: Optional[DependencyType] = None): + """ + :param dependency_type: The type of the Python dependencies to be updated or None, if all dependencies should be + updated + """ + super().__init__( + PythonDependencyModule.Filter(dependency_type) if dependency_type else PythonDependencyModule.Filter()) + self.dependency_type = dependency_type + + def run_all(self, build_unit: BuildUnit, modules: List[Module]): + requirements_files = reduce(lambda aggr, module: aggr + module.find_requirements_files(), modules, []) + pip = PipList(*requirements_files) + Log.info('Installing %s dependencies...', + ('all build-time' if self.dependency_type == DependencyType.BUILD_TIME else 'all runtime') + if self.dependency_type else 'all') + pip.install_all_packages() + Log.info('Updating outdated dependencies...') + updated_dependencies = pip.update_outdated_dependencies() + + if updated_dependencies: + table = Table(build_unit, 'Dependency', 'Requirements file', 'Previous version', 'Updated version') + for updated_dependency in updated_dependencies: + table.add_row(str(updated_dependency.package), updated_dependency.requirements_file, + str(updated_dependency.outdated), str(updated_dependency.latest)) + + table.sort_rows(0, 1) + Log.info('The following dependencies have been updated:\n\n%s', str(table)) + else: + Log.info('No dependencies must be updated!') diff --git a/build_system/util/pip.py b/build_system/util/pip.py index 72063c889..955119061 100644 --- a/build_system/util/pip.py +++ b/build_system/util/pip.py @@ -41,6 +41,76 @@ def __hash__(self) -> int: return hash(self.normalized_name) +@dataclass +class RequirementVersion: + """ + Specifies the version of a requirement. + + Attributes: + min_version: The maximum version number + max_version: The minimum version number + """ + min_version: str + max_version: str + + PREFIX_EQ = '==' + + PREFIX_GEQ = '>=' + + PREFIX_LE = '<' + + @staticmethod + def parse(version: str) -> 'RequirementVersion': + """ + Parses and returns the version of a requirement. + + :param version: The version to be parsed + :return: The version that has been parsed + """ + parts = version.strip().split(',') + + if len(parts) > 2: + raise ValueError( + 'Version of requirement must consist of one or two version numbers, separated by comma, but got: ' + + version) + + first_part = parts[0].strip() + + if len(parts) > 1: + if not first_part.startswith(RequirementVersion.PREFIX_GEQ): + raise ValueError('First version number of requirement must start with "' + RequirementVersion.PREFIX_GEQ + + '", but got: ' + version) + + second_part = parts[1].strip() + + if not second_part.startswith(RequirementVersion.PREFIX_LE): + raise ValueError('Second version number of requirement must start with "' + RequirementVersion.PREFIX_LE + + '", but got: ' + version) + + return RequirementVersion(min_version=first_part[len(RequirementVersion.PREFIX_GEQ):].strip(), + max_version=second_part[len(RequirementVersion.PREFIX_LE):].strip()) + + version_number = first_part + + if version_number.startswith(RequirementVersion.PREFIX_EQ): + version_number = version_number[len(RequirementVersion.PREFIX_EQ):].strip() + + return RequirementVersion(min_version=version_number, max_version=version_number) + + def is_range(self) -> bool: + """ + Returns whether the version specifies a range of version numbers or not. + + :return: True, if the version specifies a range of version numbers, False otherwise + """ + return self.min_version != self.max_version + + def __str__(self) -> str: + if self.is_range(): + return self.PREFIX_GEQ + ' ' + self.min_version + ', ' + self.PREFIX_LE + ' ' + self.max_version + return self.PREFIX_EQ + ' ' + self.min_version + + @dataclass class Requirement: """ @@ -51,7 +121,7 @@ class Requirement: version: The version of the package or None, if no version is specified """ package: Package - version: Optional[str] = None + version: Optional[RequirementVersion] = None @staticmethod def parse(requirement: str) -> 'Requirement': @@ -63,11 +133,11 @@ def parse(requirement: str) -> 'Requirement': """ parts = requirement.split() package = Package(name=parts[0].strip()) - version = ' '.join(parts[1:]).strip() if len(parts) > 1 else None + version = RequirementVersion.parse(' '.join(parts[1:])) if len(parts) > 1 else None return Requirement(package, version) def __str__(self) -> str: - return str(self.package) + (self.version if self.version else '') + return str(self.package) + (' ' + str(self.version) if self.version else '') def __eq__(self, other: 'Requirement') -> bool: return self.package == other.package @@ -138,9 +208,30 @@ class RequirementsFile(TextFile, Requirements): def requirements_by_package(self) -> Dict[Package, Requirement]: return { requirement.package: requirement - for requirement in [Requirement.parse(line) for line in self.lines if line.strip('\n').strip()] + for requirement in + [Requirement.parse(line.strip('\n').strip()) for line in self.lines if line.strip('\n').strip()] } + def update(self, updated_requirement: Requirement): + """ + Updates a given requirement, if it is included in the requirements file. + + :param updated_requirement: The requirement to be updated + """ + new_lines = [] + + for line in self.lines: + new_lines.append(line) + line_stripped = line.strip('\n').strip() + + if line_stripped: + requirement = Requirement.parse(line_stripped) + + if requirement.package == updated_requirement.package: + new_lines[-1] = str(updated_requirement) + '\n' + + self.write_lines(*new_lines) + class RequirementsFiles(Requirements): """ @@ -155,6 +246,61 @@ def requirements_by_package(self) -> Dict[Package, Requirement]: return reduce(lambda aggr, requirements_file: aggr | requirements_file.requirements_by_package, self.requirements_files, {}) + def lookup_requirements_by_file(self, + *packages: Package, + accept_missing: bool = False) -> Dict[str, Set[Requirement]]: + """ + Looks up the requirements for given packages in the requirements files. + + :param packages: The packages that should be looked up + :param accept_missing: False, if an error should be raised if a package is not listed in any requirements file, + True, if it should simply be ignored + :return: A dictionary that contains the paths to requirements files, as well as their + requirements for the given packages + """ + requirements_by_file = {} + + for package in packages: + found = False + + for requirements_file in self.requirements_files: + requirement = requirements_file.requirements_by_package.get(package) + + if requirement: + requirements = requirements_by_file.setdefault(requirements_file.file, set()) + requirements.add(requirement) + found = True + + if not found and not accept_missing: + raise RuntimeError('Requirement for package "' + str(package) + '" not found') + + return requirements_by_file + + def lookup_requirement_by_file(self, package: Package, accept_missing: bool = False) -> Dict[str, Requirement]: + """ + Looks up the requirement for a given package in the requirements files. + + :param package: The package that should be looked up + :param accept_missing: False, if an error should be raised if the package is not listed any requirements file, + True, if it should simply be ignored + :return: A dictionary that contains the paths to requirements files, as well as their requirement + for the given package + """ + requirements_by_file = self.lookup_requirements_by_file(package, accept_missing=accept_missing) + return { + requirements_file: requirements.pop() + for requirements_file, requirements in requirements_by_file.items() if requirements + } + + def update(self, updated_requirement: Requirement): + """ + Updates a given requirement, if it is included in one of the requirements files. + + :param updated_requirement: The requirement to be updated + """ + for requirements_file in self.requirements_files: + requirements_file.update(updated_requirement) + class Pip: """ diff --git a/doc/developer_guide/coding_standards.md b/doc/developer_guide/coding_standards.md index 64badb025..e690acdb9 100644 --- a/doc/developer_guide/coding_standards.md +++ b/doc/developer_guide/coding_standards.md @@ -206,6 +206,30 @@ To ease the life of developers, the following command provided by the project's ``` ```` +Alternatively, the following command may be used to update the versions of outdated dependencies automatically: + +````{tab} Linux + ```text + ./build update_dependencies + ``` +```` + +````{tab} macOS + ```text + ./build update_dependencies + ``` +```` + +````{tab} Windows + ``` + build.bat update_dependencies + ``` +```` + +```{note} +If you want to restrict the above commands to the build-time dependencies, required by the project's build system, or the runtime dependencies, required for running its algorithms, you can use the targets `check_build_dependencies`, `check_runtime_dependencies`, `update_build_dependencies`, and `update_runtime_dependencies` instead. +``` + ### GitHub Actions Our {ref}`Continuous Integration ` (CI) jobs heavily rely on so-called [Actions](https://github.com/marketplace?type=actions), which are reusable building blocks provided by third-party developers. As with all dependencies, updates to these Actions may introduce breaking changes. To reduce the risk of updates breaking our CI jobs, we pin the Actions to a certain version. Usually, we only restrict the major version required by a job, rather than specifying a specific version. This allows minor updates, which are less likely to cause problems, to take effect without manual intervention.