From 5276cbd575c62ed779d2f0fbea71581978e1048c Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sun, 17 Nov 2024 14:59:08 +0100 Subject: [PATCH 1/6] Prevent redundant querying of the latest version of Github Actions. --- scons/github_actions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scons/github_actions.py b/scons/github_actions.py index 2fd0473a5..c77f72fe8 100644 --- a/scons/github_actions.py +++ b/scons/github_actions.py @@ -196,12 +196,12 @@ def __determine_latest_action_versions(*workflows: Workflow) -> Set[Workflow]: for workflow in workflows: for action in workflow.actions: - latest_version = version_cache.get(action) + latest_version = version_cache.get(action.name) if not latest_version: print('Checking version of GitHub Action "' + action.name + '"...') latest_version = __query_latest_action_version(action, github_token=github_token) - version_cache[action] = latest_version + version_cache[action.name] = latest_version action.latest_version = latest_version From 2c0b78c778fe4da847999a89ddf0fd2488319070 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sun, 17 Nov 2024 15:04:34 +0100 Subject: [PATCH 2/6] Add build target for updating Github Actions. --- scons/github_actions.py | 7 +++++++ scons/sconstruct.py | 24 +++++++++++++----------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/scons/github_actions.py b/scons/github_actions.py index c77f72fe8..67e76b6f7 100644 --- a/scons/github_actions.py +++ b/scons/github_actions.py @@ -234,3 +234,10 @@ def check_github_actions(**_): workflow_files = __get_github_workflow_files(workflow_directory) workflows = __determine_latest_action_versions(*__parse_workflows(*workflow_files)) __print_outdated_actions(*workflows) + + +def update_github_actions(**_): + """ + Updates the versions of outdated GitHub Actions in the project's workflows. + """ + print('Updating versions of outdated GitHub Actions...') diff --git a/scons/sconstruct.py b/scons/sconstruct.py index d51e51d9a..b8b6b8ecf 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -15,7 +15,7 @@ from compilation import compile_cpp, compile_cython, install_cpp, install_cython, setup_cpp, setup_cython from dependencies import check_dependency_versions, install_runtime_dependencies from documentation import apidoc_cpp, apidoc_cpp_tocfile, apidoc_python, apidoc_python_tocfile, doc -from github_actions import check_github_actions +from github_actions import check_github_actions, update_github_actions from modules import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE from packaging import build_python_wheel, install_python_wheels from testing import tests_cpp, tests_python @@ -60,8 +60,9 @@ def __print_if_clean(environment, message: str): TARGET_NAME_FORMAT_CPP = TARGET_NAME_FORMAT + '_cpp' TARGET_NAME_FORMAT_MD = TARGET_NAME_FORMAT + '_md' TARGET_NAME_FORMAT_YAML = TARGET_NAME_FORMAT + '_yaml' -TARGET_NAME_CHECK_DEPENDENCIES = 'check_dependencies' -TARGET_NAME_CHECK_GITHUB_ACTIONS = 'check_github_actions' +TARGET_NAME_DEPENDENCIES_CHECK = 'check_dependencies' +TARGET_NAME_GITHUB_ACTIONS_CHECK = 'check_github_actions' +TARGET_NAME_GITHUB_ACTIONS_UPDATE = 'update_github_actions' TARGET_NAME_VENV = 'venv' TARGET_NAME_COMPILE = 'compile' TARGET_NAME_COMPILE_CPP = TARGET_NAME_COMPILE + '_cpp' @@ -87,11 +88,11 @@ def __print_if_clean(environment, message: str): TARGET_NAME_UPDATE_CHANGELOG_MAIN, TARGET_NAME_PRINT_VERSION, TARGET_NAME_PRINT_LATEST_CHANGELOG, TARGET_NAME_TEST_FORMAT, TARGET_NAME_TEST_FORMAT_PYTHON, TARGET_NAME_TEST_FORMAT_CPP, TARGET_NAME_TEST_FORMAT_MD, TARGET_NAME_TEST_FORMAT_YAML, TARGET_NAME_FORMAT, TARGET_NAME_FORMAT_PYTHON, TARGET_NAME_FORMAT_CPP, - TARGET_NAME_FORMAT_MD, TARGET_NAME_FORMAT_YAML, TARGET_NAME_CHECK_DEPENDENCIES, TARGET_NAME_CHECK_GITHUB_ACTIONS, - TARGET_NAME_VENV, TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP, TARGET_NAME_COMPILE_CYTHON, TARGET_NAME_INSTALL, - TARGET_NAME_INSTALL_CPP, TARGET_NAME_INSTALL_CYTHON, TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS, - TARGET_NAME_TESTS, TARGET_NAME_TESTS_CPP, TARGET_NAME_TESTS_PYTHON, TARGET_NAME_APIDOC, TARGET_NAME_APIDOC_CPP, - TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC + TARGET_NAME_FORMAT_MD, TARGET_NAME_FORMAT_YAML, TARGET_NAME_DEPENDENCIES_CHECK, TARGET_NAME_GITHUB_ACTIONS_CHECK, + TARGET_NAME_GITHUB_ACTIONS_UPDATE, TARGET_NAME_VENV, TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP, + TARGET_NAME_COMPILE_CYTHON, TARGET_NAME_INSTALL, TARGET_NAME_INSTALL_CPP, TARGET_NAME_INSTALL_CYTHON, + TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS, TARGET_NAME_TESTS, TARGET_NAME_TESTS_CPP, + TARGET_NAME_TESTS_PYTHON, TARGET_NAME_APIDOC, TARGET_NAME_APIDOC_CPP, TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC } DEFAULT_TARGET = TARGET_NAME_INSTALL_WHEELS @@ -148,10 +149,11 @@ def __print_if_clean(environment, message: str): env.Depends(target_format, [target_format_python, target_format_cpp, target_format_md, target_format_yaml]) # Define target for checking dependency versions... -target_check_dependencies = __create_phony_target(env, TARGET_NAME_CHECK_DEPENDENCIES, action=check_dependency_versions) +__create_phony_target(env, TARGET_NAME_DEPENDENCIES_CHECK, action=check_dependency_versions) -# Define target for checking the versions of GitHub Actions... -target_check_github_actions = __create_phony_target(env, TARGET_NAME_CHECK_GITHUB_ACTIONS, action=check_github_actions) +# Define target for checking and updating the versions of GitHub Actions... +__create_phony_target(env, TARGET_NAME_GITHUB_ACTIONS_CHECK, action=check_github_actions) +__create_phony_target(env, TARGET_NAME_GITHUB_ACTIONS_UPDATE, action=update_github_actions) # Define target for installing runtime dependencies... target_venv = __create_phony_target(env, TARGET_NAME_VENV, action=install_runtime_dependencies) From 040b81c44371f639ba271365839f8dbb0211423c Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sun, 17 Nov 2024 17:32:32 +0100 Subject: [PATCH 3/6] Implement updating of Github Actions. --- scons/github_actions.py | 223 ++++++++++++++++++++++++++++++---------- 1 file changed, 167 insertions(+), 56 deletions(-) diff --git a/scons/github_actions.py b/scons/github_actions.py index 67e76b6f7..d2bd3a981 100644 --- a/scons/github_actions.py +++ b/scons/github_actions.py @@ -6,6 +6,7 @@ import sys from dataclasses import dataclass, field +from functools import reduce from glob import glob from os import environ, path from typing import List, Optional, Set @@ -15,11 +16,7 @@ ENV_GITHUB_TOKEN = 'GITHUB_TOKEN' -SEPARATOR_VERSION = '@' - -SEPARATOR_VERSION_NUMBER = '.' - -SEPARATOR_PATH = '/' +WORKFLOW_ENCODING = 'utf-8' @dataclass @@ -32,20 +29,39 @@ class ActionVersion: """ version: str + SEPARATOR = '.' + + @staticmethod + def from_version_numbers(*version_numbers: int) -> 'ActionVersion': + """ + Creates and returns the version of a GitHub Action from one or several version numbers. + + :param version_numbers: The version numbers + :return: The version that has been created + """ + return ActionVersion(ActionVersion.SEPARATOR.join([str(version_number) for version_number in version_numbers])) + + @property + def version_numbers(self) -> List[int]: + """ + A list that stores the individual version numbers, the full version consists of. + """ + return [int(version_number) for version_number in str(self).split(self.SEPARATOR)] + def __str__(self) -> str: return self.version.lstrip('v') def __lt__(self, other: 'ActionVersion') -> bool: - first_numbers = str(self).split(SEPARATOR_VERSION_NUMBER) - second_numbers = str(other).split(SEPARATOR_VERSION_NUMBER) + first_version_numbers = self.version_numbers + second_version_numbers = other.version_numbers - for i in range(min(len(first_numbers), len(second_numbers))): - first = int(first_numbers[i]) - second = int(second_numbers[i]) + for i in range(min(len(first_version_numbers), len(second_version_numbers))): + first_version_number = first_version_numbers[i] + second_version_number = second_version_numbers[i] - if first > second: + if first_version_number > second_version_number: return False - if first < second: + if first_version_number < second_version_number: return True return False @@ -65,18 +81,21 @@ class Action: version: ActionVersion latest_version: Optional[ActionVersion] = None + SEPARATOR = '@' + @staticmethod - def parse(uses: str) -> 'Action': + def from_uses_clause(uses_clause: str) -> 'Action': """ - Parses and returns a GitHub Action as specified via the uses-clause of a workflow. + Creates and returns a GitHub Action from the uses-clause of a workflow. - :param uses: The uses-clause - :return: The GitHub Action + :param uses_clause: The uses-clause + :return: The GitHub Action that has been created """ - parts = uses.split(SEPARATOR_VERSION) + parts = uses_clause.split(Action.SEPARATOR) if len(parts) != 2: - raise ValueError('Action must contain the symbol + "' + SEPARATOR_VERSION + '", but got "' + uses + '"') + raise ValueError('Uses-clause must contain the symbol + "' + Action.SEPARATOR + '", but got "' + uses_clause + + '"') return Action(name=parts[0], version=ActionVersion(parts[1])) @@ -86,19 +105,19 @@ def repository(self) -> str: The name of the repository, where the GitHub Action is hosted. """ repository = self.name - parts = repository.split(SEPARATOR_PATH) - return SEPARATOR_PATH.join(parts[:2]) if len(parts) > 2 else repository + separator = '/' + parts = repository.split(separator) + return separator.join(parts[:2]) if len(parts) > 2 else repository + @property def is_outdated(self) -> bool: """ - Returns whether the GitHub Action is known to be outdated or not. - - :return: True, if the GitHub Action is outdated, False otherwise + True, if the GitHub Action is known to be outdated, False otherwise. """ return self.latest_version and self.version < self.latest_version def __str__(self) -> str: - return self.name + SEPARATOR_VERSION + str(self.version) + return self.name + self.SEPARATOR + str(self.version) def __eq__(self, other: 'Action') -> bool: return str(self) == str(other) @@ -114,11 +133,38 @@ class Workflow: Attributes: workflow_file: The path of the workflow definition file - actions: A set that stores the Actions in the workflow + yaml_dict: A dictionary that stores the YAML structure of the workflow definition file + actions: A set that stores all Actions in the workflow """ workflow_file: str + yaml_dict: dict actions: Set[Action] = field(default_factory=set) + TAG_USES = 'uses' + + @property + def uses_clauses(self) -> List[str]: + """ + A list that contains all uses-clauses in the workflow. + """ + uses_clauses = [] + + for job in self.yaml_dict.get('jobs', {}).values(): + for step in job.get('steps', []): + uses_clause = step.get(self.TAG_USES, None) + + if uses_clause: + uses_clauses.append(uses_clause) + + return uses_clauses + + @property + def outdated_actions(self) -> Set[Action]: + """ + A set that stores all Actions in the workflow that are known to be outdated. + """ + return {action for action in self.actions if action.is_outdated} + def __eq__(self, other: 'Workflow') -> bool: return self.workflow_file == other.workflow_file @@ -126,34 +172,56 @@ def __hash__(self): return hash(self.workflow_file) -def __get_github_workflow_files(directory: str) -> List[str]: - return glob(path.join(directory, '*.y*ml')) - - -def __load_yaml(workflow_file: str) -> dict: +def __read_workflow(workflow_file: str) -> Workflow: install_build_dependencies('pyyaml') # pylint: disable=import-outside-toplevel import yaml - with open(workflow_file, encoding='utf-8') as file: - return yaml.load(file.read(), Loader=yaml.CLoader) + with open(workflow_file, mode='r', encoding=WORKFLOW_ENCODING) as file: + yaml_dict = yaml.load(file.read(), Loader=yaml.CLoader) + return Workflow(workflow_file=workflow_file, yaml_dict=yaml_dict) + + +def __read_workflow_lines(workflow_file: str) -> List[str]: + with open(workflow_file, mode='r', encoding=WORKFLOW_ENCODING) as file: + return file.readlines() + + +def __write_workflow_lines(workflow_file: str, lines: List[str]): + with open(workflow_file, mode='w', encoding=WORKFLOW_ENCODING) as file: + file.writelines(lines) + + +def __update_workflow(workflow_file: str, *updated_actions: Action): + updated_actions_by_name = reduce(lambda aggr, x: dict(aggr, **{x.name: x}), updated_actions, {}) + lines = __read_workflow_lines(workflow_file) + uses_prefix = Workflow.TAG_USES + ':' + updated_lines = [] + + for line in lines: + updated_lines.append(line) + line_stripped = line.strip() + + if line_stripped.startswith(uses_prefix): + uses_clause = line_stripped[len(uses_prefix):].strip() + action = Action.from_uses_clause(uses_clause) + updated_action = updated_actions_by_name.get(action.name) + + if updated_action: + updated_lines[-1] = line.replace(str(action.version), str(updated_action.version)) + + __write_workflow_lines(workflow_file, updated_lines) def __parse_workflow(workflow_file: str) -> Workflow: print('Searching for GitHub Actions in workflow "' + workflow_file + '"...') - workflow = Workflow(workflow_file) - workflow_yaml = __load_yaml(workflow_file) + workflow = __read_workflow(workflow_file) - for job in workflow_yaml.get('jobs', {}).values(): - for step in job.get('steps', []): - uses = step.get('uses', None) - - if uses: - try: - action = Action.parse(uses) - workflow.actions.add(action) - except ValueError as error: - print('Failed to parse uses-clause in workflow "' + workflow_file + '": ' + str(error)) - sys.exit(-1) + for uses_clause in workflow.uses_clauses: + try: + workflow.actions.add(Action.from_uses_clause(uses_clause)) + except ValueError as error: + print('Failed to parse uses-clause in workflow "' + workflow_file + '": ' + str(error)) + sys.exit(-1) return workflow @@ -208,31 +276,73 @@ def __determine_latest_action_versions(*workflows: Workflow) -> Set[Workflow]: return set(workflows) +def __parse_all_workflows() -> Set[Workflow]: + workflow_directory = path.join('.github', 'workflows') + workflow_files = glob(path.join(workflow_directory, '*.y*ml')) + return __determine_latest_action_versions(*__parse_workflows(*workflow_files)) + + +def __print_table(header: List[str], rows: List[List[str]]): + install_build_dependencies('tabulate') + # pylint: disable=import-outside-toplevel + from tabulate import tabulate + print(tabulate(rows, headers=header)) + + def __print_outdated_actions(*workflows: Workflow): rows = [] for workflow in workflows: - for action in workflow.actions: - if action.is_outdated(): - rows.append([workflow.workflow_file, str(action.name), str(action.version), str(action.latest_version)]) + for action in workflow.outdated_actions: + rows.append([workflow.workflow_file, str(action.name), str(action.version), str(action.latest_version)]) if rows: rows.sort(key=lambda row: (row[0], row[1])) header = ['Workflow', 'Action', 'Current version', 'Latest version'] - install_build_dependencies('tabulate') - # pylint: disable=import-outside-toplevel - from tabulate import tabulate print('The following GitHub Actions are outdated:\n') - print(tabulate(rows, headers=header)) + __print_table(header=header, rows=rows) + else: + print('All GitHub Actions are up-to-date!') + + +def __update_outdated_actions(*workflows: Workflow) -> Set[Workflow]: + rows = [] + + for workflow in workflows: + outdated_actions = workflow.outdated_actions + + if outdated_actions: + workflow_file = workflow.workflow_file + updated_actions = set() + + for action in outdated_actions: + previous_version = action.version + previous_version_numbers = previous_version.version_numbers + latest_version_numbers = action.latest_version.version_numbers + max_version_numbers = min(len(previous_version_numbers), len(latest_version_numbers)) + updated_version = ActionVersion.from_version_numbers(*latest_version_numbers[:max_version_numbers]) + rows.append([workflow_file, action.name, str(previous_version), str(updated_version)]) + action.version = updated_version + updated_actions.add(action) + + __update_workflow(workflow_file, *updated_actions) + + if rows: + rows.sort(key=lambda row: (row[0], row[1])) + header = ['Workflow', 'Action', 'Previous version', 'Updated version'] + print('The following GitHub Actions have been updated:\n') + __print_table(header=header, rows=rows) + else: + print('No GitHub Actions have been updated.') + + return set(workflows) def check_github_actions(**_): """ Checks the project's GitHub workflows for outdated Actions. """ - workflow_directory = path.join('.github', 'workflows') - workflow_files = __get_github_workflow_files(workflow_directory) - workflows = __determine_latest_action_versions(*__parse_workflows(*workflow_files)) + workflows = __parse_all_workflows() __print_outdated_actions(*workflows) @@ -240,4 +350,5 @@ def update_github_actions(**_): """ Updates the versions of outdated GitHub Actions in the project's workflows. """ - print('Updating versions of outdated GitHub Actions...') + workflows = __parse_all_workflows() + __update_outdated_actions(*workflows) From 7ff470b047ee5d138601bc35da0ceedae017f618 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sun, 17 Nov 2024 17:41:04 +0100 Subject: [PATCH 4/6] Update changelog. --- .changelog-bugfix.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changelog-bugfix.md b/.changelog-bugfix.md index 598c81981..a3db40c09 100644 --- a/.changelog-bugfix.md +++ b/.changelog-bugfix.md @@ -2,4 +2,4 @@ - Releases are now automated via continuous integration, including the update of the project's changelog. - The presentation of algorithmic parameters in the documentation has been improved. -- Outdated GitHub Actions can now be printed via the build target `check_github_actions`. +- Outdated GitHub Actions can now be printed via the build target `check_github_actions`. Alternatively, the build target `update_github_actions` may be used to update them automatically. From 4ed9b9372f8874377300074de4ec749e4976875e Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sun, 17 Nov 2024 17:42:54 +0100 Subject: [PATCH 5/6] Update documentation. --- doc/developer_guide/coding_standards.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/doc/developer_guide/coding_standards.md b/doc/developer_guide/coding_standards.md index 2902c496b..25b2b67d8 100644 --- a/doc/developer_guide/coding_standards.md +++ b/doc/developer_guide/coding_standards.md @@ -47,8 +47,28 @@ The project's build system allows to automatically check for outdated GitHub Act ``` ```` +Alternatively, the following command may be used to update the versions of outdated Actions automatically: + +````{tab} Linux + ```text + ./build update_github_actions + ``` +```` + +````{tab} macOS + ```text + ./build update_github_actions + ``` +```` + +````{tab} Windows + ``` + build.bat update_github_actions + ``` +```` + ```{note} -The above command queries the [GitHub API](https://docs.github.com/rest) for the latest version of relevant GitHub Actions. You can optionally specify an [API token](https://docs.github.com/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) to be used for these queries via the command line argument `GITHUB_TOKEN`. If no token is provided, repeated requests may be prohibited due to GitHub's rate limit. +The above commands query the [GitHub API](https://docs.github.com/rest) for the latest version of relevant GitHub Actions. You can optionally specify an [API token](https://docs.github.com/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) to be used for these queries via the environment variable `GITHUB_TOKEN`. If no token is provided, repeated requests might fail due to GitHub's rate limit. ``` (testing)= From c3afdc0ccb64957559b29b6ff68be45698df0663 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sun, 17 Nov 2024 17:43:04 +0100 Subject: [PATCH 6/6] Remove outdated link from the documentation. --- doc/developer_guide/coding_standards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/developer_guide/coding_standards.md b/doc/developer_guide/coding_standards.md index 25b2b67d8..9dd1183bf 100644 --- a/doc/developer_guide/coding_standards.md +++ b/doc/developer_guide/coding_standards.md @@ -14,7 +14,7 @@ We make use of [GitHub Actions](https://docs.github.com/actions) as a [Continuou A track record of past runs can be found on GitHub in the [Actions](https://github.com/mrapp-ke/MLRL-Boomer/actions) tab. ``` -The workflow definitions of individual CI jobs can be found in the directory [.github/workflows/](https://github.com/mrapp-ke/MLRL-Boomer/tree/8ed4f36af5e449c5960a4676bc0a6a22de195979/.github/workflows). Currently, the following jobs are used in the project: +The workflow definitions of individual CI jobs can be found in the directory `.github/workflows/`. Currently, the following jobs are used in the project: - `release.yml` defines a job for releasing a new version of the software developed by this project. The job can be triggered manually for one of the branches mentioned in the section {ref}`release-process`. It automatically updates the project's changelog and publishes a new release on GitHub. - `publish.yml` is used for publishing pre-built packages on [PyPI](https://pypi.org/) (see {ref}`installation`). For this purpose, the project is built from source for each of the target platforms and architectures, using virtualization in some cases. The job is run automatically when a new release was published on [GitHub](https://github.com/mrapp-ke/MLRL-Boomer/releases). It does also increment the project's major version number and merge the release branch into its upstream branches (see {ref}`release-process`).