From c7f450d3544f07b0e056f5ac369a76fdf679ce0a Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 16 Nov 2024 21:11:42 +0100 Subject: [PATCH 01/10] Add "pyyaml" as a build-time dependency. --- scons/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/scons/requirements.txt b/scons/requirements.txt index e9b827c1d..d37a48682 100644 --- a/scons/requirements.txt +++ b/scons/requirements.txt @@ -8,6 +8,7 @@ mdformat-myst >= 0.2, < 0.3 meson >= 1.6, < 1.7 ninja >= 1.11, < 1.12 pylint >= 3.3, < 3.4 +pyyaml >= 6.0, < 6.1 setuptools scons >= 4.8, < 4.9 unittest-xml-reporting >= 3.2, < 3.3 From 46812184268dd0ec2128fdaf7e3ae2b9709cf0e0 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 16 Nov 2024 21:12:16 +0100 Subject: [PATCH 02/10] Add build target "check_github_actions". --- scons/github_actions.py | 12 ++++++++++++ scons/sconstruct.py | 9 +++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 scons/github_actions.py diff --git a/scons/github_actions.py b/scons/github_actions.py new file mode 100644 index 000000000..61a6a02e8 --- /dev/null +++ b/scons/github_actions.py @@ -0,0 +1,12 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for checking the project's GitHub workflows for outdated Actions. +""" + + +def check_github_actions(**_): + """ + Checks the project's GitHub workflows for outdated Actions. + """ + print('Checking for outdated GitHub Actions...') diff --git a/scons/sconstruct.py b/scons/sconstruct.py index ce4f9234a..d51e51d9a 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -15,6 +15,7 @@ from compilation import compile_cpp, compile_cython, install_cpp, install_cython, setup_cpp, setup_cython from dependencies import check_dependency_versions, install_runtime_dependencies from documentation import apidoc_cpp, apidoc_cpp_tocfile, apidoc_python, apidoc_python_tocfile, doc +from github_actions import check_github_actions from modules import BUILD_MODULE, CPP_MODULE, DOC_MODULE, PYTHON_MODULE from packaging import build_python_wheel, install_python_wheels from testing import tests_cpp, tests_python @@ -60,6 +61,7 @@ def __print_if_clean(environment, message: str): TARGET_NAME_FORMAT_MD = TARGET_NAME_FORMAT + '_md' TARGET_NAME_FORMAT_YAML = TARGET_NAME_FORMAT + '_yaml' TARGET_NAME_CHECK_DEPENDENCIES = 'check_dependencies' +TARGET_NAME_CHECK_GITHUB_ACTIONS = 'check_github_actions' TARGET_NAME_VENV = 'venv' TARGET_NAME_COMPILE = 'compile' TARGET_NAME_COMPILE_CPP = TARGET_NAME_COMPILE + '_cpp' @@ -85,8 +87,8 @@ def __print_if_clean(environment, message: str): TARGET_NAME_UPDATE_CHANGELOG_MAIN, TARGET_NAME_PRINT_VERSION, TARGET_NAME_PRINT_LATEST_CHANGELOG, TARGET_NAME_TEST_FORMAT, TARGET_NAME_TEST_FORMAT_PYTHON, TARGET_NAME_TEST_FORMAT_CPP, TARGET_NAME_TEST_FORMAT_MD, TARGET_NAME_TEST_FORMAT_YAML, TARGET_NAME_FORMAT, TARGET_NAME_FORMAT_PYTHON, TARGET_NAME_FORMAT_CPP, - TARGET_NAME_FORMAT_MD, TARGET_NAME_FORMAT_YAML, TARGET_NAME_CHECK_DEPENDENCIES, TARGET_NAME_VENV, - TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP, TARGET_NAME_COMPILE_CYTHON, TARGET_NAME_INSTALL, + TARGET_NAME_FORMAT_MD, TARGET_NAME_FORMAT_YAML, TARGET_NAME_CHECK_DEPENDENCIES, TARGET_NAME_CHECK_GITHUB_ACTIONS, + TARGET_NAME_VENV, TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP, TARGET_NAME_COMPILE_CYTHON, TARGET_NAME_INSTALL, TARGET_NAME_INSTALL_CPP, TARGET_NAME_INSTALL_CYTHON, TARGET_NAME_BUILD_WHEELS, TARGET_NAME_INSTALL_WHEELS, TARGET_NAME_TESTS, TARGET_NAME_TESTS_CPP, TARGET_NAME_TESTS_PYTHON, TARGET_NAME_APIDOC, TARGET_NAME_APIDOC_CPP, TARGET_NAME_APIDOC_PYTHON, TARGET_NAME_DOC @@ -148,6 +150,9 @@ def __print_if_clean(environment, message: str): # Define target for checking dependency versions... target_check_dependencies = __create_phony_target(env, TARGET_NAME_CHECK_DEPENDENCIES, action=check_dependency_versions) +# Define target for checking the versions of GitHub Actions... +target_check_github_actions = __create_phony_target(env, TARGET_NAME_CHECK_GITHUB_ACTIONS, action=check_github_actions) + # Define target for installing runtime dependencies... target_venv = __create_phony_target(env, TARGET_NAME_VENV, action=install_runtime_dependencies) From 06e2b30169466197aaec8eaebf02e1a6a4a58d13 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 16 Nov 2024 21:38:34 +0100 Subject: [PATCH 03/10] Add "pygithub" as a build-time dependency. --- scons/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/scons/requirements.txt b/scons/requirements.txt index d37a48682..72267259e 100644 --- a/scons/requirements.txt +++ b/scons/requirements.txt @@ -7,6 +7,7 @@ mdformat >= 0.7, < 0.8 mdformat-myst >= 0.2, < 0.3 meson >= 1.6, < 1.7 ninja >= 1.11, < 1.12 +pygithub >= 2.5, < 2.6 pylint >= 3.3, < 3.4 pyyaml >= 6.0, < 6.1 setuptools From c4070b62f44934c786651d987089c5e3684e16c2 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 16 Nov 2024 23:07:45 +0100 Subject: [PATCH 04/10] Add "tabulate" as a build-time dependency. --- scons/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/scons/requirements.txt b/scons/requirements.txt index 72267259e..46cc120e2 100644 --- a/scons/requirements.txt +++ b/scons/requirements.txt @@ -12,6 +12,7 @@ pylint >= 3.3, < 3.4 pyyaml >= 6.0, < 6.1 setuptools scons >= 4.8, < 4.9 +tabulate >= 0.9, < 0.10 unittest-xml-reporting >= 3.2, < 3.3 wheel >= 0.45, < 0.46 yamlfix >= 1.17, < 1.18 From ce88e2a8dc7d022de1b3f420c955e671e4b33a70 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sat, 16 Nov 2024 23:20:06 +0100 Subject: [PATCH 05/10] Implement printing of outdated Github actions. --- scons/github_actions.py | 226 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 225 insertions(+), 1 deletion(-) diff --git a/scons/github_actions.py b/scons/github_actions.py index 61a6a02e8..2fd0473a5 100644 --- a/scons/github_actions.py +++ b/scons/github_actions.py @@ -3,10 +3,234 @@ Provides utility functions for checking the project's GitHub workflows for outdated Actions. """ +import sys + +from dataclasses import dataclass, field +from glob import glob +from os import environ, path +from typing import List, Optional, Set + +from dependencies import install_build_dependencies +from environment import get_env + +ENV_GITHUB_TOKEN = 'GITHUB_TOKEN' + +SEPARATOR_VERSION = '@' + +SEPARATOR_VERSION_NUMBER = '.' + +SEPARATOR_PATH = '/' + + +@dataclass +class ActionVersion: + """ + The version of a GitHub Action. + + Attributes: + version: The full version string + """ + version: str + + def __str__(self) -> str: + return self.version.lstrip('v') + + def __lt__(self, other: 'ActionVersion') -> bool: + first_numbers = str(self).split(SEPARATOR_VERSION_NUMBER) + second_numbers = str(other).split(SEPARATOR_VERSION_NUMBER) + + for i in range(min(len(first_numbers), len(second_numbers))): + first = int(first_numbers[i]) + second = int(second_numbers[i]) + + if first > second: + return False + if first < second: + return True + + return False + + +@dataclass +class Action: + """ + A GitHub Action. + + Attributes: + name: The name of the Action + version: The version of the Action + latest_version: The latest version of the Action, if known + """ + name: str + version: ActionVersion + latest_version: Optional[ActionVersion] = None + + @staticmethod + def parse(uses: str) -> 'Action': + """ + Parses and returns a GitHub Action as specified via the uses-clause of a workflow. + + :param uses: The uses-clause + :return: The GitHub Action + """ + parts = uses.split(SEPARATOR_VERSION) + + if len(parts) != 2: + raise ValueError('Action must contain the symbol + "' + SEPARATOR_VERSION + '", but got "' + uses + '"') + + return Action(name=parts[0], version=ActionVersion(parts[1])) + + @property + def repository(self) -> str: + """ + The name of the repository, where the GitHub Action is hosted. + """ + repository = self.name + parts = repository.split(SEPARATOR_PATH) + return SEPARATOR_PATH.join(parts[:2]) if len(parts) > 2 else repository + + def is_outdated(self) -> bool: + """ + Returns whether the GitHub Action is known to be outdated or not. + + :return: True, if the GitHub Action is outdated, False otherwise + """ + return self.latest_version and self.version < self.latest_version + + def __str__(self) -> str: + return self.name + SEPARATOR_VERSION + str(self.version) + + def __eq__(self, other: 'Action') -> bool: + return str(self) == str(other) + + def __hash__(self): + return hash(str(self)) + + +@dataclass +class Workflow: + """ + A GitHub workflow. + + Attributes: + workflow_file: The path of the workflow definition file + actions: A set that stores the Actions in the workflow + """ + workflow_file: str + actions: Set[Action] = field(default_factory=set) + + def __eq__(self, other: 'Workflow') -> bool: + return self.workflow_file == other.workflow_file + + def __hash__(self): + return hash(self.workflow_file) + + +def __get_github_workflow_files(directory: str) -> List[str]: + return glob(path.join(directory, '*.y*ml')) + + +def __load_yaml(workflow_file: str) -> dict: + install_build_dependencies('pyyaml') + # pylint: disable=import-outside-toplevel + import yaml + with open(workflow_file, encoding='utf-8') as file: + return yaml.load(file.read(), Loader=yaml.CLoader) + + +def __parse_workflow(workflow_file: str) -> Workflow: + print('Searching for GitHub Actions in workflow "' + workflow_file + '"...') + workflow = Workflow(workflow_file) + workflow_yaml = __load_yaml(workflow_file) + + for job in workflow_yaml.get('jobs', {}).values(): + for step in job.get('steps', []): + uses = step.get('uses', None) + + if uses: + try: + action = Action.parse(uses) + workflow.actions.add(action) + except ValueError as error: + print('Failed to parse uses-clause in workflow "' + workflow_file + '": ' + str(error)) + sys.exit(-1) + + return workflow + + +def __parse_workflows(*workflow_files: str) -> Set[Workflow]: + return {__parse_workflow(workflow_file) for workflow_file in workflow_files} + + +def __query_latest_action_version(action: Action, github_token: Optional[str] = None) -> Optional[ActionVersion]: + repository_name = action.repository + install_build_dependencies('pygithub') + # pylint: disable=import-outside-toplevel + from github import Auth, Github, UnknownObjectException + + try: + github_auth = Auth.Token(github_token) if github_token else None + github_client = Github(auth=github_auth) + github_repository = github_client.get_repo(repository_name) + latest_release = github_repository.get_latest_release() + latest_tag = latest_release.tag_name + return ActionVersion(latest_tag) + except UnknownObjectException as error: + print('Query to GitHub API failed for action "' + str(action) + '" hosted in repository "' + repository_name + + '": ' + str(error)) + sys.exit(-1) + + +def __get_github_token() -> Optional[str]: + github_token = get_env(environ, ENV_GITHUB_TOKEN) + + if not github_token: + print('No GitHub API token is set. You can specify it via the environment variable ' + ENV_GITHUB_TOKEN + '.') + + return github_token + + +def __determine_latest_action_versions(*workflows: Workflow) -> Set[Workflow]: + github_token = __get_github_token() + version_cache = {} + + for workflow in workflows: + for action in workflow.actions: + latest_version = version_cache.get(action) + + if not latest_version: + print('Checking version of GitHub Action "' + action.name + '"...') + latest_version = __query_latest_action_version(action, github_token=github_token) + version_cache[action] = latest_version + + action.latest_version = latest_version + + return set(workflows) + + +def __print_outdated_actions(*workflows: Workflow): + rows = [] + + for workflow in workflows: + for action in workflow.actions: + if action.is_outdated(): + rows.append([workflow.workflow_file, str(action.name), str(action.version), str(action.latest_version)]) + + if rows: + rows.sort(key=lambda row: (row[0], row[1])) + header = ['Workflow', 'Action', 'Current version', 'Latest version'] + install_build_dependencies('tabulate') + # pylint: disable=import-outside-toplevel + from tabulate import tabulate + print('The following GitHub Actions are outdated:\n') + print(tabulate(rows, headers=header)) def check_github_actions(**_): """ Checks the project's GitHub workflows for outdated Actions. """ - print('Checking for outdated GitHub Actions...') + workflow_directory = path.join('.github', 'workflows') + workflow_files = __get_github_workflow_files(workflow_directory) + workflows = __determine_latest_action_versions(*__parse_workflows(*workflow_files)) + __print_outdated_actions(*workflows) From 19971beb59eeaa43c69942a8c7149fe97a934668 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sun, 17 Nov 2024 00:30:09 +0100 Subject: [PATCH 06/10] Update changelog. --- .changelog-bugfix.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.changelog-bugfix.md b/.changelog-bugfix.md index 826663de5..598c81981 100644 --- a/.changelog-bugfix.md +++ b/.changelog-bugfix.md @@ -2,3 +2,4 @@ - Releases are now automated via continuous integration, including the update of the project's changelog. - The presentation of algorithmic parameters in the documentation has been improved. +- Outdated GitHub Actions can now be printed via the build target `check_github_actions`. From fe9a09884bbdc18f6c25edbfc3c052c7446338e2 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sun, 17 Nov 2024 00:41:11 +0100 Subject: [PATCH 07/10] Update documentation. --- doc/developer_guide/coding_standards.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/doc/developer_guide/coding_standards.md b/doc/developer_guide/coding_standards.md index 455d50bbd..cea8f2c72 100644 --- a/doc/developer_guide/coding_standards.md +++ b/doc/developer_guide/coding_standards.md @@ -27,6 +27,30 @@ The workflow definitions of individual CI jobs can be found in the directory [.g - `merge_feature.yml` and `merge_bugfix.yml` are used to merge changes that have been pushed to the feature or bugfix branch into downstream branches via pull requests (see {ref}`release-process`). - `merge_release.yml` is used to merge all changes included in a new release published on [GitHub](https://github.com/mrapp-ke/MLRL-Boomer/releases) into upstream branches and update the version numbers of these branches. +The project's build system allows to automatically check for outdated GitHub Actions used in the workflows mentioned above. These are reusable building blocks provided by third-party developers. The following command prints a list of all outdated Actions: + +````{tab} Linux + ```text + ./build check_github_actions + ``` +```` + +````{tab} macOS + ```text + ./build check_github_actions + ``` +```` + +````{tab} Windows + ``` + build.bat check_github_actions + ``` +```` + +```{note} +The above command queries the [GitHub API](https://docs.github.com/rest) for the latest version of relevant GitHub Actions. You can optionally specify an [API token](https://docs.github.com/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) to be used for these queries via the command line argument `GITHUB_TOKEN`. If no token is provided, repeated requests may be prohibited due to GitHub's rate limit. +``` + (testing)= ## Testing the Code From 03d359365ff6bb96aee65df1a29611f43a32c51d Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Sun, 17 Nov 2024 00:48:46 +0100 Subject: [PATCH 08/10] Use language-independent links to the Github documentation. --- doc/developer_guide/coding_standards.md | 2 +- doc/developer_guide/compilation.md | 2 +- doc/developer_guide/documentation.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/developer_guide/coding_standards.md b/doc/developer_guide/coding_standards.md index cea8f2c72..2902c496b 100644 --- a/doc/developer_guide/coding_standards.md +++ b/doc/developer_guide/coding_standards.md @@ -8,7 +8,7 @@ As it is common for Open Source projects, where everyone is invited to contribut ## Continuous Integration -We make use of [GitHub Actions](https://docs.github.com/en/actions) as a [Continuous Integration](https://en.wikipedia.org/wiki/Continuous_integration) (CI) server for running predefined jobs, such as automated tests, in a controlled environment. Whenever certain parts of the project's repository have changed, relevant jobs are automatically executed. +We make use of [GitHub Actions](https://docs.github.com/actions) as a [Continuous Integration](https://en.wikipedia.org/wiki/Continuous_integration) (CI) server for running predefined jobs, such as automated tests, in a controlled environment. Whenever certain parts of the project's repository have changed, relevant jobs are automatically executed. ```{tip} A track record of past runs can be found on GitHub in the [Actions](https://github.com/mrapp-ke/MLRL-Boomer/actions) tab. diff --git a/doc/developer_guide/compilation.md b/doc/developer_guide/compilation.md index 0727fe0b1..bd67c2a6d 100644 --- a/doc/developer_guide/compilation.md +++ b/doc/developer_guide/compilation.md @@ -209,7 +209,7 @@ The shared libraries that have been created in the previous steps from the C++ s This should result in the compilation files, which were previously located in the `cpp/build/` directory, to be copied into the `cython/` subdirectories that are contained by each Python module (e.g., into the directory `python/subprojects/common/mlrl/common/cython/`). ```{tip} -When shared libaries are built via {ref}`Continuous Integration ` jobs, the resulting files for different platform are saved as [artifacts](https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts) and can be downloaded as zip archives. +When shared libaries are built via {ref}`Continuous Integration ` jobs, the resulting files for different platform are saved as [artifacts](https://docs.github.com/actions/using-workflows/storing-workflow-data-as-artifacts) and can be downloaded as zip archives. ``` ## Installing Extension Modules diff --git a/doc/developer_guide/documentation.md b/doc/developer_guide/documentation.md index 59fa83228..f617affab 100644 --- a/doc/developer_guide/documentation.md +++ b/doc/developer_guide/documentation.md @@ -5,7 +5,7 @@ The documentation of the BOOMER algorithm and other software provided by this project is publicly available at [https://mlrl-boomer.readthedocs.io](https://mlrl-boomer.readthedocs.io/en/latest/). This website should be the primary source of information for everyone who wants to learn about our work. However, if you want to generate the documentation from scratch, e.g., for offline use on your own computer, follow the instructions below. ```{tip} -The documentation is regularly built by our {ref}`Continuous Integration ` jobs. The documentation generated by a particular job is saved as an [artifact](https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts) and can be downloaded as a zip archive. +The documentation is regularly built by our {ref}`Continuous Integration ` jobs. The documentation generated by a particular job is saved as an [artifact](https://docs.github.com/actions/using-workflows/storing-workflow-data-as-artifacts) and can be downloaded as a zip archive. ``` ## Prerequisites From 772c71ead363437c630e2d577c80837605a1e294 Mon Sep 17 00:00:00 2001 From: michael-rapp <6638695+michael-rapp@users.noreply.github.com> Date: Sun, 17 Nov 2024 00:07:02 +0000 Subject: [PATCH 09/10] [Bot] Merge bugfix into feature branch. --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index a8839f70d..d33c3a212 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -0.11.2 \ No newline at end of file +0.12.0 \ No newline at end of file From a9e3b72e4c691307e1713e2f19d2b8aac81d42e6 Mon Sep 17 00:00:00 2001 From: "issue-api-tokens[bot]" <167710561+issue-api-tokens[bot]@users.noreply.github.com> Date: Sun, 17 Nov 2024 00:24:24 +0000 Subject: [PATCH 10/10] [Bot] Merge feature into main branch. --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index d33c3a212..afaf360d3 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -0.12.0 \ No newline at end of file +1.0.0 \ No newline at end of file