From a2e1c4ac39c2ca149dc4a7fae076f924284ee887 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 7 Nov 2024 00:38:28 +0100 Subject: [PATCH 1/8] Implement validation of changelog files. --- scons/changelog.py | 184 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 scons/changelog.py diff --git a/scons/changelog.py b/scons/changelog.py new file mode 100644 index 000000000..206d3ea58 --- /dev/null +++ b/scons/changelog.py @@ -0,0 +1,184 @@ +""" +Author: Michael Rapp (michael.rapp.ml@gmail.com) + +Provides utility functions for validating and updating the project's changelog. +""" +import sys + +from dataclasses import dataclass, field +from enum import Enum, auto +from os.path import isfile +from typing import List, Optional + +PREFIX_HEADER = '# ' + +PREFIX_DASH = '- ' + +PREFIX_ASTERISK = '* ' + +CHANGELOG_FILE_MAIN = '.changelog-main.md' + +CHANGELOG_FILE_FEATURE = '.changelog-feature.md' + +CHANGELOG_FILE_BUGFIX = '.changelog-bugfix.md' + +CHANGELOG_ENCODING = 'utf-8' + + +class LineType(Enum): + """ + Represents different types of lines that may occur in a changelog. + """ + BLANK = auto() + HEADER = auto() + ENUMERATION = auto() + + @staticmethod + def parse(line: str) -> Optional['LineType']: + """ + Parses a given line and returns its type. + + :return: The type of the given line or None, if the line is invalid + """ + if not line or line.isspace(): + return LineType.BLANK + if line.startswith(PREFIX_HEADER): + return LineType.HEADER + if line.startswith(PREFIX_DASH) or line.startswith(PREFIX_ASTERISK): + return LineType.ENUMERATION + return None + + +@dataclass +class Changeset: + """ + A changeset, consisting of a header and textual descriptions of several changes. + + Attributes: + header: The header of the changeset + changes: A list that stores the textual descriptions of the changes + """ + header: str + changes: List[str] = field(default_factory=list) + + +@dataclass +class Line: + """ + A single line in a changelog. + + Attributes: + line_number: The line number, starting at 1 + line_type: The type of the line + line: The original content of the line + content: The content of the line with Markdown keywords being stripped away + """ + line_number: int + line_type: LineType + line: str + content: str + + +def __read_lines(changelog_file: str, skip_if_missing: bool = False) -> List[str]: + if skip_if_missing and not isfile(changelog_file): + return [] + + with open(changelog_file, mode='r', encoding=CHANGELOG_ENCODING) as file: + return file.readlines() + + +def __parse_line(changelog_file: str, line_number: int, line: str) -> Line: + line = line.strip('\n') + line_type = LineType.parse(line) + + if not line_type: + print('Line ' + str(line_number) + ' of file "' + changelog_file + + '" is invalid: Must be blank, a top-level header (starting with "' + PREFIX_HEADER + + '"), or an enumeration (starting with "' + PREFIX_DASH + '" or "' + PREFIX_ASTERISK + '"), but is "' + + line + '"') + sys.exit(-1) + + content = line + + if line_type != LineType.BLANK: + content = line.lstrip(PREFIX_HEADER).lstrip(PREFIX_DASH).lstrip(PREFIX_ASTERISK) + + if not content or content.isspace(): + print('Line ' + str(line_number) + ' of file "' + changelog_file + + '" is is invalid: Content must not be blank, but is "' + line + '"') + sys.exit(-1) + + return Line(line_number=line_number, line_type=line_type, line=line, content=content) + + +def __validate_line(changelog_file: str, current_line: Optional[Line], previous_line: Optional[Line]): + current_line_is_enumeration = current_line and current_line.line_type == LineType.ENUMERATION + + if current_line_is_enumeration and not previous_line: + print('File "' + changelog_file + '" must start with a top-level header (starting with "' + PREFIX_HEADER + + '")') + sys.exit(-1) + + current_line_is_header = current_line and current_line.line_type == LineType.HEADER + previous_line_is_header = previous_line and previous_line.line_type == LineType.HEADER + + if (current_line_is_header and previous_line_is_header) or (not current_line and previous_line_is_header): + print('Header "' + previous_line.line + '" at line ' + str(previous_line.line_number) + ' of file "' + + changelog_file + '" is not followed by any content') + sys.exit(-1) + + +def __parse_lines(changelog_file: str, lines: List[str]) -> List[Line]: + previous_line = None + parsed_lines = [] + + for i, line in enumerate(lines): + current_line = __parse_line(changelog_file=changelog_file, line_number=(i + 1), line=line) + + if current_line.line_type != LineType.BLANK: + __validate_line(changelog_file=changelog_file, current_line=current_line, previous_line=previous_line) + previous_line = current_line + parsed_lines.append(current_line) + + __validate_line(changelog_file=changelog_file, current_line=None, previous_line=previous_line) + return parsed_lines + + +def __parse_changesets(changelog_file: str, skip_if_missing: bool = False) -> List[Changeset]: + changesets = [] + lines = __parse_lines(changelog_file, __read_lines(changelog_file, skip_if_missing=skip_if_missing)) + + for line in lines: + if line.line_type == LineType.HEADER: + changesets.append(Changeset(header=line.content)) + elif line.line_type == LineType.ENUMERATION: + current_changeset = changesets[-1] + current_changeset.changes.append(line.content) + + return changesets + + +def __validate_changelog(changelog_file: str): + print('Validating changelog file "' + changelog_file + '"...') + __parse_changesets(changelog_file, skip_if_missing=True) + + +def validate_changelog_bugfix(**_): + """ + Validates the changelog file that lists bugfixes. + """ + __validate_changelog(CHANGELOG_FILE_BUGFIX) + + +def validate_changelog_feature(**_): + """ + Validates the changelog file that lists new features. + """ + __validate_changelog(CHANGELOG_FILE_FEATURE) + + +def validate_changelog_main(**_): + """ + Validates the changelog file that lists major updates. + """ + __validate_changelog(CHANGELOG_FILE_MAIN) From 00c58a10639e9b2c7f1867e214d55095f8b6bc9f Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 7 Nov 2024 00:38:42 +0100 Subject: [PATCH 2/8] Add build targets for validating changelog files. --- scons/sconstruct.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scons/sconstruct.py b/scons/sconstruct.py index 65b26aa0b..0e2483dca 100644 --- a/scons/sconstruct.py +++ b/scons/sconstruct.py @@ -8,6 +8,7 @@ from functools import reduce from os import path +from changelog import validate_changelog_bugfix, validate_changelog_feature, validate_changelog_main from code_style import check_cpp_code_style, check_md_code_style, check_python_code_style, check_yaml_code_style, \ enforce_cpp_code_style, enforce_md_code_style, enforce_python_code_style, enforce_yaml_code_style from compilation import compile_cpp, compile_cython, install_cpp, install_cython, setup_cpp, setup_cython @@ -39,6 +40,9 @@ def __print_if_clean(environment, message: str): TARGET_NAME_INCREMENT_PATCH_VERSION = 'increment_patch_version' TARGET_NAME_INCREMENT_MINOR_VERSION = 'increment_minor_version' TARGET_NAME_INCREMENT_MAJOR_VERSION = 'increment_major_version' +TARGET_NAME_VALIDATE_CHANGELOG_BUGFIX = 'validate_changelog_bugfix' +TARGET_NAME_VALIDATE_CHANGELOG_FEATURE = 'validate_changelog_feature' +TARGET_NAME_VALIDATE_CHANGELOG_MAIN = 'validate_changelog_main' TARGET_NAME_TEST_FORMAT = 'test_format' TARGET_NAME_TEST_FORMAT_PYTHON = TARGET_NAME_TEST_FORMAT + '_python' TARGET_NAME_TEST_FORMAT_CPP = TARGET_NAME_TEST_FORMAT + '_cpp' @@ -70,7 +74,8 @@ def __print_if_clean(environment, message: str): VALID_TARGETS = { TARGET_NAME_INCREMENT_DEVELOPMENT_VERSION, TARGET_NAME_RESET_DEVELOPMENT_VERSION, TARGET_NAME_APPLY_DEVELOPMENT_VERSION, TARGET_NAME_INCREMENT_PATCH_VERSION, TARGET_NAME_INCREMENT_MINOR_VERSION, - TARGET_NAME_INCREMENT_MAJOR_VERSION, TARGET_NAME_TEST_FORMAT, TARGET_NAME_TEST_FORMAT_PYTHON, + TARGET_NAME_INCREMENT_MAJOR_VERSION, TARGET_NAME_VALIDATE_CHANGELOG_BUGFIX, TARGET_NAME_VALIDATE_CHANGELOG_FEATURE, + TARGET_NAME_VALIDATE_CHANGELOG_MAIN, TARGET_NAME_TEST_FORMAT, TARGET_NAME_TEST_FORMAT_PYTHON, TARGET_NAME_TEST_FORMAT_CPP, TARGET_NAME_TEST_FORMAT_MD, TARGET_NAME_TEST_FORMAT_YAML, TARGET_NAME_FORMAT, TARGET_NAME_FORMAT_PYTHON, TARGET_NAME_FORMAT_CPP, TARGET_NAME_FORMAT_MD, TARGET_NAME_FORMAT_YAML, TARGET_NAME_CHECK_DEPENDENCIES, TARGET_NAME_VENV, TARGET_NAME_COMPILE, TARGET_NAME_COMPILE_CPP, @@ -101,6 +106,11 @@ def __print_if_clean(environment, message: str): __create_phony_target(env, TARGET_NAME_INCREMENT_MINOR_VERSION, action=increment_minor_version) __create_phony_target(env, TARGET_NAME_INCREMENT_MAJOR_VERSION, action=increment_major_version) +# Define targets for validating changelogs... +__create_phony_target(env, TARGET_NAME_VALIDATE_CHANGELOG_BUGFIX, action=validate_changelog_bugfix) +__create_phony_target(env, TARGET_NAME_VALIDATE_CHANGELOG_FEATURE, action=validate_changelog_feature) +__create_phony_target(env, TARGET_NAME_VALIDATE_CHANGELOG_MAIN, action=validate_changelog_main) + # Define targets for checking code style definitions... target_test_format_python = __create_phony_target(env, TARGET_NAME_TEST_FORMAT_PYTHON, action=check_python_code_style) target_test_format_cpp = __create_phony_target(env, TARGET_NAME_TEST_FORMAT_CPP, action=check_cpp_code_style) From 29b0302e7beb9855425f41a86647a4e63369ca97 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 7 Nov 2024 00:12:52 +0100 Subject: [PATCH 3/8] Add Github workflow for validating changelogs. --- .github/workflows/test_changelog.yml | 54 ++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/test_changelog.yml diff --git a/.github/workflows/test_changelog.yml b/.github/workflows/test_changelog.yml new file mode 100644 index 000000000..b9948440f --- /dev/null +++ b/.github/workflows/test_changelog.yml @@ -0,0 +1,54 @@ +--- +name: Validate changelog +on: + pull_request: + types: + - opened + - reopened + - synchronize +jobs: + test_changelog: + name: Validate changelog + runs-on: ubuntu-latest + steps: + - name: Look up Git repository in cache + uses: actions/cache/restore@v4 + with: + path: .git/ + key: test-changelog-cache-git-${{ github.run_id }} + restore-keys: | + test-changelog-cache-git + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version-file: .version-python + - name: Detect changes + uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + build_files: &build_files + - '.github/workflows/test_changelog.yml' + - 'build' + - 'build.bat' + - 'scons/**' + bugfix: + - *build_files + - '.changelog-bugfix.md' + feature: + - *build_files + - '.changelog-feature.md' + main: + - *build_files + - '.changelog-main.md' + - name: Validate bugfix changelog + if: steps.filter.outputs.bugfix == 'true' + run: ./build validate_changelog_bugfix + - name: Validate feature changelog + if: steps.filter.outputs.feature == 'true' + run: ./build validate_changelog_feature + - name: Validate main changelog + if: steps.filter.outputs.main == 'true' + run: ./build validate_changelog_main From 429552920fcc951787de836352f9e28eb447b265 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 7 Nov 2024 00:42:51 +0100 Subject: [PATCH 4/8] Fix names of Github workflow files in the documentation. --- doc/developer_guide/coding_standards.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/developer_guide/coding_standards.md b/doc/developer_guide/coding_standards.md index 2bc5b7ee4..1874f561e 100644 --- a/doc/developer_guide/coding_standards.md +++ b/doc/developer_guide/coding_standards.md @@ -16,8 +16,8 @@ A track record of past runs can be found on Github in the [Actions](https://gith The workflow definitions of individual CI jobs can be found in the directory [.github/workflows/](https://github.com/mrapp-ke/MLRL-Boomer/tree/8ed4f36af5e449c5960a4676bc0a6a22de195979/.github/workflows). Currently, the following jobs are used in the project: -- `release.yml` is used for publishing pre-built packages on [PyPI](https://pypi.org/) (see {ref}`installation`). For this purpose, the project is built from source for each of the target platforms and architectures, using virtualization in some cases. The job is run automatically when a new release was published on [Github](https://github.com/mrapp-ke/MLRL-Boomer/releases). It does also increment the project's major version number and merge the release branch into its upstream branches (see {ref}`release-process`). -- `release_development.yml` publishes development versions of packages on [Test-PyPI](https://test.pypi.org/) whenever changes to the project's source code have been pushed to the main branch. The packages built by each of these runs are also saved as [artifacts](https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts) and can be downloaded as zip archives. +- `publish.yml` is used for publishing pre-built packages on [PyPI](https://pypi.org/) (see {ref}`installation`). For this purpose, the project is built from source for each of the target platforms and architectures, using virtualization in some cases. The job is run automatically when a new release was published on [Github](https://github.com/mrapp-ke/MLRL-Boomer/releases). It does also increment the project's major version number and merge the release branch into its upstream branches (see {ref}`release-process`). +- `publish_development.yml` publishes development versions of packages on [Test-PyPI](https://test.pypi.org/) whenever changes to the project's source code have been pushed to the main branch. The packages built by each of these runs are also saved as [artifacts](https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts) and can be downloaded as zip archives. - `test_release.yml` ensures that the packages to be released for different architectures and Python versions can be built. The job is run for pull requests that modify relevant parts of the source code. - `test_build.yml` builds the project for each of the supported target platforms, i.e., Linux, Windows, and MacOS (see {ref}`compilation`). In the Linux environment, this job does also execute all available unit and integration tests (see {ref}`testing`). It is run for pull requests whenever relevant parts of the project's source code have been modified. - `test_doc.yml` generates the latest documentation (see {ref}`documentation`) whenever relevant parts of the source code are affected by a pull request. @@ -180,7 +180,7 @@ We do not allow directly pushing to the above branches. Instead, all changes mus Once modifications to one of the branches have been merged, {ref}`Continuous Integration ` jobs are used to automatically update downstream branches via pull requests. If all checks for such pull requests are successful, they are merged automatically. If there are any merge conflicts, they must be resolved manually. Following this procedure, changes to the feature brach are merged into the main branch (see `merge_feature.yml`), whereas changes to the bugfix branch are first merged into the feature branch and then into the main branch (see `merge_bugfix.yml`). -Whenever a new release has been published, the release branch is merged into the upstream branches (see `merge_release.yml`), i.e., major releases result in the feature and bugfix branches being updated, whereas minor releases result in the bugfix branch being updated. The version of the release branch and the affected branches are updated accordingly. The version of a branch is specified in the file `.version` in the project's root directory. Similarly, the file `.version-dev` is used to keep track of the version number used for development releases (see `release_development.yml`). +Whenever a new release has been published, the release branch is merged into the upstream branches (see `merge_release.yml`), i.e., major releases result in the feature and bugfix branches being updated, whereas minor releases result in the bugfix branch being updated. The version of the release branch and the affected branches are updated accordingly. The version of a branch is specified in the file `.version` in the project's root directory. Similarly, the file `.version-dev` is used to keep track of the version number used for development releases (see `publish_development.yml`). (dependencies)= From b852df0ef915ee70ed46a18540eddb6fae8e48f4 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 7 Nov 2024 00:43:48 +0100 Subject: [PATCH 5/8] Rename Github workflow file. --- .github/workflows/{test_release.yml => test_publish.yml} | 2 +- doc/developer_guide/coding_standards.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{test_release.yml => test_publish.yml} (97%) diff --git a/.github/workflows/test_release.yml b/.github/workflows/test_publish.yml similarity index 97% rename from .github/workflows/test_release.yml rename to .github/workflows/test_publish.yml index 45e444d6b..b9b03ad43 100644 --- a/.github/workflows/test_release.yml +++ b/.github/workflows/test_publish.yml @@ -29,7 +29,7 @@ jobs: with: filters: | build_files: - - '.github/workflows/test_release.yml' + - '.github/workflows/test_publish.yml' - '.github/workflows/template_publish.yml' - '.github/workflows/template_publish_non_native.yml' - '.github/workflows/template_publish_platform.yml' diff --git a/doc/developer_guide/coding_standards.md b/doc/developer_guide/coding_standards.md index 1874f561e..7b8693b41 100644 --- a/doc/developer_guide/coding_standards.md +++ b/doc/developer_guide/coding_standards.md @@ -18,7 +18,7 @@ The workflow definitions of individual CI jobs can be found in the directory [.g - `publish.yml` is used for publishing pre-built packages on [PyPI](https://pypi.org/) (see {ref}`installation`). For this purpose, the project is built from source for each of the target platforms and architectures, using virtualization in some cases. The job is run automatically when a new release was published on [Github](https://github.com/mrapp-ke/MLRL-Boomer/releases). It does also increment the project's major version number and merge the release branch into its upstream branches (see {ref}`release-process`). - `publish_development.yml` publishes development versions of packages on [Test-PyPI](https://test.pypi.org/) whenever changes to the project's source code have been pushed to the main branch. The packages built by each of these runs are also saved as [artifacts](https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts) and can be downloaded as zip archives. -- `test_release.yml` ensures that the packages to be released for different architectures and Python versions can be built. The job is run for pull requests that modify relevant parts of the source code. +- `test_publish.yml` ensures that the packages to be released for different architectures and Python versions can be built. The job is run for pull requests that modify relevant parts of the source code. - `test_build.yml` builds the project for each of the supported target platforms, i.e., Linux, Windows, and MacOS (see {ref}`compilation`). In the Linux environment, this job does also execute all available unit and integration tests (see {ref}`testing`). It is run for pull requests whenever relevant parts of the project's source code have been modified. - `test_doc.yml` generates the latest documentation (see {ref}`documentation`) whenever relevant parts of the source code are affected by a pull request. - `test_format.yml` ensures that all source files in the project adhere to our coding style guidelines (see {ref}`code-style`). This job is run automatically for pull requests whenever they include any changes affecting the relevant source files. From 3fc64a1d169be35e9d35aa73086d902a2a2594d3 Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 7 Nov 2024 00:44:25 +0100 Subject: [PATCH 6/8] Remove obsolete information from the documentation. --- doc/developer_guide/coding_standards.md | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/developer_guide/coding_standards.md b/doc/developer_guide/coding_standards.md index 7b8693b41..406bc2920 100644 --- a/doc/developer_guide/coding_standards.md +++ b/doc/developer_guide/coding_standards.md @@ -22,7 +22,6 @@ The workflow definitions of individual CI jobs can be found in the directory [.g - `test_build.yml` builds the project for each of the supported target platforms, i.e., Linux, Windows, and MacOS (see {ref}`compilation`). In the Linux environment, this job does also execute all available unit and integration tests (see {ref}`testing`). It is run for pull requests whenever relevant parts of the project's source code have been modified. - `test_doc.yml` generates the latest documentation (see {ref}`documentation`) whenever relevant parts of the source code are affected by a pull request. - `test_format.yml` ensures that all source files in the project adhere to our coding style guidelines (see {ref}`code-style`). This job is run automatically for pull requests whenever they include any changes affecting the relevant source files. -- `test_file_changes.yml` prevents pull requests from modifying certain files that must not be modified manually, but are intended to only be updated via Github Actions. - `merge_feature.yml` and `merge_bugfix.yml` are used to merge changes that have been pushed to the feature or bugfix branch into downstream branches via pull requests (see {ref}`release-process`). - `merge_release.yml` is used to merge all changes included in a new release published on [Github](https://github.com/mrapp-ke/MLRL-Boomer/releases) into upstream branches and update the version numbers of these branches. From 532786e51ce074541a058e6e3b1b28e9865b386e Mon Sep 17 00:00:00 2001 From: Michael Rapp Date: Thu, 7 Nov 2024 00:47:37 +0100 Subject: [PATCH 7/8] Mention Github workflow for validating changelogs in the documentation. --- doc/developer_guide/coding_standards.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/developer_guide/coding_standards.md b/doc/developer_guide/coding_standards.md index 406bc2920..350d8edc1 100644 --- a/doc/developer_guide/coding_standards.md +++ b/doc/developer_guide/coding_standards.md @@ -22,6 +22,7 @@ The workflow definitions of individual CI jobs can be found in the directory [.g - `test_build.yml` builds the project for each of the supported target platforms, i.e., Linux, Windows, and MacOS (see {ref}`compilation`). In the Linux environment, this job does also execute all available unit and integration tests (see {ref}`testing`). It is run for pull requests whenever relevant parts of the project's source code have been modified. - `test_doc.yml` generates the latest documentation (see {ref}`documentation`) whenever relevant parts of the source code are affected by a pull request. - `test_format.yml` ensures that all source files in the project adhere to our coding style guidelines (see {ref}`code-style`). This job is run automatically for pull requests whenever they include any changes affecting the relevant source files. +- `test_changelog.yml` ensures that all changelog files in the project adhere to the structure that is necessary to be processed automatically when publishing a new release. This job is run for pull requests if they modify one of the changelog files. - `merge_feature.yml` and `merge_bugfix.yml` are used to merge changes that have been pushed to the feature or bugfix branch into downstream branches via pull requests (see {ref}`release-process`). - `merge_release.yml` is used to merge all changes included in a new release published on [Github](https://github.com/mrapp-ke/MLRL-Boomer/releases) into upstream branches and update the version numbers of these branches. From 28e3733e33cb8a263d838f9de93a73f3434bc7ad Mon Sep 17 00:00:00 2001 From: michael-rapp <6638695+michael-rapp@users.noreply.github.com> Date: Thu, 7 Nov 2024 00:20:48 +0000 Subject: [PATCH 8/8] [Bot] Merge bugfix into feature branch. --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index a8839f70d..d33c3a212 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -0.11.2 \ No newline at end of file +0.12.0 \ No newline at end of file