Modularize build system

.changelog-bugfix.md

@@ -1,5 +1,6 @@
 # Quality-of-Life Improvements
 
+- The build system now uses a lightweight custom implementation instead of [SCons](https://scons.org/) and is better modularized to avoid unnecessary runs of Continuous Integration jobs when only certain parts of it are modified.
 - Releases are now automated via Continuous Integration, including the update of the project's changelog.
 - The presentation of algorithmic parameters in the documentation has been improved.
 - Outdated GitHub Actions can now be printed via the build target `check_github_actions`. Alternatively, the build target `update_github_actions` may be used to update them automatically.

.github/workflows/test_build.yml

@@ -35,9 +35,16 @@ jobs:
               - '.github/workflows/test_build.yml'
               - 'build'
               - 'build.bat'
-              - 'scons/**'
+              - 'build_system/main.py'
+              - 'build_system/core/**'
+              - 'build_system/util/**'
+              - 'build_system/targets/paths.py'
+              - 'build_system/targets/compilation/*'
+              - 'build_system/targets/testing/*'
             cpp: &cpp
               - *build_files
+              - 'build_system/targets/compilation/cpp/*'
+              - 'build_system/targets/testing/cpp/*'
               - 'cpp/**/include/**'
               - 'cpp/**/src/**'
               - '**/*.pxd'
@@ -48,6 +55,8 @@ jobs:
               - 'cpp/**/test/**'
             python: &python
               - *build_files
+              - 'build_system/targets/compilation/cython/*'
+              - 'build_system/targets/testing/python/*'
               - 'python/requirements.txt'
               - 'python/**/mlrl/**'
             python_tests: &python_tests

.github/workflows/test_changelog.yml

@@ -33,7 +33,11 @@ jobs:
               - '.github/workflows/test_changelog.yml'
               - 'build'
               - 'build.bat'
-              - 'scons/**'
+              - 'build_system/main.py'
+              - 'build_system/core/**'
+              - 'build_system/util/**'
+              - 'build_system/targets/paths.py'
+              - 'build_system/targets/versioning/*'
             bugfix:
               - *build_files
               - '.changelog-bugfix.md'

.github/workflows/test_doc.yml

@@ -34,15 +34,21 @@ jobs:
               - '.github/workflows/test_doc.yml'
               - 'build'
               - 'build.bat'
-              - 'scons/**'
+              - 'build_system/main.py'
+              - 'build_system/core/**'
+              - 'build_system/util/**'
+              - 'build_system/targets/paths.py'
             cpp: &cpp
               - *build_files
+              - 'build_system/targets/documentation/cpp/*'
               - 'cpp/**/include/**'
             python: &python
               - *build_files
+              - 'build_system/targets/documentation/python/*'
               - 'python/**/mlrl/**'
             doc: &doc
               - *build_files
+              - 'build_system/targets/documentation/*'
               - 'doc/**'
             any:
               - *cpp

.github/workflows/test_format.yml

@@ -33,23 +33,31 @@ jobs:
               - '.github/workflows/test_format.yml'
               - 'build'
               - 'build.bat'
-              - 'scons/**'
+              - 'build_system/main.py'
+              - 'build_system/core/**'
+              - 'build_system/util/**'
+              - 'build_system/targets/paths.py'
             cpp:
               - *build_files
+              - 'build_system/targets/code_style/*'
+              - 'build_system/targets/code_style/cpp/*'
+              - '.cpplint.cfg'
               - '**/*.hpp'
               - '**/*.cpp'
-              - '.clang-format'
             python:
               - *build_files
+              - 'build_system/targets/code_style/*'
+              - 'build_system/targets/code_style/python/*'
               - '**/*.py'
-              - '.isort.cfg'
-              - '.pylintrc'
-              - '.style.yapf'
             md:
               - *build_files
+              - 'build_system/targets/code_style/*'
+              - 'build_system/targets/code_style/markdown/*'
               - '**/*.md'
             yaml:
               - *build_files
+              - 'build_system/targets/code_style/*'
+              - 'build_system/targets/code_style/yaml/*'
               - '**/*.y*ml'
       - name: Check C++ code style
         if: steps.filter.outputs.cpp == 'true'

.github/workflows/test_publish.yml

@@ -36,7 +36,11 @@ jobs:
               - '.github/workflows/template_publish_pure.yml'
               - 'build'
               - 'build.bat'
-              - 'scons/**'
+              - 'build_system/main.py'
+              - 'build_system/core/**'
+              - 'build_system/util/**'
+              - 'build_system/targets/paths.py'
+              - 'build_system/targets/packaging/*'
       - name: Read Python version
         uses: juliangruber/read-file-action@v1
         id: python_version

.gitignore

@@ -1,6 +1,6 @@
 # Build files
 __pycache__/
-scons/build/
+build_system/build/
 python/**/build/
 python/**/dist/
 python/**/*egg-info/

build

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 VENV_DIR="venv"
-SCONS_DIR="scons"
+BUILD_SYSTEM_DIR="build_system"
 CLEAN=false
 
 set -e
@@ -22,13 +22,12 @@ fi
 
 if [ -d "$VENV_DIR" ]; then
     . $VENV_DIR/bin/activate
-    python3 -c "import sys; sys.path.append('$SCONS_DIR'); import dependencies; dependencies.install_build_dependencies('scons')"
-    scons --silent --file $SCONS_DIR/sconstruct.py $@
+    python3 $BUILD_SYSTEM_DIR/main.py $@
     deactivate
 fi
 
 if [ $CLEAN = true ] && [ -d $VENV_DIR ]; then
     echo "Removing virtual Python environment..."
     rm -rf $VENV_DIR
-    rm -rf $SCONS_DIR/build
+    rm -rf $BUILD_SYSTEM_DIR/build
 fi

build.bat

@@ -1,7 +1,7 @@
 @echo off
 
 set "VENV_DIR=venv"
-set "SCONS_DIR=scons"
+set "BUILD_SYSTEM_DIR=build_system"
 set "CLEAN=false"
 
 if not "%1"=="" if "%2"=="" (
@@ -20,16 +20,15 @@ if not exist "%VENV_DIR%" (
 
 if exist "%VENV_DIR%" (
     call %VENV_DIR%\Scripts\activate || exit
-    .\%VENV_DIR%\Scripts\python -c "import sys;sys.path.append('%SCONS_DIR%');import dependencies;dependencies.install_build_dependencies('scons')" || exit
-    .\%VENV_DIR%\Scripts\python -m SCons --silent --file %SCONS_DIR%\sconstruct.py %* || exit
+    .\%VENV_DIR%\Scripts\python %BUILD_SYSTEM_DIR%\main.py %* || exit
     call deactivate || exit
 )
 
 if "%CLEAN%"=="true" if exist "%VENV_DIR%" (
     echo Removing virtual Python environment...
     rd /s /q "%VENV_DIR%" || exit
 
-    if exist "%SCONS_DIR%\build" (
-        rd /s /q "%SCONS_DIR%\build" || exit
+    if exist "%BUILD_SYSTEM_DIR%\build" (
+        rd /s /q "%BUILD_SYSTEM_DIR%\build" || exit
     )
 )

build_system/core/build_unit.py

@@ -0,0 +1,57 @@
+"""
+Author: Michael Rapp ([email protected])
+
+Provides classes that provide information about independent units of the build system.
+"""
+from os import path
+from typing import List
+
+
+class BuildUnit:
+    """
+    An independent unit of the build system that may come with its own built-time dependencies.
+    """
+
+    BUILD_SYSTEM_DIRECTORY = 'build_system'
+
+    BUILD_DIRECTORY_NAME = 'build'
+
+    def __init__(self, root_directory: str = BUILD_SYSTEM_DIRECTORY):
+        """
+        :param root_directory: The root directory of this unit
+        """
+        self.root_directory = root_directory
+
+    @staticmethod
+    def for_file(file) -> 'BuildUnit':
+        """
+        Creates and returns a `BuildUnit` for a given file.
+
+        :param file:    The file for which a `BuildUnit` should be created
+        :return:        The `BuildUnit` that has been created
+        """
+        return BuildUnit(path.relpath(path.dirname(file), path.dirname(BuildUnit.BUILD_SYSTEM_DIRECTORY)))
+
+    @property
+    def build_directory(self) -> str:
+        """
+        The path to the build directory of this unit.
+        """
+        return path.join(self.root_directory, self.BUILD_DIRECTORY_NAME)
+
+    def find_requirements_files(self) -> List[str]:
+        """
+        The path to the requirements file that specifies the build-time dependencies of this unit.
+        """
+        requirements_files = []
+        current_directory = self.root_directory
+
+        while path.basename(current_directory) != self.BUILD_SYSTEM_DIRECTORY:
+            requirements_file = path.join(current_directory, 'requirements.txt')
+
+            if path.isfile(requirements_file):
+                requirements_files.append(requirements_file)
+
+            current_directory = path.dirname(current_directory)
+
+        return requirements_files

build_system/core/changes.py

@@ -0,0 +1,132 @@
+"""
+Author: Michael Rapp ([email protected])
+
+Provides classes for detecting changes in files.
+"""
+import json
+
+from functools import cached_property
+from os import path
+from typing import Dict, List, Set
+
+from core.modules import Module
+from util.io import TextFile, create_directories
+
+
+class JsonFile(TextFile):
+    """
+    Allows to read and write the content of a JSON file.
+    """
+
+    @cached_property
+    def json(self) -> Dict:
+        """
+        The content of the JSON file as a dictionary.
+        """
+        lines = self.lines
+
+        if lines:
+            return json.loads('\n'.join(lines))
+
+        return {}
+
+    def write_json(self, dictionary: Dict):
+        """
+        Writes a given dictionary to the JSON file.
+
+        :param dictionary: The dictionary to be written
+        """
+        self.write_lines(json.dumps(dictionary, indent=4))
+
+    def write_lines(self, *lines: str):
+        super().write_lines(*lines)
+
+        try:
+            del self.json
+        except AttributeError:
+            pass
+
+
+class ChangeDetection:
+    """
+    Allows to detect changes in tracked files.
+    """
+
+    class CacheFile(JsonFile):
+        """
+        A JSON file that stores checksums for tracked files.
+        """
+
+        @staticmethod
+        def __checksum(file: str) -> str:
+            return str(path.getmtime(file))
+
+        def __init__(self, file: str):
+            """
+            :param file: The path to the JSON file
+            """
+            super().__init__(file, accept_missing=True)
+            create_directories(path.dirname(file))
+
+        def update(self, module_name: str, files: Set[str]):
+            """
+            Updates the checksums of given files.
+
+            :param module_name: The name of the module, the files belong to
+            :param files:       A set that contains the paths of the files to be updated
+            """
+            cache = self.json
+            module_cache = cache.setdefault(module_name, {})
+
+            for invalid_key in [file for file in module_cache.keys() if file not in files]:
+                del module_cache[invalid_key]
+
+            for file in files:
+                module_cache[file] = self.__checksum(file)
+
+            if module_cache:
+                cache[module_name] = module_cache
+            else:
+                del cache[module_name]
+
+            if cache:
+                self.write_json(cache)
+            else:
+                self.delete()
+
+        def has_changed(self, module_name: str, file: str) -> bool:
+            """
+            Returns whether a file has changed according to the cache or not.
+
+            :param module_name: The name of the module, the file belongs to
+            :param file:        The file to be checked
+            :return:            True, if the file has changed, False otherwise
+            """
+            module_cache = self.json.get(module_name, {})
+            return file not in module_cache or module_cache[file] != self.__checksum(file)
+
+    def __init__(self, cache_file: str):
+        """
+        :param cache_file: The path to the file that should be used for tracking files
+        """
+        self.cache_file = ChangeDetection.CacheFile(cache_file)
+
+    def track_files(self, module: Module, *files: str):
+        """
+        Updates the cache to keep track of given files.
+
+        :param module:  The module, the files belong to
+        :param files:   The files to be tracked
+        """
+        self.cache_file.update(str(module), set(files))
+
+    def get_changed_files(self, module: Module, *files: str) -> List[str]:
+        """
+        Filters given files and returns only those that have changed.
+
+        :param module:  The module, the files belong to
+        :param files:   The files to be filtered
+        :return:        A list that contains the files that have changed
+        """
+        module_name = str(module)
+        return [file for file in files if self.cache_file.has_changed(module_name, file)]