Update quality tools.

- Add pyproject-fmt to quality tools.
- Remove safety from quality tools.
- Remove duplication between unittest.sh scripts.
- Remove duplication between quality.sh scripts.
- Remove duplication between pip-compile.sh scripts.
- Remove duplication between pip-install.sh scripts.

Closes #8928.

.circleci/config.yml

@@ -10,7 +10,7 @@ jobs:
         auth:
           username: $DOCKERHUB_USERNAME
           password: $DOCKERHUB_PASSWORD
-    parallelism: 5
+    parallelism: 6
     steps:
       - checkout
       - run: |
@@ -19,7 +19,8 @@ jobs:
             1) component=components/notifier;;
             2) component=components/api_server;;
             3) component=components/shared_code;;
-            4) component=tests/feature_tests;;
+            4) component=tests/application_tests;;
+            5) component=tests/feature_tests;;
           esac
           cd $component
           mkdir -p build
@@ -36,6 +37,10 @@ jobs:
           path: components/api_server/build
       - store_artifacts:
           path: components/shared_code/build
+      - store_artifacts:
+          path: components/application_tests/build
+      - store_artifacts:
+          path: components/feature_tests/build
 
   unittest_frontend:
     docker:
@@ -65,6 +70,18 @@ jobs:
           ci/unittest.sh
           ci/quality.sh
 
+  unittest_release:
+    machine:
+      image: default
+    steps:
+      - checkout
+      - run: |
+          cd release
+          python3 -m venv venv
+          . venv/bin/activate
+          ci/pip-install.sh
+          ci/quality.sh
+
   application_tests:
     machine:
       image: default
@@ -119,6 +136,8 @@ workflows:
           context: QualityTime
       - unittest_docs:
           context: QualityTime
+      - unittest_release:
+          context: QualityTime
       - docker/hadolint:
           context: QualityTime
           dockerfiles: "components/collector/Dockerfile:components/database/Dockerfile:\

.github/workflows/application-tests-quality.yml

@@ -0,0 +1,27 @@
+name: Application tests quality
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    - name: Install dependencies
+      run: |
+        cd tests/application_tests
+        ci/pip-install.sh
+    - name: Test
+      run: |
+        cd tests/application_tests
+        ci/unittest.sh
+    - name: Quality
+      run: |
+        cd tests/application_tests
+        ci/quality.sh

.github/workflows/release-quality.yml

@@ -0,0 +1,22 @@
+name: Release script quality
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    - name: Install dependencies and run quality checks
+      run: |
+        cd release
+        python -m venv venv
+        . venv/bin/activate
+        ci/pip-install.sh
+        ci/quality.sh

ci/base.sh

@@ -4,15 +4,15 @@
 
 set -e
 
-run () {
+run() {
     # Show the invoked command using a subdued text color so it's clear which tool is running.
     header='\033[95m'
     endstyle='\033[0m'
     echo -e "${header}$*${endstyle}"
     eval "$*"
 }
 
-spec () {
+spec() {
     # The versions of tools are specified in pyproject.toml. This function calls the spec.py script which in turn
     # reads the version numbers from the pyproject.toml file.
 
@@ -21,6 +21,13 @@ spec () {
     python $SCRIPT_DIR/spec.py $*
 }
 
+run_pipx() {
+    # Look up the version of the command using the spec function and run the command using pipx.
+    command=$1
+    shift 1
+    run pipx run `spec $command` $@
+}
+
 # Don't install tools in the global pipx home folder, but locally for each component:
 export PIPX_HOME=.pipx
 export PIPX_BIN_DIR=$PIPX_HOME/bin

ci/pip-base.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Get the dir of this script so the base.sh script that is in the same dir as this script can be sourced:
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source $SCRIPT_DIR/base.sh
+
+run_pip_compile() {
+    if [[ -f "requirements/requirements.txt" ]]; then
+        run pip-compile --output-file requirements/requirements.txt pyproject.toml
+    fi
+    run pip-compile --extra dev --output-file requirements/requirements-dev.txt pyproject.toml
+}
+
+run_pip_install() {
+    run pip install --ignore-installed --quiet --use-pep517 $@
+}

ci/python_files_and_folders.py

@@ -0,0 +1,13 @@
+"""Determine the Python files and folders in the current directory."""
+
+from pathlib import Path
+
+def python_files_and_folders() -> list[str]:
+    """Return the Python files and folders in the current directory."""
+    python_files = [python_file.name for python_file in Path(".").glob('*.py') if not python_file.name.startswith(".")]
+    python_folders = [folder_name for folder_name in ("src", "tests") if Path(folder_name).exists()]
+    return python_files + python_folders
+
+
+if __name__ == "__main__":
+    print(" ".join(python_files_and_folders()))

ci/quality-base.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# Get the dir of this script so the base.sh script that is in the same dir as this script can be sourced:
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source $SCRIPT_DIR/base.sh
+
+PYTHON_FILES_AND_FOLDERS=`python $SCRIPT_DIR/python_files_and_folders.py`
+
+run_ruff() {
+    run_pipx ruff check .
+    run_pipx ruff format --check .
+}
+
+run_fixit() {
+    run_pipx fixit lint $PYTHON_FILES_AND_FOLDERS
+}
+
+run_mypy() {
+    run_pipx mypy --python-executable=$(which python) $PYTHON_FILES_AND_FOLDERS
+}
+
+run_mypy_pydantic() {
+    # To use the pydantic plugin, we need to first install mypy and then inject the plugin
+    run pipx install --force `spec mypy`  # --force works around this bug: https://github.com/pypa/pipx/issues/795
+    run pipx inject mypy `spec pydantic`
+    run $PIPX_BIN_DIR/mypy src --python-executable=$(which python)
+}
+
+run_pyproject_fmt() {
+    run_pipx pyproject-fmt --check pyproject.toml
+}
+
+run_bandit() {
+    run_pipx bandit --configfile pyproject.toml --quiet --recursive src
+}
+
+run_pip_audit() {
+    run_pipx pip-audit --strict --progress-spinner=off `python $SCRIPT_DIR/requirements_files.py`
+}
+
+run_vulture() {
+    run_pipx vulture --min-confidence 0 $PYTHON_FILES_AND_FOLDERS .vulture_ignore_list.py $@
+}
+
+run_vale() {
+    run_pipx vale sync
+    run_pipx vale --no-wrap src/*.md
+}
+
+run_markdownlint() {
+    run ./node_modules/markdownlint-cli/markdownlint.js src/*.md
+}
+
+check_python_quality() {
+    run_ruff
+    run_fixit
+    run_mypy
+    run_pyproject_fmt
+    run_pip_audit
+    run_bandit
+    run_vulture
+}

ci/requirements_files.py

@@ -0,0 +1,13 @@
+"""Determine the Python requirements files."""
+
+from pathlib import Path
+
+def requirements_files() -> list[str]:
+    """Return the Python requirements files in the requirements directory."""
+    # We never return the internal requirements file, because it does not need to be checked
+    requirements_filenames = ("requirements/requirements.txt", "requirements/requirements-dev.txt")
+    return [f"-r {filename}" for filename in requirements_filenames if Path(filename).exists()]
+
+
+if __name__ == "__main__":
+    print(" ".join(requirements_files()))

ci/unittest-base.sh

@@ -1,8 +1,15 @@
 #!/bin/bash
 
-# Get the dir of this script so the vbase.sh script that is in the same dir as this script can be sourced:
+# Get the dir of this script so the base.sh script that is in the same dir as this script can be sourced:
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 source $SCRIPT_DIR/base.sh
 
 # Turn on development mode, see https://docs.python.org/3/library/devmode.html
 export PYTHONDEVMODE=1
+
+run_coverage() {
+    run coverage run -m unittest --quiet
+    run coverage report --fail-under=0
+    run coverage html --fail-under=0
+    run coverage xml  # Fail if coverage is too low, but only after the text and HTML reports have been generated
+}

components/api_server/ci/pip-compile.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 
-source ../../ci/base.sh
+source ../../ci/pip-base.sh
 
 # Update the compiled requirements files
-run pip-compile --output-file requirements/requirements.txt pyproject.toml
-run pip-compile --extra dev --output-file requirements/requirements-dev.txt pyproject.toml
+run_pip_compile

components/api_server/ci/pip-install.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-source ../../ci/base.sh
+source ../../ci/pip-base.sh
 
 # Install the requirements
-run pip install --ignore-installed --quiet --use-pep517 -r requirements/requirements-dev.txt
-run pip install --ignore-installed --quiet --use-pep517 -r requirements/requirements-internal.txt
+run_pip_install -r requirements/requirements-dev.txt
+run_pip_install -r requirements/requirements-internal.txt

components/api_server/ci/quality.sh

@@ -1,31 +1,5 @@
 #!/bin/bash
 
-source ../../ci/base.sh
+source ../../ci/quality-base.sh
 
-# Ruff
-run pipx run `spec ruff` check .
-run pipx run `spec ruff` format --check .
-
-# Fixit
-run pipx run `spec fixit` lint src tests
-
-# Mypy
-run pipx run `spec mypy` --python-executable=$(which python) src
-
-# pip-audit
-run pipx run `spec pip-audit` --strict --progress-spinner=off -r requirements/requirements.txt -r requirements/requirements-dev.txt
-
-# Safety
-# Vulnerability ID: 67599
-# ADVISORY: ** DISPUTED ** An issue was discovered in pip (all versions) because it installs the version with the
-# highest version number, even if the user had intended to obtain a private package from a private index. This only
-# affects use of the --extra-index-url option, and exploitation requires that the...
-# CVE-2018-20225
-# For more information about this vulnerability, visit https://data.safetycli.com/v/67599/97c
-run pipx run `spec safety` check --bare --ignore 67599 -r requirements/requirements.txt -r requirements/requirements-dev.txt
-
-# Bandit
-run pipx run `spec bandit` --quiet --recursive src/
-
-# Vulture
-run pipx run `spec vulture` --min-confidence 0 src/ tests/ .vulture_ignore_list.py
+check_python_quality

components/api_server/ci/unittest.sh

@@ -4,7 +4,4 @@ source ../../ci/unittest-base.sh
 
 export COVERAGE_RCFILE=../../.coveragerc
 
-coverage run -m unittest --quiet
-coverage report --fail-under=0
-coverage html --fail-under=0
-coverage xml  # Fail if coverage is too low, but only after the text and HTML reports have been generated
+run_coverage