test: Add mutation testing with Cosmic Ray

.github/workflows/python-mutation-tests.yml

@@ -0,0 +1,50 @@
+name: Python mutation tests
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '30 3 * * *'
+  # Todo; remove the push, they take to long
+  push:
+    branches:
+      - add-mutation-testing-with-cosmic-ray
+
+jobs:
+  tests:
+    name: Python mutation tests
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Display Python version
+        run: python --version
+
+      - name: Install dependencies
+        run: |
+          pip install poetry
+          poetry install
+
+      - name: Run mutation tests
+        run: |
+          # Todo: Can we make it faster in GitHub Actions
+          # Todo: Drop the --force, it just here for easy local copy and paste
+          poetry run cosmic-ray init setup.cfg cosmic-ray.sqlite --force
+          echo "Initialised"
+          poetry run cosmic-ray --verbosity INFO exec setup.cfg cosmic-ray.sqlite
+
+      - name: Report results
+        # Todo: Only show failures https://github.com/sixty-north/cosmic-ray/issues/561
+        run: poetry run cr-report cosmic-ray.sqlite --show-diff
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}

.github/workflows/unit-tests.yml → .github/workflows/python-unit-tests.yml

@@ -1,4 +1,4 @@
-name: Unit tests
+name: Python unit tests
 
 on: [push]
 

.gitignore

@@ -26,6 +26,11 @@ __py_cache__/
 
 .pytest_cache/
 
+# Mutation testing
+cosmic-ray.sqlite
+cosmic-ray-worker*
+mutants
+
 staticfiles/
 node_modules/
 

README.md

@@ -63,8 +63,17 @@ To allow pdb to work correctly, disable multiple processes using the `--numproce
 
 `poetry run pytest --numprocesses 0`
 
+##### Monitoring the quality of our Python unit tests
+
 We use [Codecov](https://app.codecov.io/github/uktrade/platform-tools) to monitor the comprehensiveness and performance of our unit tests.
 
+We use [Cosmic Ray](https://cosmic-ray.readthedocs.io/en/latest/index.html) to run [Python mutation tests in GitHub Actions](https://github.com/uktrade/platform-tools/actions/workflows/python-mutation-tests.yml) on a schedule to help identify areas of our code which are not covered by our tests.
+
+You can run the mutation tests locally, but:
+
+- they take a very long time to run as they run the whole unit test suite for each mutation.
+- they actually your source files while they run, so you cannot work on the code until it's finished.
+
 #### Manual testing
 
 You may want to test any CLI changes locally.

pyproject.toml

@@ -37,7 +37,6 @@ requests = "^2.31.0"
 prettytable = "^3.9.0"
 semver = "^3.0.2"
 tomlkit = "^0.12.2"
-checkov = "^3.1.67"
 slack-sdk = "^3.27.1"
 jsonschema = "~=4.17.0"
 cfn-lint = "^1.4.2"
@@ -53,6 +52,8 @@ freezegun = "^1.2.2"
 parameterized = "^0.9.0"
 pytest-xdist = "^3.5.0"
 pytest-cov = "^6.0.0"
+cosmic-ray = "^8.4.1"
+mutmut = "^3.2.1"
 
 [build-system]
 requires = ["poetry-core"]

setup.cfg

@@ -0,0 +1,22 @@
+[cosmic-ray]
+module-path = "utils"
+timeout = 10.0
+excluded-modules = ["**/node_modules/**/*"]
+test-command = "poetry run pytest tests/utils"
+
+[cosmic-ray.distributor]
+name = "local"
+# name = "http"
+
+# [cosmic-ray.distributor.http]
+# worker-urls = [
+#     "http://localhost:9876",
+#     "http://localhost:9877",
+#     "http://localhost:9878",
+#     "http://localhost:9879"
+# ]
+
+[mutmut]
+paths_to_mutate = "utils/,dbt_platform_helper"
+runner = "poetry run pytest tests/utils"
+tests_dir = "tests/"

tests/platform_helper/conftest.py

@@ -39,6 +39,7 @@ def __init__(self):
 @pytest.fixture
 def fakefs(fs):
     """Mock file system fixture with the templates and schemas dirs retained."""
+    print(f"BASE_DIR: {BASE_DIR}")
     fs.add_real_directory(BASE_DIR / "dbt_platform_helper/templates", lazy_read=True)
     fs.add_real_directory(FIXTURES_DIR, lazy_read=True)
     fs.add_real_file(BASE_DIR / "dbt_platform_helper/addon-plans.yml")