Split CI into multiple yaml files (#43)

.devcontainer/devcontainer.json

@@ -7,7 +7,7 @@
     },
     "remoteEnv": {
         // Allow X11 apps to run inside the container
-        "DISPLAY": "${localEnv:DISPLAY}",
+        "DISPLAY": "${localEnv:DISPLAY}"
     },
     "customizations": {
         "vscode": {
@@ -22,7 +22,8 @@
                 "tamasfe.even-better-toml",
                 "redhat.vscode-yaml",
                 "ryanluker.vscode-coverage-gutters",
-                "charliermarsh.ruff"
+                "charliermarsh.ruff",
+                "ms-azuretools.vscode-docker"
             ]
         }
     },
@@ -41,5 +42,5 @@
     // Mount the parent as /workspaces so we can pip install peers as editable
     "workspaceMount": "source=${localWorkspaceFolder}/..,target=/workspaces,type=bind",
     // After the container is created, install the python project in editable form
-    "postCreateCommand": "pip install -c requirements/constraints.txt -e '.[dev]'"
+    "postCreateCommand": "pip install -e '.[dev]'"
 }

.github/actions/install_requirements/action.yml

@@ -0,0 +1,34 @@
+name: Install requirements
+description: Install a version of python then call pip install and report what was installed
+inputs:
+  python-version:
+    description: Python version to install, default is from Dockerfile
+    default: "dev"
+  pip-install:
+    description: Parameters to pass to pip install
+    default: "-e .[dev]"
+
+runs:
+  using: composite
+  steps:
+    - name: Get version of python
+      run: |
+        PYTHON_VERSION="${{ inputs.python-version }}"
+        if [ $PYTHON_VERSION == "dev" ]; then
+          PYTHON_VERSION=$(sed -n "s/ARG PYTHON_VERSION=//p" Dockerfile)
+        fi
+        echo "PYTHON_VERSION=$PYTHON_VERSION" >> "$GITHUB_ENV"
+      shell: bash
+
+    - name: Setup python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+
+    - name: Install packages
+      run: pip install ${{ inputs.pip-install }}
+      shell: bash
+
+    - name: Report what was installed
+      run: pip freeze
+      shell: bash

.github/workflows/_check.yml

@@ -1,15 +1,15 @@
 on:
   workflow_call:
     outputs:
-      branch-pr:
+      not-in-pr:
         description: The PR number if the branch is in one
-        value: ${{ jobs.pr.outputs.branch-pr }}
+        value: ${{ jobs.pr.outputs.not-in-pr }}
 
 jobs:
   pr:
     runs-on: "ubuntu-latest"
     outputs:
-      branch-pr: ${{ steps.script.outputs.result }}
+      not-in-pr: ${{ steps.script.outputs.result }}
     steps:
       - uses: actions/github-script@v7
         id: script
@@ -23,5 +23,6 @@ jobs:
             })
             if (prs.data.length) {
               console.log(`::notice ::Skipping CI on branch push as it is already run in PR #${prs.data[0]["number"]}`)
-              return prs.data[0]["number"]
+            } else {
+              return "not-in-pr"
             }

.github/workflows/_container.yml

@@ -0,0 +1,56 @@
+on:
+  workflow_call:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # Need this to get version number from last tag
+          fetch-depth: 0
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Docker Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and export to Docker local cache
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          # Need load and tags so we can test it below
+          load: true
+          tags: tag_for_testing
+
+      - name: Test cli works in cached runtime image
+        run: docker run --rm tag_for_testing --version
+
+      - name: Create tags for publishing image
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tags: |
+            type=ref,event=tag
+            type=raw,value=latest
+
+      - name: Push cached image to container registry
+        if: github.ref_type == 'tag'
+        uses: docker/build-push-action@v5
+        # This does not build the image again, it will find the image in the
+        # Docker cache and publish it
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/_dist.yml

@@ -0,0 +1,36 @@
+on:
+  workflow_call:
+
+jobs:
+  build:
+    runs-on: "ubuntu-latest"
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # Need this to get version number from last tag
+          fetch-depth: 0
+
+      - name: Build sdist and wheel
+        run: >
+          export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) &&
+          pipx run build
+
+      - name: Upload sdist and wheel as artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist
+
+      - name: Check for packaging errors
+        run: pipx run twine check --strict dist/*
+
+      - name: Install produced wheel
+        uses: ./.github/actions/install_requirements
+        with:
+          pip-install: dist/*.whl
+
+      - name: Test module --version works using the installed wheel
+        # If more than one module in src/ replace with module name to test
+        run: python -m $(ls --hide='*.egg-info' src | head -1) --version

.github/workflows/_docs.yml

@@ -9,18 +9,14 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          submodules: true
+          # Need this to get version number from last tag
+          fetch-depth: 0
 
       - name: Install system packages
         run: sudo apt-get install graphviz
 
-      - name: Setup python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
-
       - name: Install python packages
-        run: pip install -c requirements/constraints.txt -e .[dev]
+        uses: ./.github/actions/install_requirements
 
       - name: Build docs
         run: tox -e docs

.github/workflows/_pypi.yml

@@ -0,0 +1,19 @@
+on:
+  workflow_call:
+
+jobs:
+  upload:
+    runs-on: ubuntu-latest
+    environment: release
+
+    steps:
+      - name: Download dist artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist
+
+      - name: Publish to PyPI using trusted publishing
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_TOKEN }}

.github/workflows/_test.yml

@@ -0,0 +1,46 @@
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        type: string
+        description: The version of python to install
+        required: true
+      runs-on:
+        type: string
+        description: The runner to run this job on
+        required: true
+
+env:
+  # https://github.com/pytest-dev/pytest/issues/2042
+  PY_IGNORE_IMPORTMISMATCH: "1"
+
+jobs:
+  run:
+    runs-on: ${{ inputs.runs-on }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # Need this to get version number from last tag
+          fetch-depth: 0
+
+      - if: inputs.python-version == 'dev'
+        name: Install dev versions of python packages
+        uses: ./.github/actions/install_requirements
+
+      - if: inputs.python-version != 'dev'
+        name: Install latest versions of python packages
+        uses: ./.github/actions/install_requirements
+        with:
+          python-version: ${{ inputs.python-version }}
+          pip-install: ".[dev]"
+
+      - name: Run tests
+        run: tox -e pytest
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          name: ${{ inputs.python-version }}/${{ inputs.runs-on }}
+          files: cov.xml

.github/workflows/_tox.yml

@@ -14,16 +14,9 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Setup python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
 
       - name: Install python packages
-        run: pip install -c requirements/constraints.txt -e .[dev]
+        uses: ./.github/actions/install_requirements
 
       - name: Run tox
         run: tox -e ${{ inputs.tox }}

.github/workflows/ci.yml

@@ -6,28 +6,31 @@ on:
 
 jobs:
   check:
-    # TODO: Use the CI straight from the template
     uses: ./.github/workflows/_check.yml
 
-  # TODO: Use the CI straight from the template
-  test:
+  lint:
     needs: check
-    if: ${{ ! needs.check.outputs.branch-pr }}
+    if: needs.check.outputs.not-in-pr
     uses: ./.github/workflows/_tox.yml
     with:
-      tox: pre-commit,pytest
+      tox: pre-commit
+
+  test:
+    needs: check
+    if: needs.check.outputs.not-in-pr
+    uses: ./.github/workflows/_test.yml
+    with:
+      python-version: dev
+      runs-on: ubuntu-latest
 
   docs:
     needs: check
-    if: ${{ ! needs.check.outputs.branch-pr }}
+    if: needs.check.outputs.not-in-pr
     uses: ./.github/workflows/_docs.yml
-    permissions:
-      contents: write
 
   release:
     if: github.ref_type == 'tag'
     needs: docs
-    # TODO: Use the CI straight from the template
     uses: ./.github/workflows/_release.yml
     permissions:
       contents: write

.github/workflows/periodic.yml

@@ -0,0 +1,13 @@
+name: Periodic
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Run weekly to check URL links still resolve
+    - cron: "0 8 * * WED"
+
+jobs:
+  linkcheck:
+    uses: ./.github/workflows/_tox.yml
+    with:
+      tox: docs build -- -b linkcheck

.pre-commit-config.yaml

@@ -21,10 +21,3 @@ repos:
         entry: ruff format --force-exclude
         types: [python]
         require_serial: true
-
-      - id: constraints
-        name: check constraints match installed
-        language: system
-        entry: ./requirements/regenerate.sh --if-non-empty
-        files: ^(pyproject\.toml|requirements/constraints\.txt)$
-        pass_filenames: false

Dockerfile

@@ -1,12 +1,13 @@
-# This file is for use as a devcontainer
-#
 # The devcontainer should use the developer target and run as root with podman
 # or docker with user namespaces.
 ARG PYTHON_VERSION=3.11
 FROM python:${PYTHON_VERSION} as developer
-RUN apt-get update && apt-get upgrade -y && \
-    apt-get install -y --no-install-recommends \
+
+# Add any system dependencies for the developer/build environment here
+RUN apt-get update && apt-get install -y --no-install-recommends \
     graphviz \
     && rm -rf /var/lib/apt/lists/*
+
+# Set up a virtual environment and put it in PATH
 RUN python -m venv /venv
-ENV PATH=/venv/bin:$PATH
+ENV PATH=/venv/bin:$PATH

copier.yml

@@ -76,8 +76,7 @@ docs_type:
         - README
         - sphinx
 
-_subdirectory: "template"
-
+# Internal variables
 repo_url:
     type: str
     default: "https://github.com/{{github_org}}/{{repo_name}}"
@@ -88,5 +87,12 @@ docs_url:
     default: "https://{{github_org | lower}}.github.io/{{repo_name}}"
     when: false
 
+sphinx:
+    type: bool
+    default: "{{ docs_type == 'sphinx' }}"
+    when: false
+
+_subdirectory: "template"
+
 _tasks:
     - "git init --initial-branch=main"