Add support for constraints

.devcontainer/devcontainer.json

@@ -42,5 +42,5 @@
     // Mount the parent as /workspaces so we can pip install peers as editable
     "workspaceMount": "source=${localWorkspaceFolder}/..,target=/workspaces,type=bind",
     // After the container is created, install the python project in editable form
-    "postCreateCommand": "pip install -e '.[dev]'"
+    "postCreateCommand": "pip install $([ -f dev-requirements.txt ] && echo '-c dev-requirements.txt') -e '.[dev]'"
 }

.github/actions/install_requirements/action.yml

@@ -6,7 +6,7 @@ inputs:
     default: "dev"
   pip-install:
     description: Parameters to pass to pip install
-    default: "-e .[dev]"
+    default: "$([ -f dev-requirements.txt ] && echo '-c dev-requirements.txt') -e .[dev]"
 
 runs:
   using: composite

.github/workflows/_docs.yml

@@ -28,23 +28,20 @@ jobs:
       - name: Remove environment.pickle
         run: rm build/html/.doctrees/environment.pickle
 
-      - name: Sanitize ref name for docs version
-        run: echo "DOCS_VERSION=${GITHUB_REF_NAME//[^A-Za-z0-9._-]/_}" >> $GITHUB_ENV
-
-      - name: Move to versioned directory
-        run: mv build/html build/$DOCS_VERSION
-
       - name: Upload built docs artifact
         uses: actions/upload-artifact@v4
         with:
           name: docs
           path: build
 
-      - name: Add other static pages files
-        run: cp .github/pages/* build
+      - name: Sanitize ref name for docs version
+        run: echo "DOCS_VERSION=${GITHUB_REF_NAME//[^A-Za-z0-9._-]/_}" >> $GITHUB_ENV
+
+      - name: Move to versioned directory
+        run: mv build/html .github/pages/$DOCS_VERSION
 
       - name: Write switcher.json
-        run: python .github/pages/make_switcher.py --add $DOCS_VERSION ${{ github.repository }} build/switcher.json
+        run: python .github/pages/make_switcher.py --add $DOCS_VERSION ${{ github.repository }} .github/pages/switcher.json
 
       - name: Publish Docs to gh-pages
         if: github.ref_type == 'tag' || github.ref_name == 'main'
@@ -53,5 +50,5 @@ jobs:
         uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: build
-          keep_files: true
+          publish_dir: .github/pages
+          keep_files: true

.github/workflows/_release.yml

@@ -8,15 +8,16 @@ jobs:
     steps:
       - name: Download artifacts
         uses: actions/download-artifact@v4
+        with:
+          merge-multiple: true
 
-      - name: Prepare release files
+      - name: Zip up docs
         run: |
-          if [ -d docs ]; then
-            cd docs && zip -r docs.zip *
-            echo 'DOCS=docs/docs.zip' >> $GITHUB_ENV
-          fi
-          if [ -d dist ]; then
-            echo 'DIST=dist/*' >> $GITHUB_ENV
+          set -vxeuo pipefail
+          if [ -d html ]; then
+            mv html $GITHUB_REF_NAME
+            zip -r docs.zip $GITHUB_REF_NAME
+            rm -rf $GITHUB_REF_NAME
           fi
 
       - name: Create GitHub Release
@@ -25,9 +26,7 @@ jobs:
         uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v0.1.15
         with:
           prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }}
-          files: |
-            ${{ env.DOCS }}
-            ${{ env.DIST }}
+          files: "*"
           generate_release_notes: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/_test.yml

@@ -32,6 +32,17 @@ jobs:
         name: Install dev versions of python packages
         uses: ./.github/actions/install_requirements
 
+      - if: inputs.python-version == 'dev'
+        name: Write the requirements as an artifact
+        run: pip freeze --exclude-editable > /tmp/dev-requirements.txt
+
+      - if: inputs.python-version == 'dev'
+        name: Upload dev-requirements.txt
+        uses: actions/upload-artifact@v4
+        with:
+          name: dev-requirements
+          path: /tmp/dev-requirements.txt
+
       - if: inputs.python-version != 'dev'
         name: Install latest versions of python packages
         uses: ./.github/actions/install_requirements

docs/how-to/lock-requirements.md

@@ -0,0 +1,39 @@
+# Lock requirements
+
+## Introduction
+
+By design this project only defines dependencies in one place, i.e. in the `requires` table in `pyproject.toml`.
+
+In the `requires` table it is possible to pin versions of some dependencies as needed. For library projects it is best to leave  pinning to a minimum so that your library can be used by the widest range of applications.
+
+When CI builds the project it will use the latest compatible set of dependencies available (after applying your pins and any dependencies' pins).
+
+This approach means that there is a possibility that a future build may break because an updated release of a dependency has made a breaking change.
+
+The correct way to fix such an issue is to work out the minimum pinning in `requires` that will resolve the problem. However this can be quite hard to do and may be time consuming when simply trying to release a minor update.
+
+For this reason we provide a mechanism for locking all dependencies to the same version as a previous successful release. This is a quick fix that should guarantee a successful CI build.
+
+## Finding the lock files
+
+Every release of the project will have a set of requirements files published as release assets.
+
+For example take a look at the release page for python-copier-template [here](https://github.com/DiamondLightSource/python-copier-template/releases/tag/1.1.0)
+
+There is a single `dev-requirements.txt` file showing as an asset on the release. This has been created using `pip freeze --exclude-editable` on a successful test run using the same version of python as the devcontainer, and will contain a full list of the dependencies and sub-dependencies with pinned versions. You can download this file by clicking on it.
+
+## Applying the lock file
+
+To apply a lockfile:
+
+- copy the requirements file you have downloaded to the root of your repository
+- commit it into the repo
+- push the changes
+
+The CI looks for a `dev-requirements.txt` in the root and will pass it to pip as a constraint when installing the dev environment. If a package is required to be installed by `pyproject.toml` then `pip` will use the version specified in `dev-requirements.txt`.
+
+## Removing dependency locking from CI
+
+Once the reasons for locking the build have been resolved it is a good idea to go back to an unlocked build. This is because you get an early indication of any incoming problems.
+
+To restore unlocked builds in CI simply remove `dev-requirements.txt` from the root of the repo and push.