Add layer of security to conda and pip uploading

.github/workflows/conda-build.yml

@@ -17,6 +17,10 @@ on:
         required: false
         type: string
         default: ${{ github.ref_name }}
+      environment:
+        description: "GitHub environment in which secrets are stored"
+        type: string
+        default: "release"
     secrets:
       ANACONDA_TOKEN:
         required: true
@@ -44,6 +48,7 @@ jobs:
 
   token-exists:
     runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
     steps:
     - name: check if ANACONDA_TOKEN exists
       env:
@@ -58,6 +63,7 @@ jobs:
 
   conda-build:
     runs-on: ${{ matrix.os }}
+    environment: ${{ inputs.environment }}
     needs: [n-builds, token-exists]
     strategy:
       matrix: ${{ fromJson(needs.n-builds.outputs.matrix) }}

.github/workflows/conda-upload.yml

@@ -15,6 +15,10 @@ on:
         required: false
         type: string
         default: ""
+      environment:
+        description: "GitHub environment in which secrets are stored"
+        type: string
+        default: "release"
     secrets:
       ANACONDA_TOKEN:
         required: true
@@ -26,6 +30,7 @@ defaults:
 jobs:
   conda-publish:
     runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
     env:
       PACKAGENAME: "conda-build[a-zA-Z0-9-.]*-${{ inputs.package_name }}-${{ inputs.version }}"
     steps:

.github/workflows/pip-build.yml

@@ -12,6 +12,10 @@ on:
         required: false
         type: string
         default: ${{ github.ref_name }}
+      environment:
+        description: "GitHub environment in which secrets are stored"
+        type: string
+        default: "pre-release"
     secrets:
       TEST_PYPI_API_TOKEN:
         required: true
@@ -50,6 +54,62 @@ jobs:
     - name: Build package
       run: python -m build
 
+    - name: Create built package artifact ready for upload on release
+      uses: actions/upload-artifact@v3
+      with:
+        name: pip-build-${{ inputs.package_name }}-${{ inputs.version }}
+        path: dist/*
+        retention-days: 7
+
+  pip-test-build:  # check local install succeeds before uploading to test-pypi
+    needs: [pip-build]
+    if: needs.pip-build.result == 'success'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: mamba-org/setup-micromamba@v1
+      with:
+        micromamba-version: latest
+        environment-name: pipbuild
+        create-args: >-
+          python=3.11
+          pip
+          build
+        post-cleanup: all
+        cache-environment: true
+
+    - name: Download built package from same workflow
+      uses: actions/download-artifact@v3
+      with:
+        name: ${{ inputs.package_name }}
+        path: dist/
+
+    - name: try installing from local build
+      run: |
+        pip install dist/${{ inputs.package_name }}-*.tar.gz
+
+  pip-test-upload:  # upload to test-pypi and then check install from there succeeds
+    needs: [pip-build, pip-test-build]
+    environment: ${{ inputs.environment }}
+    if: needs.pip-build.result == 'success' && needs.pip-test-build.result == 'success'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: mamba-org/setup-micromamba@v1
+      with:
+        micromamba-version: latest
+        environment-name: pipbuild
+        create-args: >-
+          python=3.11
+          pip
+          build
+        post-cleanup: all
+        cache-environment: true
+
+    - name: Download built package from same workflow
+      uses: actions/download-artifact@v3
+      with:
+        name: ${{ inputs.package_name }}
+        path: dist/
+
     - name: Publish distribution 📦 to TestPyPI
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
@@ -64,11 +124,4 @@ jobs:
     - name: try installing from TestPyPI
       run: |
         cd ${{ runner.temp }}
-        pip install -i https://test.pypi.org/simple/ --no-deps ${{ inputs.package_name }}==${{ env.VERSION }}
-
-    - name: Create built package artifact ready for upload on release
-      uses: actions/upload-artifact@v3
-      with:
-        name: pip-build-${{ inputs.package_name }}-${{ inputs.version }}
-        path: dist/*
-        retention-days: 7
+        pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple ${{ inputs.package_name }}==${{ env.VERSION }}

.github/workflows/pip-upload.yml

@@ -15,6 +15,10 @@ on:
         required: false
         type: string
         default: ${{ github.ref_name }}
+      environment:
+        description: "GitHub environment in which secrets are stored"
+        type: string
+        default: "pre-release"
     secrets:
       PYPI_API_TOKEN:
         required: true
@@ -26,6 +30,7 @@ defaults:
 jobs:
   pip-publish:
     runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
     env:
       PACKAGENAME: "pip-build-${{ inputs.package_name }}-${{ inputs.version }}"
 

README.md

@@ -1,4 +1,5 @@
 # actions-city-modelling-lab
+
 Reusable GitHub Action workflows for use across City Modelling Lab repositories
 
 ## Example usage
@@ -13,7 +14,7 @@ name: CI
 on:
   push:
   branches:
-    - "**"
+  - "**"
 
 jobs:
   test:
@@ -33,11 +34,11 @@ name: CI
 on:
   pull_request:
     branches:
-      - main
+    - main
     types:
-      - opened
-      - ready_for_review
-      - review_requested
+    - opened
+    - ready_for_review
+    - review_requested
 
 jobs:
   test:
@@ -74,7 +75,7 @@ name: CI
 on:
   push:
     branches:
-      - "**"
+    - "**"
 
 jobs:
   test:
@@ -134,25 +135,75 @@ _Required secrets_: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_S3_CODE_B
 
 _URL_: `arup-group/actions-city-modelling-lab/.github/workflows/conda-build.yml`
 
-_description_: Build a conda package and store it as an artefact in your gitHub repository.
+_description_: Build a conda package and store it as an artefact in your GitHub repository.
 This could be used in a release pull request, ready to upload the build to Anaconda in a tagged release of your package.
 
 _Inputs_:
- - package_name: Name of your package, as defined in your `pyproject.toml` or `setup.py` file (if your repo is a Python project).
 
-_Required secrets_: `ANACONDA_TOKEN` (required to verify that later upload will not fail)
+- package_name: Name of your package, as defined in your `pyproject.toml` or `setup.py` file (if your repo is a Python project).
+- version (optional, default=the version tag linked to the commit): version of the package you want to build.
+- recipe_dir (optional, default="conda.recipe"): Directory in which to find the recipe (i.e. configuration) for building the package.
+- environment (optional, default="pre-release"): GitHub environment in which secrets are stored.
+Environments help to ensure that only certain operations are available to different user types.
+E.g., releasing packages can be given an extra layer of security whereby a maintainer has to approve an action before it can run.
+
+_Required secrets_: `ANACONDA_TOKEN` (required to verify that later upload will not fail) stored in a GitHub actions environment of the same name as `environment`.
 
 ### Upload a conda package
 
 _URL_: `arup-group/actions-city-modelling-lab/.github/workflows/conda-upload.yml`
 
-_description_: Upload a built conda package stored as an artefact in your gitHub repository (see [](#build-a-conda-package)).
-This could be used when you publish a new release of your package on gitHub.
+_description_: Upload a built conda package stored as an artefact in your GitHub repository (see [](#build-a-conda-package)).
+This could be used when you publish a new release of your package on GitHub.
+
+_Inputs_:
+
+- package_name: Name of your package, as defined in your `pyproject.toml` or `setup.py` file (if your repo is a Python project).
+- version (optional, default=the version tag linked to the commit): version of the package you want to upload.
+- build_workflow (optional, default=""): If you have built your package using a job in a _different_ workflow file you will need to give its name here (e.g. `pre-release.yml`).
+- environment (optional, default="pre-release"): GitHub environment in which secrets are stored.
+Environments help to ensure that only certain operations are available to different user types.
+E.g., releasing packages can be given an extra layer of security whereby a maintainer has to approve an action before it can run.
+
+_Required secrets_: `ANACONDA_TOKEN` stored in a GitHub actions environment of the same name as `environment`.
+
+### Build a pip package for upload to PyPI
+
+_URL_: `arup-group/actions-city-modelling-lab/.github/workflows/pip-build.yml`
+
+_description_: Build a pip package and store it as an artefact in your GitHub repository.
+This could be used in a release pull request, ready to upload the build to PyPI in a tagged release of your package.
+The built package will be uploaded to Test-PyPI to allow you to test installing it from PyPI without having actually released the package yet.
+
+To test installing from Test-PyPI: `pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple [package-name]==[package-version]`
 
 _Inputs_:
- - package_name: Name of your package, as defined in your `pyproject.toml` or `setup.py` file (if your repo is a Python project).
 
-_Required secrets_: `ANACONDA_TOKEN`
+- package_name: Name of your package, as defined in your `pyproject.toml` or `setup.py` file (if your repo is a Python project).
+- version (optional, default=the version tag linked to the commit): version of the package you want to build.
+- environment (optional, default="pre-release"): GitHub environment in which secrets are stored.
+Environments help to ensure that only certain operations are available to different user types.
+E.g., releasing packages can be given an extra layer of security whereby a maintainer has to approve an action before it can run.
+
+_Required secrets_: `TEST_PYPI_API_TOKEN` stored in a GitHub actions environment of the same name as `environment`.
+
+### Upload a pip package to PyPI
+
+_URL_: `arup-group/actions-city-modelling-lab/.github/workflows/pip-upload.yml`
+
+_description_: Upload a built pip package stored as an artefact in your GitHub repository (see [](#build-a-pip-package-for-upload-to-pypi)).
+This could be used when you publish a new release of your package on GitHub.
+
+_Inputs_:
+
+- package_name: Name of your package, as defined in your `pyproject.toml` or `setup.py` file (if your repo is a Python project).
+- version (optional, default=the version tag linked to the commit): version of the package you want to upload.
+- build_workflow (optional, default=""): If you have built your package using a job in a _different_ workflow file you will need to give its name here (e.g. `pre-release.yml`).
+- environment (optional, default="pre-release"): GitHub environment in which secrets are stored.
+Environments help to ensure that only certain operations are available to different user types.
+E.g., releasing packages can be given an extra layer of security whereby a maintainer has to approve an action before it can run.
+
+_Required secrets_: `PYPI_API_TOKEN` stored in a GitHub actions environment of the same name as `environment`.
 
 ### Deploy documentation
 
@@ -161,13 +212,14 @@ _URL_: `arup-group/actions-city-modelling-lab/.github/workflows/docs-deploy.yml`
 _description_: Deploy [MkDocs](https://www.mkdocs.org/) documentation using [mike](https://github.com/jimporter/mike) to your repository's `gh-pages` branch.
 
 _Inputs_:
- - package_name: Name of your package, as defined in your `pyproject.toml` or `setup.py` file (if your repo is a Python project).
- - development_version_name (optional, default="develop"): The name of the docs version which follows the project's `main` branch builds, i.e., not linked to a versioned release.
- - deploy_type: What type of doc deployment to undertake, option of: ["test", "update_latest", "update_stable"]
+
+- package_name: Name of your package, as defined in your `pyproject.toml` or `setup.py` file (if your repo is a Python project).
+- development_version_name (optional, default="develop"): The name of the docs version which follows the project's `main` branch builds, i.e., not linked to a versioned release.
+- deploy_type: What type of doc deployment to undertake, option of: ["test", "update_latest", "update_stable"]
    `test` will not deploy any documentation, only dry-run the doc build pipeline to check there are no errors.
    `update_latest` will build the docs and use it to update the `develop` version of your `gh-pages` branch, assuming the alias `latest` links to the named version `develop`.
    `update_stable` will build the docs and use it to add a new version of your docs on `gh-pages` branch and will update the alias `stable` to point at this version.
- - notebook_kernel: If jupyter notebooks are included in the docs, specify the kernel name they expect, e.g. the package name.
+- notebook_kernel: If jupyter notebooks are included in the docs, specify the kernel name they expect, e.g. the package name.
 
 _Required secrets_: None
 
@@ -178,15 +230,16 @@ _URL_: `arup-group/actions-city-modelling-lab/.github/workflows/python-install-l
 _description_: Run your tests using [pytest](https://docs.pytest.org), (optionally) check your code quality with [Ruff](https://beta.ruff.rs/docs/), and (optionally) upload your test coverage report to [codecov](https://about.codecov.io/).
 
 _Inputs_:
- - os: Operating system to run this workflow on. Should match a valid Github runner name.
- - py3version: Minor version of Python version 3 to run the test on (e.g. `11` for python v3.11).
- -  mamba_env_name (optional, default={{inputs.os}}-3{{inputs.py3version}}): Name of the Mamba environment. If it matches a name of a cached environment in the caller repository, that cache will be used.
- - additional_mamba_args (optional, default=""): Any additional arguments to pass to micromamba when creating the python environment.
- - cache_mamba_env (optional, default=true): If true, cache the mamba environment for speedier CI. Caches use the env name + a hash of the passed arguments. NOTE: this can lead to large amounts of cache space being used (600MB per env)
- - notebook_kernel (optional, default=""): If jupyter notebooks are tested, specify the kernel name they expect, e.g. the package name
- - lint (optional, default=true): If true, check code quality with the Ruff linter.
- - pytest_args (optional, default=""): Additional arguments to pass to pytest.
- - upload_to_codecov (optional, default=false): If true, upload coverage report to codecov. This assumes your repository is public as it does not expect an API key.
+
+- os: Operating system to run this workflow on. Should match a valid Github runner name.
+- py3version: Minor version of Python version 3 to run the test on (e.g. `11` for python v3.11).
+-  mamba_env_name (optional, default={{inputs.os}}-3{{inputs.py3version}}): Name of the Mamba environment. If it matches a name of a cached environment in the caller repository, that cache will be used.
+- additional_mamba_args (optional, default=""): Any additional arguments to pass to micromamba when creating the python environment.
+- cache_mamba_env (optional, default=true): If true, cache the mamba environment for speedier CI. Caches use the env name + a hash of the passed arguments. NOTE: this can lead to large amounts of cache space being used (600MB per env)
+- notebook_kernel (optional, default=""): If jupyter notebooks are tested, specify the kernel name they expect, e.g. the package name
+- lint (optional, default=true): If true, check code quality with the Ruff linter.
+- pytest_args (optional, default=""): Additional arguments to pass to pytest.
+- upload_to_codecov (optional, default=false): If true, upload coverage report to codecov. This assumes your repository is public as it does not expect an API key.
 
 _Required secrets_: None
 
@@ -197,9 +250,10 @@ _URL_: `arup-group/actions-city-modelling-lab/.github/workflows/python-memory-pr
 _description_: Run a subset of your tests marked as "high_mem" using [pytest](https://docs.pytest.org) and [memray](https://bloomberg.github.io/memray/).
 
 _Inputs_:
- - py3version: Minor version of Python version 3 to run the test on (e.g. `11` for python v3.11).
- - additional_mamba_args (optional, default=""): Any additional arguments to pass to micromamba when creating the python environment.
- - upload_flamegraph (optional, default=False): If True, upload the memory profiling flamegraph as an action artefact, stored for 90 days.
+
+- py3version: Minor version of Python version 3 to run the test on (e.g. `11` for python v3.11).
+- additional_mamba_args (optional, default=""): Any additional arguments to pass to micromamba when creating the python environment.
+- upload_flamegraph (optional, default=False): If True, upload the memory profiling flamegraph as an action artefact, stored for 90 days.
 
 _Required secrets_: None
 
@@ -210,9 +264,10 @@ _URL_: `arup-group/actions-city-modelling-lab/.github/workflows/slack-notify.yml
 _description_: Have a bot notify you of build success or failure on a Slack feed of your choice.
 
 _Inputs_:
- - result: Result of running the caller workflow (e.g., 'success', 'failure', 'skipped').
- - channel: Slack channel to which the bot notification is sent.
- - message: Sub-string to include in the message, e.g. the name of the "caller" workflow.
+
+- result: Result of running the caller workflow (e.g., 'success', 'failure', 'skipped').
+- channel: Slack channel to which the bot notification is sent.
+- message: Sub-string to include in the message, e.g. the name of the "caller" workflow.
 
 _Required secrets_: `SLACK_WEBHOOK`