From 05ef08a53af3459d723a3b34941b5800b173d955 Mon Sep 17 00:00:00 2001 From: Bryn Pickering <17178478+brynpickering@users.noreply.github.com> Date: Mon, 19 Feb 2024 15:29:43 +0000 Subject: [PATCH] Add layer of security to conda and pip uploading; update README --- .github/workflows/conda-build.yml | 6 ++ .github/workflows/conda-upload.yml | 5 ++ .github/workflows/pip-build.yml | 69 +++++++++++++++-- .github/workflows/pip-upload.yml | 5 ++ README.md | 119 +++++++++++++++++++++-------- 5 files changed, 164 insertions(+), 40 deletions(-) diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index cb858d6..3da5fb0 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -17,6 +17,10 @@ on: required: false type: string default: ${{ github.ref_name }} + environment: + description: "GitHub environment in which secrets are stored" + type: string + default: "release" secrets: ANACONDA_TOKEN: required: true @@ -44,6 +48,7 @@ jobs: token-exists: runs-on: ubuntu-latest + environment: ${{ inputs.environment }} steps: - name: check if ANACONDA_TOKEN exists env: @@ -58,6 +63,7 @@ jobs: conda-build: runs-on: ${{ matrix.os }} + environment: release needs: [n-builds, token-exists] strategy: matrix: ${{ fromJson(needs.n-builds.outputs.matrix) }} diff --git a/.github/workflows/conda-upload.yml b/.github/workflows/conda-upload.yml index 35ac49c..153ecb1 100644 --- a/.github/workflows/conda-upload.yml +++ b/.github/workflows/conda-upload.yml @@ -15,6 +15,10 @@ on: required: false type: string default: "" + environment: + description: "GitHub environment in which secrets are stored" + type: string + default: "release" secrets: ANACONDA_TOKEN: required: true @@ -26,6 +30,7 @@ defaults: jobs: conda-publish: runs-on: ubuntu-latest + environment: ${{ inputs.environment }} env: PACKAGENAME: "conda-build[a-zA-Z0-9-.]*-${{ inputs.package_name }}-${{ inputs.version }}" steps: diff --git a/.github/workflows/pip-build.yml b/.github/workflows/pip-build.yml index e31c294..898dbb8 100644 --- a/.github/workflows/pip-build.yml +++ b/.github/workflows/pip-build.yml @@ -12,6 +12,10 @@ on: required: false type: string default: ${{ github.ref_name }} + environment: + description: "GitHub environment in which secrets are stored" + type: string + default: "pre-release" secrets: TEST_PYPI_API_TOKEN: required: true @@ -50,6 +54,62 @@ jobs: - name: Build package run: python -m build + - name: Create built package artifact ready for upload on release + uses: actions/upload-artifact@v3 + with: + name: pip-build-${{ inputs.package_name }}-${{ inputs.version }} + path: dist/* + retention-days: 7 + + pip-test-build: # check local install succeeds before uploading to test-pypi + needs: [pip-build] + if: needs.pip-build.result == 'success' + runs-on: ubuntu-latest + steps: + - uses: mamba-org/setup-micromamba@v1 + with: + micromamba-version: latest + environment-name: pipbuild + create-args: >- + python=3.11 + pip + build + post-cleanup: all + cache-environment: true + + - name: Download built package from same workflow + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.package_name }} + path: dist/ + + - name: try installing from local build + run: | + pip install dist/${{ inputs.package_name }}-*.tar.gz + + pip-test-upload: # upload to test-pypi and then check install from there succeeds + needs: [pip-build, pip-test-build] + environment: ${{ inputs.environment }} + if: needs.pip-build.result == 'success' && needs.pip-test-build.result == 'success' + runs-on: ubuntu-latest + steps: + - uses: mamba-org/setup-micromamba@v1 + with: + micromamba-version: latest + environment-name: pipbuild + create-args: >- + python=3.11 + pip + build + post-cleanup: all + cache-environment: true + + - name: Download built package from same workflow + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.package_name }} + path: dist/ + - name: Publish distribution 📦 to TestPyPI uses: pypa/gh-action-pypi-publish@release/v1 with: @@ -64,11 +124,4 @@ jobs: - name: try installing from TestPyPI run: | cd ${{ runner.temp }} - pip install -i https://test.pypi.org/simple/ --no-deps ${{ inputs.package_name }}==${{ env.VERSION }} - - - name: Create built package artifact ready for upload on release - uses: actions/upload-artifact@v3 - with: - name: pip-build-${{ inputs.package_name }}-${{ inputs.version }} - path: dist/* - retention-days: 7 \ No newline at end of file + pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple ${{ inputs.package_name }}==${{ env.VERSION }} diff --git a/.github/workflows/pip-upload.yml b/.github/workflows/pip-upload.yml index 1370cef..1da4131 100644 --- a/.github/workflows/pip-upload.yml +++ b/.github/workflows/pip-upload.yml @@ -15,6 +15,10 @@ on: required: false type: string default: ${{ github.ref_name }} + environment: + description: "GitHub environment in which secrets are stored" + type: string + default: "pre-release" secrets: PYPI_API_TOKEN: required: true @@ -26,6 +30,7 @@ defaults: jobs: pip-publish: runs-on: ubuntu-latest + environment: ${{ inputs.environment }} env: PACKAGENAME: "pip-build-${{ inputs.package_name }}-${{ inputs.version }}" diff --git a/README.md b/README.md index 2faba9a..bbec0b9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # actions-city-modelling-lab + Reusable GitHub Action workflows for use across City Modelling Lab repositories ## Example usage @@ -13,7 +14,7 @@ name: CI on: push: branches: - - "**" + - "**" jobs: test: @@ -33,11 +34,11 @@ name: CI on: pull_request: branches: - - main + - main types: - - opened - - ready_for_review - - review_requested + - opened + - ready_for_review + - review_requested jobs: test: @@ -74,7 +75,7 @@ name: CI on: push: branches: - - "**" + - "**" jobs: test: @@ -134,25 +135,75 @@ _Required secrets_: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_S3_CODE_B _URL_: `arup-group/actions-city-modelling-lab/.github/workflows/conda-build.yml` -_description_: Build a conda package and store it as an artefact in your gitHub repository. +_description_: Build a conda package and store it as an artefact in your GitHub repository. This could be used in a release pull request, ready to upload the build to Anaconda in a tagged release of your package. _Inputs_: - - package_name: Name of your package, as defined in your `pyproject.toml` or `setup.py` file (if your repo is a Python project). -_Required secrets_: `ANACONDA_TOKEN` (required to verify that later upload will not fail) +- package_name: Name of your package, as defined in your `pyproject.toml` or `setup.py` file (if your repo is a Python project). +- version (optional, default=the version tag linked to the commit): version of the package you want to build. +- recipe_dir (optional, default="conda.recipe"): Directory in which to find the recipe (i.e. configuration) for building the package. +- environment (optional, default="pre-release"): GitHub environment in which secrets are stored. +Environments help to ensure that only certain operations are available to different user types. +E.g., releasing packages can be given an extra layer of security whereby a maintainer has to approve an action before it can run. + +_Required secrets_: `ANACONDA_TOKEN` (required to verify that later upload will not fail) stored in a GitHub actions environment of the same name as `environment`. ### Upload a conda package _URL_: `arup-group/actions-city-modelling-lab/.github/workflows/conda-upload.yml` -_description_: Upload a built conda package stored as an artefact in your gitHub repository (see [](#build-a-conda-package)). -This could be used when you publish a new release of your package on gitHub. +_description_: Upload a built conda package stored as an artefact in your GitHub repository (see [](#build-a-conda-package)). +This could be used when you publish a new release of your package on GitHub. + +_Inputs_: + +- package_name: Name of your package, as defined in your `pyproject.toml` or `setup.py` file (if your repo is a Python project). +- version (optional, default=the version tag linked to the commit): version of the package you want to upload. +- build_workflow (optional, default=""): If you have built your package using a job in a _different_ workflow file you will need to give its name here (e.g. `pre-release.yml`). +- environment (optional, default="pre-release"): GitHub environment in which secrets are stored. +Environments help to ensure that only certain operations are available to different user types. +E.g., releasing packages can be given an extra layer of security whereby a maintainer has to approve an action before it can run. + +_Required secrets_: `ANACONDA_TOKEN` stored in a GitHub actions environment of the same name as `environment`. + +### Build a pip package for upload to PyPI + +_URL_: `arup-group/actions-city-modelling-lab/.github/workflows/pip-build.yml` + +_description_: Build a pip package and store it as an artefact in your GitHub repository. +This could be used in a release pull request, ready to upload the build to PyPI in a tagged release of your package. +The built package will be uploaded to Test-PyPI to allow you to test installing it from PyPI without having actually released the package yet. + +To test installing from Test-PyPI: `pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple [package-name]==[package-version]` _Inputs_: - - package_name: Name of your package, as defined in your `pyproject.toml` or `setup.py` file (if your repo is a Python project). -_Required secrets_: `ANACONDA_TOKEN` +- package_name: Name of your package, as defined in your `pyproject.toml` or `setup.py` file (if your repo is a Python project). +- version (optional, default=the version tag linked to the commit): version of the package you want to build. +- environment (optional, default="pre-release"): GitHub environment in which secrets are stored. +Environments help to ensure that only certain operations are available to different user types. +E.g., releasing packages can be given an extra layer of security whereby a maintainer has to approve an action before it can run. + +_Required secrets_: `TEST_PYPI_API_TOKEN` stored in a GitHub actions environment of the same name as `environment`. + +### Upload a pip package to PyPI + +_URL_: `arup-group/actions-city-modelling-lab/.github/workflows/pip-upload.yml` + +_description_: Upload a built pip package stored as an artefact in your GitHub repository (see [](#build-a-pip-package-for-upload-to-pypi)). +This could be used when you publish a new release of your package on GitHub. + +_Inputs_: + +- package_name: Name of your package, as defined in your `pyproject.toml` or `setup.py` file (if your repo is a Python project). +- version (optional, default=the version tag linked to the commit): version of the package you want to upload. +- build_workflow (optional, default=""): If you have built your package using a job in a _different_ workflow file you will need to give its name here (e.g. `pre-release.yml`). +- environment (optional, default="pre-release"): GitHub environment in which secrets are stored. +Environments help to ensure that only certain operations are available to different user types. +E.g., releasing packages can be given an extra layer of security whereby a maintainer has to approve an action before it can run. + +_Required secrets_: `PYPI_API_TOKEN` stored in a GitHub actions environment of the same name as `environment`. ### Deploy documentation @@ -161,13 +212,14 @@ _URL_: `arup-group/actions-city-modelling-lab/.github/workflows/docs-deploy.yml` _description_: Deploy [MkDocs](https://www.mkdocs.org/) documentation using [mike](https://github.com/jimporter/mike) to your repository's `gh-pages` branch. _Inputs_: - - package_name: Name of your package, as defined in your `pyproject.toml` or `setup.py` file (if your repo is a Python project). - - development_version_name (optional, default="develop"): The name of the docs version which follows the project's `main` branch builds, i.e., not linked to a versioned release. - - deploy_type: What type of doc deployment to undertake, option of: ["test", "update_latest", "update_stable"] + +- package_name: Name of your package, as defined in your `pyproject.toml` or `setup.py` file (if your repo is a Python project). +- development_version_name (optional, default="develop"): The name of the docs version which follows the project's `main` branch builds, i.e., not linked to a versioned release. +- deploy_type: What type of doc deployment to undertake, option of: ["test", "update_latest", "update_stable"] `test` will not deploy any documentation, only dry-run the doc build pipeline to check there are no errors. `update_latest` will build the docs and use it to update the `develop` version of your `gh-pages` branch, assuming the alias `latest` links to the named version `develop`. `update_stable` will build the docs and use it to add a new version of your docs on `gh-pages` branch and will update the alias `stable` to point at this version. - - notebook_kernel: If jupyter notebooks are included in the docs, specify the kernel name they expect, e.g. the package name. +- notebook_kernel: If jupyter notebooks are included in the docs, specify the kernel name they expect, e.g. the package name. _Required secrets_: None @@ -178,15 +230,16 @@ _URL_: `arup-group/actions-city-modelling-lab/.github/workflows/python-install-l _description_: Run your tests using [pytest](https://docs.pytest.org), (optionally) check your code quality with [Ruff](https://beta.ruff.rs/docs/), and (optionally) upload your test coverage report to [codecov](https://about.codecov.io/). _Inputs_: - - os: Operating system to run this workflow on. Should match a valid Github runner name. - - py3version: Minor version of Python version 3 to run the test on (e.g. `11` for python v3.11). - - mamba_env_name (optional, default={{inputs.os}}-3{{inputs.py3version}}): Name of the Mamba environment. If it matches a name of a cached environment in the caller repository, that cache will be used. - - additional_mamba_args (optional, default=""): Any additional arguments to pass to micromamba when creating the python environment. - - cache_mamba_env (optional, default=true): If true, cache the mamba environment for speedier CI. Caches use the env name + a hash of the passed arguments. NOTE: this can lead to large amounts of cache space being used (600MB per env) - - notebook_kernel (optional, default=""): If jupyter notebooks are tested, specify the kernel name they expect, e.g. the package name - - lint (optional, default=true): If true, check code quality with the Ruff linter. - - pytest_args (optional, default=""): Additional arguments to pass to pytest. - - upload_to_codecov (optional, default=false): If true, upload coverage report to codecov. This assumes your repository is public as it does not expect an API key. + +- os: Operating system to run this workflow on. Should match a valid Github runner name. +- py3version: Minor version of Python version 3 to run the test on (e.g. `11` for python v3.11). +- mamba_env_name (optional, default={{inputs.os}}-3{{inputs.py3version}}): Name of the Mamba environment. If it matches a name of a cached environment in the caller repository, that cache will be used. +- additional_mamba_args (optional, default=""): Any additional arguments to pass to micromamba when creating the python environment. +- cache_mamba_env (optional, default=true): If true, cache the mamba environment for speedier CI. Caches use the env name + a hash of the passed arguments. NOTE: this can lead to large amounts of cache space being used (600MB per env) +- notebook_kernel (optional, default=""): If jupyter notebooks are tested, specify the kernel name they expect, e.g. the package name +- lint (optional, default=true): If true, check code quality with the Ruff linter. +- pytest_args (optional, default=""): Additional arguments to pass to pytest. +- upload_to_codecov (optional, default=false): If true, upload coverage report to codecov. This assumes your repository is public as it does not expect an API key. _Required secrets_: None @@ -197,9 +250,10 @@ _URL_: `arup-group/actions-city-modelling-lab/.github/workflows/python-memory-pr _description_: Run a subset of your tests marked as "high_mem" using [pytest](https://docs.pytest.org) and [memray](https://bloomberg.github.io/memray/). _Inputs_: - - py3version: Minor version of Python version 3 to run the test on (e.g. `11` for python v3.11). - - additional_mamba_args (optional, default=""): Any additional arguments to pass to micromamba when creating the python environment. - - upload_flamegraph (optional, default=False): If True, upload the memory profiling flamegraph as an action artefact, stored for 90 days. + +- py3version: Minor version of Python version 3 to run the test on (e.g. `11` for python v3.11). +- additional_mamba_args (optional, default=""): Any additional arguments to pass to micromamba when creating the python environment. +- upload_flamegraph (optional, default=False): If True, upload the memory profiling flamegraph as an action artefact, stored for 90 days. _Required secrets_: None @@ -210,9 +264,10 @@ _URL_: `arup-group/actions-city-modelling-lab/.github/workflows/slack-notify.yml _description_: Have a bot notify you of build success or failure on a Slack feed of your choice. _Inputs_: - - result: Result of running the caller workflow (e.g., 'success', 'failure', 'skipped'). - - channel: Slack channel to which the bot notification is sent. - - message: Sub-string to include in the message, e.g. the name of the "caller" workflow. + +- result: Result of running the caller workflow (e.g., 'success', 'failure', 'skipped'). +- channel: Slack channel to which the bot notification is sent. +- message: Sub-string to include in the message, e.g. the name of the "caller" workflow. _Required secrets_: `SLACK_WEBHOOK`