From a3fd71ed0c46ca00796abf044de1174d803d6595 Mon Sep 17 00:00:00 2001 From: cpelley Date: Thu, 29 Aug 2024 12:11:42 +0100 Subject: [PATCH 1/2] matrix package version --- .github/workflows/tests.yml | 37 +++++++++++++++++++++++-------------- pyproject.toml | 5 +++++ 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 02f5a15..46a7d4e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,6 +7,17 @@ jobs: test: if: "!contains(github.event.head_commit.message, '[skip ci]')" runs-on: ubuntu-latest + strategy: + matrix: + include: + # lower bound version of packages + - python-version: '3.9' + pip-args: '.[tests,dev,minver]' + label: 'lower-bound' + # upper bound version of packages (unpinned) + - python-version: '3.x' + pip-args: '.[tests,dev]' + label: 'unpinned' steps: # SETUP @@ -20,24 +31,16 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: ${{ matrix.python-version }} cache: 'pip' - name: Add checkout directory to PYTHONPATH run: echo "PYTHONPATH=$(pwd):$PYTHONPATH" >> $GITHUB_ENV - - name: Cache pip environment - id: cache-pip-env - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: pip-${{ hashFiles('pyproject.toml') }} - restore-keys: pip- - - name: Install dependencies id: install-dependencies run: | - pip install .[tests,dev] + pip install ${{ matrix.pip-args }} pip uninstall dagrunner -y - name: Set up SSH for localhost @@ -53,7 +56,6 @@ jobs: - name: Run pytest + coverage report gen run: pytest --cov=dagrunner --cov-report=term --cov-report=html | tee coverage_output.txt; test ${PIPESTATUS[0]} -eq 0 - # TESTS (main branch) - name: Cache ref branch coverage report @@ -81,6 +83,7 @@ jobs: - name: Compare coverage id: comp-coverage + if: ${{ matrix.label == 'unpinned' }} run: | pr_coverage_total=$(grep TOTAL coverage_output.txt | awk '{print $NF}' | awk '{print substr($0, 1, length($0)-1)}') echo "pr_coverage_total=$pr_coverage_total" | tee -a $GITHUB_OUTPUT @@ -93,7 +96,7 @@ jobs: fi - name: Comment coverage report - if: steps.comp-coverage.outputs.coverage_decreased == 'true' + if: ${{ matrix.label == 'unpinned' && steps.comp-coverage.outputs.coverage_decreased == 'true' }} uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -110,7 +113,7 @@ jobs: }); - name: Upload coverage report - if: steps.comp-coverage.outputs.coverage_decreased == 'true' + if: ${{ matrix.label == 'unpinned' && steps.comp-coverage.outputs.coverage_decreased == 'true' }} uses: actions/upload-artifact@v4 with: name: coverage-report-pr @@ -121,9 +124,11 @@ jobs: # PRE-COMMIT - name: Python interpreter version sha (PYSHA) + if: ${{ matrix.label == 'unpinned' }} run: echo "PYSHA=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV - name: Cache pre-commit + if: ${{ matrix.label == 'unpinned' }} uses: actions/cache@v3 id: pre-commit-cache with: @@ -131,21 +136,25 @@ jobs: key: pre-commit|${{ env.PYSHA }}|${{ hashFiles('.pre-commit-config.yaml') }} - name: pre-commit install + if: ${{ matrix.label == 'unpinned' }} run: | pre-commit install - name: pre-commit run + if: ${{ matrix.label == 'unpinned' }} run: | pre-commit run --all-files # DOCUMENTATION - name: Build documentation + if: ${{ matrix.label == 'unpinned' }} run: | ./docs/gen_docs dagrunner ./docs - name: Check if documentation has changed id: check-docs + if: ${{ matrix.label == 'unpinned' }} run: | echo "changed=$(git diff --quiet --exit-code || echo true)" | tee -a $GITHUB_OUTPUT @@ -154,7 +163,7 @@ jobs: # success of all previous steps. A pushed commit will not trigger the re-running # of this workflow. - name: Commit and push documentation changes - if: steps.check-docs.outputs.changed == 'true' + if: ${{ matrix.label == 'unpinned' && steps.check-docs.outputs.changed == 'true' }} run: | git config user.name github-actions[bot] git config user.email 41898282+github-actions[bot]@users.noreply.github.com diff --git a/pyproject.toml b/pyproject.toml index de8a231..508a59f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,11 @@ dev = [ "pre-commit", "pytest-cov" ] +# specifying lower bound versions of packages (testing purposes) +minver=[ + 'networkx == 3.2', + 'dask[distributed] == 2024.1.1', +] [project.urls] Repository = "https://github.com/MetOffice/dagrunner" From f32e41823535947360de8645f9de56360018fa5a Mon Sep 17 00:00:00 2001 From: cpelley Date: Thu, 29 Aug 2024 12:58:57 +0100 Subject: [PATCH 2/2] separate jobs --- .github/workflows/tests.yml | 85 +++++++++++++++++++++++-------------- README.md | 2 +- 2 files changed, 54 insertions(+), 33 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 46a7d4e..18bfe40 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,24 +4,54 @@ on: [pull_request] jobs: - test: + lower-bound-packages: if: "!contains(github.event.head_commit.message, '[skip ci]')" runs-on: ubuntu-latest - strategy: - matrix: - include: - # lower bound version of packages - - python-version: '3.9' - pip-args: '.[tests,dev,minver]' - label: 'lower-bound' - # upper bound version of packages (unpinned) - - python-version: '3.x' - pip-args: '.[tests,dev]' - label: 'unpinned' steps: # SETUP + ############################ + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} # Ensure branch is checked out, not detached state (so we can push a commit later) + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.9 + cache: 'pip' + - name: Add checkout directory to PYTHONPATH + run: echo "PYTHONPATH=$(pwd):$PYTHONPATH" >> $GITHUB_ENV + + - name: Install dependencies + id: install-dependencies + run: | + pip install .[tests,minver] + pip uninstall dagrunner -y + + - name: Set up SSH for localhost + run: | + ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa + cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys + ssh-keyscan -H $(hostname) >> ~/.ssh/known_hosts + chmod 600 ~/.ssh/authorized_keys + + # TESTS + ############################ + - name: Run pytest + run: pytest . + + + unpinned-packages: + if: "!contains(github.event.head_commit.message, '[skip ci]')" + runs-on: ubuntu-latest + + steps: + # SETUP + ############################ - name: Checkout code uses: actions/checkout@v4 with: @@ -31,7 +61,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: 3.x cache: 'pip' - name: Add checkout directory to PYTHONPATH @@ -40,7 +70,7 @@ jobs: - name: Install dependencies id: install-dependencies run: | - pip install ${{ matrix.pip-args }} + pip install .[tests,dev] pip uninstall dagrunner -y - name: Set up SSH for localhost @@ -51,13 +81,12 @@ jobs: chmod 600 ~/.ssh/authorized_keys # TESTS (inc. test coverage) - - # excluded logging as per https://github.com/MetOffice/dagrunner/issues/39 + ############################ - name: Run pytest + coverage report gen run: pytest --cov=dagrunner --cov-report=term --cov-report=html | tee coverage_output.txt; test ${PIPESTATUS[0]} -eq 0 - # TESTS (main branch) - + # TESTS ('main' branch) + ############################ - name: Cache ref branch coverage report id: cache-ref-coverage uses: actions/cache@v4 @@ -72,7 +101,6 @@ jobs: path: ref ref: ${{ github.base_ref }} - # excluded logging as per https://github.com/MetOffice/dagrunner/issues/39 - name: Run tests with coverage for ref branch if: steps.cache-ref-coverage.outputs.cache-hit != 'true' run: | @@ -80,10 +108,9 @@ jobs: pytest --cov=dagrunner --cov-report=term --cov-report=html | tee coverage_output.txt; test ${PIPESTATUS[0]} -eq 0 # TESTS (compare coverage) - + ############################ - name: Compare coverage id: comp-coverage - if: ${{ matrix.label == 'unpinned' }} run: | pr_coverage_total=$(grep TOTAL coverage_output.txt | awk '{print $NF}' | awk '{print substr($0, 1, length($0)-1)}') echo "pr_coverage_total=$pr_coverage_total" | tee -a $GITHUB_OUTPUT @@ -96,7 +123,7 @@ jobs: fi - name: Comment coverage report - if: ${{ matrix.label == 'unpinned' && steps.comp-coverage.outputs.coverage_decreased == 'true' }} + if: steps.comp-coverage.outputs.coverage_decreased == 'true' uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -113,7 +140,7 @@ jobs: }); - name: Upload coverage report - if: ${{ matrix.label == 'unpinned' && steps.comp-coverage.outputs.coverage_decreased == 'true' }} + if: steps.comp-coverage.outputs.coverage_decreased == 'true' uses: actions/upload-artifact@v4 with: name: coverage-report-pr @@ -122,13 +149,11 @@ jobs: htmlcov/ # PRE-COMMIT - + ############################ - name: Python interpreter version sha (PYSHA) - if: ${{ matrix.label == 'unpinned' }} run: echo "PYSHA=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV - name: Cache pre-commit - if: ${{ matrix.label == 'unpinned' }} uses: actions/cache@v3 id: pre-commit-cache with: @@ -136,25 +161,21 @@ jobs: key: pre-commit|${{ env.PYSHA }}|${{ hashFiles('.pre-commit-config.yaml') }} - name: pre-commit install - if: ${{ matrix.label == 'unpinned' }} run: | pre-commit install - name: pre-commit run - if: ${{ matrix.label == 'unpinned' }} run: | pre-commit run --all-files # DOCUMENTATION - + ############################ - name: Build documentation - if: ${{ matrix.label == 'unpinned' }} run: | ./docs/gen_docs dagrunner ./docs - name: Check if documentation has changed id: check-docs - if: ${{ matrix.label == 'unpinned' }} run: | echo "changed=$(git diff --quiet --exit-code || echo true)" | tee -a $GITHUB_OUTPUT @@ -163,7 +184,7 @@ jobs: # success of all previous steps. A pushed commit will not trigger the re-running # of this workflow. - name: Commit and push documentation changes - if: ${{ matrix.label == 'unpinned' && steps.check-docs.outputs.changed == 'true' }} + if: steps.check-docs.outputs.changed == 'true' run: | git config user.name github-actions[bot] git config user.email 41898282+github-actions[bot]@users.noreply.github.com diff --git a/README.md b/README.md index 91d6ddf..8e0c741 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ +![Python Project](https://img.shields.io/badge/language-Python%20>=3.9-blue?logo=python&logoColor=white) [![GitHub Tag](https://img.shields.io/github/v/tag/MetOffice/dagrunner)](https://github.com/MetOffice/dagrunner/tags) [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) ![Experimental](https://img.shields.io/badge/status-experimental-orange) -![Python Project](https://img.shields.io/badge/language-Python-blue?logo=python&logoColor=white) ![Static Badge](https://img.shields.io/badge/install-pip-blue) ![PR CI status](https://github.com/MetOffice/dagrunner/actions/workflows/tests.yml/badge.svg) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit)