Use caching to speed up testdata fetching, fix upstream builds, scipy…

… suggestions (#1906) ### What kind of change does this PR introduce? * Employs caching of `pip` dependencies, the `tox` environments and the `xclim-testdata` cache to speed up the CI setup on subsequent builds. * Deals with an issue that was affecting the upstream build dependencies, causing their installation to silently fail. ### Does this PR introduce a breaking change? No. ### Other information: https://github.com/actions/cache Preliminary results suggest that this will shave at least 30 seconds off of the setup times for most builds (except for `prefetch`-based `tox` environments, as this build is designed to test the fetching of the test data). Also worth noting: Caches are immutable once created!
Ouranosinc · Sep 17, 2024 · 61a5274 · 61a5274
2 parents 08656c1 + 004b95a
commit 61a5274
Show file tree

Hide file tree

Showing 30 changed files with 214 additions and 144 deletions.
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
@@ -1,21 +1,25 @@
-* xclim version:
-* Python version:
-* Operating System:
+- xclim version:
+- Python version:
+- Operating System:
 
 ### Description
+
 <!--Describe what you were trying to get done.
 Tell us what happened, what went wrong, and what you expected to happen.-->
 
-
 ### What I Did
+
 <!--Paste the command(s) you ran and the output.
 If there was a crash, please include the traceback below.-->
+
 ```
 $ pip install foo --bar
 ```
 
 ### What I Received
+
 <!--Paste the output or the stack trace of the problem you experienced here.-->
+
 ```
 Traceback (most recent call last):
   File "/path/to/file/script.py", line 3326, in run_code

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -3,13 +3,19 @@ updates:
   - package-ecosystem: github-actions
     directory: /
     schedule:
-      interval: daily
+      interval: weekly
       time: '12:00'
-    open-pull-requests-limit: 5
+    groups:
+      actions:
+        patterns:
+          - "*"
 
   - package-ecosystem: pip
     directory: /
     schedule:
-      interval: daily
+      interval: weekly
       time: '12:00'
-    open-pull-requests-limit: 5
+    groups:
+      python:
+        patterns:
+          - "*"
diff --git a/.github/publish-mastodon.template.md → .github/publish-mastodon-template.md b/.github/publish-mastodon.template.md → .github/publish-mastodon-template.md
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -40,8 +40,7 @@ jobs:
       (github.event_name == 'push')
     strategy:
       matrix:
-        python-version:
-          - "3.9"
+        python-version: [ "3.9" ]
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
@@ -58,9 +57,17 @@ jobs:
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
       - name: Install CI libraries
         run: |
           python -m pip install --require-hashes -r CI/requirements_ci.txt
+      - name: Environment Caching
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: |
+            ${{ matrix.testdata-cache }}
+            .tox
+          key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-${{ hashFiles('pyproject.toml', 'tox.ini') }}-lint
       - name: Run pylint
         run: |
           python -m pylint --rcfile=.pylintrc.toml --disable=import-error --exit-zero xclim
@@ -69,13 +76,14 @@ jobs:
           python -m tox -e lint
 
   test-preliminary:
-    name: Python${{ matrix.python-version }} (ubuntu-latest)
+    name: Python${{ matrix.python-version }} (${{ matrix.os }})
     needs: lint
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version:
-          - "3.9"
+        os: [ 'ubuntu-latest' ]
+        python-version: [ "3.9" ]
+        testdata-cache: [ '~/.cache/xclim-testdata' ]
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
@@ -95,9 +103,17 @@ jobs:
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
       - name: Install CI libraries
         run: |
           python -m pip install --require-hashes -r CI/requirements_ci.txt
+      - name: Environment Caching
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: |
+            ${{ matrix.testdata-cache }}
+            .tox
+          key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-${{ hashFiles('pyproject.toml', 'tox.ini') }}-Python${{ matrix.python-version }}
       - name: Test with tox
         run: |
           python -m tox -- -m 'not slow'
@@ -118,39 +134,47 @@ jobs:
       matrix:
         include:
           # Linux builds
-          - os: ubuntu-latest
+          - os: 'ubuntu-latest'
+            testdata-cache: '~/.cache/xclim-testdata'
             markers: -m 'not slow'
             python-version: "3.10"
             tox-env: standard
-          - os: ubuntu-latest
+          - os: 'ubuntu-latest'
+            testdata-cache: '~/.cache/xclim-testdata'
             markers: -m 'not slow'
             python-version: "3.11"
             tox-env: standard
-          - os: ubuntu-latest
+          - os: 'ubuntu-latest'
+            testdata-cache: '~/.cache/xclim-testdata'
             markers: -m 'not slow'
             python-version: "3.12"
             tox-env: standard
           # Windows builds
-          - os: windows-latest
+          - os: 'windows-latest'
+            testdata-cache: 'C:\Users\runneradmin\AppData\Local\xclim-testdata\xclim-testdata\Cache'
             markers: -m 'not slow'
             python-version: "3.9"
             tox-env: py39-coverage-prefetch  # Test data prefetch is needed for Windows
           # macOS builds
-          - os: macos-latest
-            python-version: "3.10"
+          - os: 'macos-latest'
+            testdata-cache: '~/Library/Caches/xclim-testdata'
             markers: ''  # Slow tests
+            python-version: "3.10"
             tox-env: py310-coverage-extras
           # Specialized tests
-          - os: ubuntu-latest
+          - os: 'ubuntu-latest'
+            testdata-cache: '~/.cache/xclim-testdata'
             markers: -m 'not requires_internet and not slow'
             python-version: "3.9"
             tox-env: py39-coverage-offline-prefetch
-          - os: ubuntu-latest
-            markers: ''
+          - os: 'ubuntu-latest'
+            testdata-cache: '~/.cache/xclim-testdata'
+            markers: '' # No markers for notebooks
             python-version: "3.10"
             tox-env: notebooks
-          - os: ubuntu-latest
-            markers: ''
+          - os: 'ubuntu-latest'
+            testdata-cache: '~/.cache/xclim-testdata'
+            markers: '' # No markers for doctests
             python-version: "3.12"
             tox-env: doctests
     steps:
@@ -182,9 +206,19 @@ jobs:
         uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
       - name: Install CI libraries
         run: |
           python -m pip install --require-hashes -r CI/requirements_ci.txt
+      - name: Environment Caching
+        # if prefetch is not in tox-env
+        if: contains(matrix.tox-env, 'prefetch') == false
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: |
+            ${{ matrix.testdata-cache }}
+            .tox
+          key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-${{ hashFiles('pyproject.toml', 'tox.ini') }}-Python${{ matrix.python-version }}
       - name: Test with tox
         if: ${{ matrix.tox-env == 'standard' }}
         run: |
@@ -212,8 +246,9 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-latest ]
+        os: [ 'ubuntu-latest' ]
         python-version: [ "3.9", "3.12" ]
+        testdata-cache: [ '~/.cache/xclim-testdata' ]
     defaults:
       run:
         shell: bash -l {0}
@@ -247,6 +282,12 @@ jobs:
       - name: Micromamba version
         run: |
           echo "micromamba: $(micromamba --version)"
+      - name: Test Data Caching
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: |
+            ${{ matrix.testdata-cache }}
+          key: ${{ runner.os }}-xclim-testdata-${{ env.XCLIM_TESTDATA_BRANCH }}-conda-${{ hashFiles('pyproject.toml', 'tox.ini') }}-Python${{ matrix.python-version }}
       - name: Install xclim
         run: |
           python -m pip install --no-user --editable .
@@ -284,18 +325,13 @@ jobs:
       - test-pypi
       - test-conda
     runs-on: ubuntu-latest
-    container: python:3-slim
     steps:
-      - name: Checkout Repository
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - name: Harden Runner
+        uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1
         with:
-          sparse-checkout: |
-            CI/requirements_ci.txt
-      - name: Install CI libraries
-        run: |
-          python -m pip install --require-hashes -r CI/requirements_ci.txt
-      - name: Coveralls finished
-        run: |
-          python -m coveralls --finish
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          disable-sudo: true
+          egress-policy: audit
+      - name: Coveralls Finished
+        uses: coverallsapp/github-action@643bc377ffa44ace6394b2b5d0d3950076de9f63 # v2.3.0
+        with:
+          parallel-finished: true
diff --git a/.github/workflows/publish-mastodon.yml b/.github/workflows/publish-mastodon.yml
@@ -70,7 +70,7 @@ jobs:
         id: render_template
         uses: chuhlomin/render-template@807354a04d9300c9c2ac177c0aa41556c92b3f75 # v1.10
         with:
-          template: .github/publish-mastodon.template.md
+          template: .github/publish-mastodon-template.md
           vars: |
             version: ${{ env.version }}
 

diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml
@@ -32,8 +32,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version:
-          - "3.10"
+        python-version: [ "3.12" ]
+        testdata-cache: [ '~/.cache/xclim-testdata' ]
     defaults:
       run:
         shell: bash -l {0}
@@ -66,8 +66,8 @@ jobs:
           create-args: >-
             eigen
             pybind11
-            python=${{ matrix.python-version }}
             pytest-reportlog
+            python=${{ matrix.python-version }}
       - name: Micromamba version
         run: |
           echo "micromamba: $(micromamba --version)"
@@ -84,6 +84,12 @@ jobs:
           micromamba list
           xclim show_version_info
           python -m pip check || true
+      - name: Test Data Caching
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: |
+            ${{ matrix.testdata-cache }}
+          key: ${{ runner.os }}-xclim-testdata-upstream-${{ hashFiles('pyproject.toml', 'tox.ini') }}
       - name: Run Tests
         if: success()
         id: status
@@ -97,4 +103,5 @@ jobs:
           && github.repository_owner == 'Ouranosinc'
         uses: xarray-contrib/issue-from-pytest-log@f94477e45ef40e4403d7585ba639a9a3bcc53d43 # v1.3.0
         with:
+          issue-title: "⚠️ Nightly upstream-dev CI failed for Python${{ matrix.python-version }} ⚠️"
           log-path: output-${{ matrix.python-version }}-log.jsonl
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -13,7 +13,7 @@ repos:
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
-        exclude: '.ipynb|.github/publish-mastodon.template.md'
+        exclude: '.ipynb|.github/publish-mastodon-template.md'
       - id: check-json
       - id: check-toml
       - id: check-yaml
@@ -36,15 +36,11 @@ repos:
     rev: 24.8.0
     hooks:
       - id: black
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.13.2
-    hooks:
-      - id: isort
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.6.3
     hooks:
       - id: ruff
-        args: [ '--fix' ]
+        args: [ '--fix', '--show-fixes' ]
   - repo: https://github.com/pylint-dev/pylint
     rev: v3.2.7
     hooks:
@@ -68,20 +64,35 @@ repos:
         additional_dependencies: [ 'pyupgrade==3.16.0' ]
       - id: nbqa-black
         additional_dependencies: [ 'black==24.4.2' ]
-      - id: nbqa-isort
-        additional_dependencies: [ 'isort==5.13.2' ]
   - repo: https://github.com/kynan/nbstripout
     rev: 0.7.1
     hooks:
       - id: nbstripout
         files: '.ipynb'
         args: [ '--extra-keys', 'metadata.kernelspec' ]
+  - repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.10.0
+    hooks:
+#      - id: python-check-blanket-noqa
+#      - id: python-check-blanket-type-ignore
+      - id: python-no-eval
+      - id: python-no-log-warn
+      - id: python-use-type-annotations
+      - id: rst-directive-colons
+      - id: rst-inline-touching-normal
+      - id: text-unicode-replacement-char
+  - repo: https://github.com/executablebooks/mdformat
+    rev: 0.7.17
+    hooks:
+      - id: mdformat
+        exclude: '.github/\w+.md|.github/publish-mastodon-template.md|docs/paper/paper.md'
   - repo: https://github.com/keewis/blackdoc
     rev: v0.3.9
     hooks:
       - id: blackdoc
-        additional_dependencies: [ 'black==24.4.2' ]
+        additional_dependencies: [ 'black==24.8.0' ]
         exclude: '(xclim/indices/__init__.py|docs/installation.rst)'
+      - id: blackdoc-autoupdate-black
   - repo: https://github.com/codespell-project/codespell
     rev: v2.3.0
     hooks: