Merge branch 'main' into bugfix/tensordot

EPFL-Center-for-Imaging · Aug 19, 2024 · 39a20a8 · 39a20a8
2 parents 83e636f + 7373f4b
commit 39a20a8
Show file tree

Hide file tree

Showing 129 changed files with 3,456 additions and 2,490 deletions.
diff --git a/.github/workflows/additional.yml b/.github/workflows/additional.yml
@@ -41,7 +41,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
     steps:
       - name: Checkout source
         uses: actions/[email protected]
@@ -53,7 +53,7 @@ jobs:
           miniforge-version: latest
           use-mamba: true
           channel-priority: strict
-          python-version: "3.9"
+          python-version: "3.10"
           activate-environment: test-environment
           auto-activate-base: false
 

diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
@@ -14,5 +14,5 @@ jobs:
       - uses: actions/[email protected]
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.10'
       - uses: pre-commit/[email protected]
diff --git a/.github/workflows/test-report.yaml b/.github/workflows/test-report.yaml
@@ -61,7 +61,7 @@ jobs:
           mv test_report.html test_short_report.html deploy/
 
       - name: Deploy 🚀
-        uses: JamesIves/[email protected].1
+        uses: JamesIves/[email protected].3
         with:
           branch: gh-pages
           folder: deploy
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -26,11 +26,9 @@ jobs:
       fail-fast: false
       matrix:
         os: ["windows-latest", "ubuntu-latest", "macos-14"]
-        environment: ["3.9", "3.10", "3.11", "3.12"]
+        environment: ["3.10", "3.11", "3.12"]
         extra: [null]
         exclude:
-          - os: "macos-14"
-            environment: "3.9"
           - os: "macos-14"
             environment: "3.10"
           - os: "macos-14"
@@ -85,6 +83,13 @@ jobs:
         if: ${{ matrix.extra == 'pandas-nightly' }}
         run: python -m pip install --pre --extra-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple pandas -U
 
+      - name: Disable query planning for 3.10 builds
+        if: ${{ matrix.environment == '3.10' }}
+        run: |
+          export DASK_DATAFRAME__QUERY_PLANNING="False"
+          echo "DASK_DATAFRAME__QUERY_PLANNING: $DASK_DATAFRAME__QUERY_PLANNING"
+          echo "DASK_DATAFRAME__QUERY_PLANNING=$DASK_DATAFRAME__QUERY_PLANNING" >> $GITHUB_ENV
+
       - name: Reconfigure pytest-timeout
         # No SIGALRM available on Windows
         if: ${{ matrix.os != 'windows-latest' }}

diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml
@@ -80,7 +80,7 @@ jobs:
           && github.event_name != 'pull_request'
           && github.repository == 'dask/dask'
           && steps.run_tests.outcome == 'failure'
-        uses: xarray-contrib/issue-from-pytest-log@v1.2.8
+        uses: xarray-contrib/issue-from-pytest-log@v1.3.0
         with:
           log-path: output-log.jsonl
           issue-title: ⚠️ Upstream CI failed ⚠️

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -19,14 +19,14 @@ repos:
     hooks:
       - id: pyupgrade
         args:
-          - --py39-plus
+          - --py310-plus
   - repo: https://github.com/psf/black
     rev: 23.12.1
     hooks:
       - id: black
         language_version: python3
         args:
-          - --target-version=py39
+          - --target-version=py310
   - repo: https://github.com/pycqa/flake8
     rev: 7.0.0
     hooks:

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -3,11 +3,12 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.11"
+    python: "3.12"
 
 sphinx:
   configuration: docs/source/conf.py
-  fail_on_warning: true
+  # Temporarily turning off to get docs build passing
+  # fail_on_warning: true
 
 python:
   install:

diff --git a/continuous_integration/environment-3.10.yaml b/continuous_integration/environment-3.10.yaml
@@ -1,5 +1,5 @@
 # This job includes coverage
-name: test-environment
+name: test-environment-310
 channels:
   - conda-forge
   - nodefaults
@@ -25,8 +25,8 @@ dependencies:
   # Optional dependencies
   - mimesis
   - numpy=1.23
-  - pandas=1.5
-  - numba<0.59.0  # Removed numba.generated_jit which pandas<2.1 uses
+  - pandas=2.1
+  - numba
   - flask
   - fastparquet
   - h5py
@@ -39,7 +39,8 @@ dependencies:
   - pyspark
   - tiledb
   - xarray
-  - sqlalchemy<2  # `pandas=1.5` doesn't support `sqlalchemy=2`
+  - numexpr>=2.8  # Something is pinning it to 2.7
+  - sqlalchemy
   - pyarrow=10
   - coverage
   - jsonschema
@@ -75,3 +76,4 @@ dependencies:
   - pip
   - pip:
     - git+https://github.com/dask/distributed
+    - git+https://github.com/dask/dask-expr
diff --git a/continuous_integration/environment-3.11.yaml b/continuous_integration/environment-3.11.yaml
@@ -25,7 +25,7 @@ dependencies:
   # Optional dependencies
   - mimesis
   - numpy
-  - pandas=2.0
+  - pandas=2
   - numba
   - flask
   - fastparquet
@@ -76,4 +76,4 @@ dependencies:
   - pip
   - pip:
     - git+https://github.com/dask/distributed
-    - git+https://github.com/dask-contrib/dask-expr
+    - git+https://github.com/dask/dask-expr
diff --git a/continuous_integration/environment-3.12.yaml b/continuous_integration/environment-3.12.yaml
@@ -23,13 +23,14 @@ dependencies:
   - moto<5
   # Optional dependencies
   - mimesis
-  - numpy
+  - numpy >=2  # only tested here
   - pandas
   - numba
   - flask
   - fastparquet
   - h5py
-  - pytables
+  # Temporarily removing to allow `numpy >=2` to be installed
+  # - pytables
   - zarr
   # `tiledb-py=0.17.5` lead to strange seg faults in CI, However 0.18 is needed for 3.11
   # https://github.com/dask/dask/pull/9569
@@ -75,4 +76,4 @@ dependencies:
   - pip
   - pip:
     - git+https://github.com/dask/distributed
-    - git+https://github.com/dask-contrib/dask-expr
+    - git+https://github.com/dask/dask-expr
diff --git a/continuous_integration/environment-3.9.yaml b/continuous_integration/environment-3.9.yaml
diff --git a/continuous_integration/environment-mindeps-array.yaml b/continuous_integration/environment-mindeps-array.yaml
@@ -4,11 +4,11 @@ channels:
 dependencies:
   # required dependencies
   - packaging=20.0
-  - python=3.9
-  - pyyaml=5.3.1
+  - python=3.10
+  - pyyaml=5.4.1
   - click=8.1
-  - cloudpickle=1.5.0
-  - partd=1.2.0
+  - cloudpickle=3.0.0
+  - partd=1.4.0
   - fsspec=2021.09.0
   - importlib-metadata=4.13.0
   - toolz=0.10.0

diff --git a/continuous_integration/environment-mindeps-dataframe.yaml b/continuous_integration/environment-mindeps-dataframe.yaml
@@ -4,21 +4,23 @@ channels:
 dependencies:
   # required dependencies
   - packaging=20.0
-  - python=3.9
-  - pyyaml=5.3.1
+  - python=3.10
+  - pyyaml=5.4.1
   - click=8.1
-  - cloudpickle=1.5.0
-  - partd=1.2.0
+  - cloudpickle=3.0.0
+  - partd=1.4.0
   - fsspec=2021.09.0
   - importlib-metadata=4.13.0
   - toolz=0.10.0
   # optional dependencies pulled in by pip install dask[dataframe]
   - numpy=1.21
-  - pandas=1.3
+  - pandas=2.0
   # test dependencies
   - pre-commit
   - pytest
   - pytest-cov
   - pytest-rerunfailures
   - pytest-timeout
   - pytest-xdist
+  - pip:
+    - git+https://github.com/dask/dask-expr
diff --git a/continuous_integration/environment-mindeps-distributed.yaml b/continuous_integration/environment-mindeps-distributed.yaml
@@ -4,11 +4,11 @@ channels:
 dependencies:
   # required dependencies
   - packaging=20.0
-  - python=3.9
-  - pyyaml=5.3.1
+  - python=3.10
+  - pyyaml=5.4.1
   - click=8.1
-  - cloudpickle=1.5.0
-  - partd=1.2.0
+  - cloudpickle=3.0.0
+  - partd=1.4.0
   - fsspec=2021.09.0
   - importlib-metadata=4.13.0
   - toolz=0.10.0

diff --git a/continuous_integration/environment-mindeps-non-optional.yaml b/continuous_integration/environment-mindeps-non-optional.yaml
@@ -4,11 +4,11 @@ channels:
 dependencies:
   # required dependencies
   - packaging=20.0
-  - python=3.9
-  - pyyaml=5.3.1
+  - python=3.10
+  - pyyaml=5.4.1
   - click=8.1
-  - cloudpickle=1.5.0
-  - partd=1.2.0
+  - cloudpickle=3.0.0
+  - partd=1.4.0
   - fsspec=2021.09.0
   - importlib-metadata=4.13.0
   - toolz=0.10.0

diff --git a/continuous_integration/environment-mindeps-optional.yaml b/continuous_integration/environment-mindeps-optional.yaml
@@ -4,17 +4,17 @@ channels:
 dependencies:
   # required dependencies
   - packaging=20.0
-  - python=3.9
-  - pyyaml=5.3.1
+  - python=3.10
+  - pyyaml=5.4.1
   - click=8.1
-  - cloudpickle=1.5.0
-  - partd=1.2.0
+  - cloudpickle=3.0.0
+  - partd=1.4.0
   - fsspec=2021.09.0
   - importlib-metadata=4.13.0
   - toolz=0.10.0
   # optional dependencies pulled in by pip install dask[array,dataframe]
   - numpy=1.21
-  - pandas=1.3
+  - pandas=2.0
   # optional dependencies pulled in by pip install dask[diagnostics]
   - bokeh=2.4.2
   - jinja2=2.10.3
@@ -24,33 +24,34 @@ dependencies:
   # optional dependencies used by dask
   - cachey=0.1.1
   - crick=0.0.3
-  - cytoolz=0.11.0
+  - cytoolz=0.11.2
   - dask-ml=1.4.0
-  - fastavro=1.1.0
+  - fastavro=1.4.7
   - fastparquet=0.8.2
-  - h5py=2.10.0
+  - h5py=3.4.0
   - ipycytoscape=1.0.1
-  - IPython=7.16.1
-  - matplotlib=3.4.1
+  - IPython=7.30.1
+  - matplotlib=3.5.0
   - mimesis=5.3.0
-  - mmh3=2.5.1
-  - psutil=5.7.2
+  - mmh3=3.0.0
+  - psutil=5.8.0
   - python-cityhash=0.4.6
   - python-graphviz=0.8.4
-  - python-snappy=0.5.4
-  - python-xxhash=2.0.0
+  - python-snappy=0.6.0
+  - python-xxhash=2.0.2
   - s3fs=2021.9.0
-  - scikit-image=0.17.2
-  - scipy=1.5.2
+  - scikit-image=0.19.0
+  - scipy=1.7.2
   - sparse=0.12.0
-  - sqlalchemy=1.4.16
+  - sqlalchemy=1.4.26
   - tblib=1.6.0
-  - tiledb-py=0.8.1
+  - tiledb-py=0.12
   - zarr=2.12.0
   - pip
   - pip:
       # optional dependencies pulled in by pip install dask[distributed]
       - git+https://github.com/dask/distributed
+      - git+https://github.com/dask/dask-expr
   # test dependencies
   - pre-commit
   - pytest