Merge branch 'main' into optimize-engine-flox

* main: (64 commits) import `normalize_axis_index` from `numpy.lib` on `numpy>=2` (#364) Optimize `min_count` when `expected_groups` is not provided. (#236) Use threadpool for finding labels in chunk (#327) Manually fuse reindexing intermediates with blockwise reduction for cohorts. (#300) Bump codecov/codecov-action from 4.1.1 to 4.3.1 (#362) Add cubed notebook for hourly climatology example using "map-reduce" method (#356) Optimize bitmask finding for chunk size 1 and single chunk cases (#360) Edits to climatology doc (#361) Fix benchmarks (#358) Trim CI (#355) [pre-commit.ci] pre-commit autoupdate (#350) Initial minimal working Cubed example for "map-reduce" (#352) Bump codecov/codecov-action from 4.1.0 to 4.1.1 (#349) `method` heuristics: Avoid dot product as much as possible (#347) Fix nanlen with strings (#344) Fix direct quantile reduction (#343) Fix upstream-dev CI, silence warnings (#341) Bump codecov/codecov-action from 4.0.0 to 4.1.0 (#338) Fix direct reductions of Xarray objects (#339) Test with py3.12 (#336) ...
xarray-contrib · May 2, 2024 · 11e2c7d · 11e2c7d
2 parents 4ab01d3 + fb180e8
commit 11e2c7d
Show file tree

Hide file tree

Showing 42 changed files with 2,302 additions and 773 deletions.
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -41,15 +41,19 @@ jobs:
  OMP_NUM_THREADS: 1
  ASV_FACTOR: 1.5
  ASV_SKIP_SLOW: 1
+ BASE_SHA: ${{ github.event.pull_request.base.sha }}
+ LAST_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ HEAD_LABEL: ${{ github.event.pull_request.head.label }}
+ BASE_LABEL: ${{ github.event.pull_request.base.label }}
  run: |
  # set -x
  # ID this runner
  asv machine --yes
- echo "Baseline: ${{ github.event.pull_request.base.sha }} (${{ github.event.pull_request.base.label }})"
- echo "Contender: ${GITHUB_SHA} (${{ github.event.pull_request.head.label }})"
+ echo "Baseline: $LAST_HEAD_SHA ($BASE_LABEL)"
+ echo "Contender: ${GITHUB_SHA} ($HEAD_LABEL)"
  # Run benchmarks for current commit against base
  ASV_OPTIONS="--split --show-stderr --factor $ASV_FACTOR"
- asv continuous $ASV_OPTIONS ${{ github.event.pull_request.base.sha }} ${GITHUB_SHA} \
+ asv continuous $ASV_OPTIONS $BASE_SHA ${GITHUB_SHA} \
  | sed "/Traceback \|failed$\|PERFORMANCE DECREASED/ s/^/::error::/" \
  | tee benchmarks.log
  # Report and export results for subsequent steps
@@ -64,7 +68,7 @@ jobs:
  cp benchmarks/README_CI.md benchmarks.log .asv/results/
  working-directory: ${{ env.ASV_DIR }}
 
- - uses: actions/upload-artifact@v3
+ - uses: actions/upload-artifact@v4
  if: always()
  with:
  name: asv-benchmark-results-${{ runner.os }}

diff --git a/.github/workflows/ci-additional.yaml b/.github/workflows/ci-additional.yaml
@@ -41,7 +41,7 @@ jobs:
 
  env:
  CONDA_ENV_FILE: ci/environment.yml
- PYTHON_VERSION: "3.11"
+ PYTHON_VERSION: "3.12"
 
  steps:
  - uses: actions/checkout@v4
@@ -59,7 +59,7 @@ jobs:
  environment-name: flox-tests
  init-shell: bash
  cache-environment: true
- cache-env-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
+ cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
  create-args: |
  python=${{ env.PYTHON_VERSION }}
 
@@ -77,7 +77,7 @@ jobs:
  --ignore flox/tests \
  --cov=./ --cov-report=xml
  - name: Upload code coverage to Codecov
- uses: codecov/codecov-action@v3.1.4
+ uses: codecov/codecov-action@v4.3.1
  with:
  file: ./coverage.xml
  flags: unittests
@@ -95,7 +95,7 @@ jobs:
  shell: bash -l {0}
  env:
  CONDA_ENV_FILE: ci/environment.yml
- PYTHON_VERSION: "3.11"
+ PYTHON_VERSION: "3.12"
 
  steps:
  - uses: actions/checkout@v4
@@ -112,7 +112,7 @@ jobs:
  environment-name: flox-tests
  init-shell: bash
  cache-environment: true
- cache-env-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
+ cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
  create-args: |
  python=${{ env.PYTHON_VERSION }}
  - name: Install flox
@@ -131,7 +131,7 @@ jobs:
  python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report
 
  - name: Upload mypy coverage to Codecov
- uses: codecov/codecov-action@v3.1.4
+ uses: codecov/codecov-action@v4.3.1
  with:
  file: mypy_report/cobertura.xml
  flags: mypy

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -24,8 +24,11 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- os: ["ubuntu-latest", "windows-latest"]
- python-version: ["3.9", "3.11"]
+ os: ["ubuntu-latest"]
+ python-version: ["3.9", "3.12"]
+ include:
+ - os: "windows-latest"
+ python-version: "3.12"
  steps:
  - uses: actions/checkout@v4
  with:
@@ -49,7 +52,7 @@ jobs:
  run: |
  pytest -n auto --cov=./ --cov-report=xml
  - name: Upload code coverage to Codecov
- uses: codecov/codecov-action@v3.1.4
+ uses: codecov/codecov-action@v4.3.1
  with:
  file: ./coverage.xml
  flags: unittests
@@ -66,11 +69,9 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- python-version: ["3.11"]
- env: ["no-xarray", "no-dask"]
+ python-version: ["3.12"]
+ env: ["no-dask"] # "no-xarray", "no-numba"
  include:
- - env: "no-numba"
- python-version: "3.12"
  - env: "minimal-requirements"
  python-version: "3.9"
  steps:
@@ -93,7 +94,7 @@ jobs:
  run: |
  python -m pytest -n auto --cov=./ --cov-report=xml
  - name: Upload code coverage to Codecov
- uses: codecov/codecov-action@v3.1.4
+ uses: codecov/codecov-action@v4.3.1
  with:
  file: ./coverage.xml
  flags: unittests
@@ -119,8 +120,9 @@ jobs:
  environment-name: xarray-tests
  init-shell: bash
  cache-environment: true
- create-args: |
- python=3.10
+ create-args: >-
+ python=3.11
+ pint>=0.22
  - name: Install xarray
  run: |
  python -m pip install --no-deps .
@@ -144,4 +146,11 @@ jobs:
  id: status
  run: |
  set -euo pipefail
- python -m pytest -n auto xarray/tests/test_groupby.py
+ python -m pytest -n auto \
+ xarray/tests/test_groupby.py \
+ xarray/tests/test_units.py::TestDataArray::test_computation_objects \
+ xarray/tests/test_units.py::TestDataArray::test_grouped_operations \
+ xarray/tests/test_units.py::TestDataArray::test_resample \
+ xarray/tests/test_units.py::TestDataset::test_computation_objects \
+ xarray/tests/test_units.py::TestDataset::test_grouped_operations \
+ xarray/tests/test_units.py::TestDataset::test_resample
diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml
@@ -10,7 +10,7 @@ jobs:
  steps:
  - uses: actions/checkout@v4
  - name: Set up Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
  with:
  python-version: "3.x"
  - name: Install dependencies

diff --git a/.github/workflows/testpypi-release.yaml b/.github/workflows/testpypi-release.yaml
@@ -21,10 +21,10 @@ jobs:
  with:
  fetch-depth: 0
 
- - uses: actions/setup-python@v4
+ - uses: actions/setup-python@v5
  name: Install Python
  with:
- python-version: "3.11"
+ python-version: "3.12"
 
  - name: Install dependencies
  run: |
@@ -53,7 +53,7 @@ jobs:
  echo "✅ Looks good"
  fi
 
- - uses: actions/upload-artifact@v3
+ - uses: actions/upload-artifact@v4
  with:
  name: releases
  path: dist
@@ -62,11 +62,11 @@ jobs:
  needs: build-artifacts
  runs-on: ubuntu-latest
  steps:
- - uses: actions/setup-python@v4
+ - uses: actions/setup-python@v5
  name: Install Python
  with:
- python-version: "3.11"
- - uses: actions/download-artifact@v3
+ python-version: "3.12"
+ - uses: actions/download-artifact@v4
  with:
  name: releases
  path: dist

diff --git a/.github/workflows/upstream-dev-ci.yaml b/.github/workflows/upstream-dev-ci.yaml
@@ -7,6 +7,10 @@ on:
  types: [opened, reopened, synchronize, labeled]
  branches:
  - main
+ paths:
+ - ".github/workflows/upstream-dev-ci.yaml"
+ - "ci/upstream-dev-env.yml"
+ - "flox/*"
  schedule:
  - cron: "0 0 * * *" # Daily “At 00:00” UTC
  workflow_dispatch: # allows you to trigger the workflow run manually
@@ -19,14 +23,18 @@ jobs:
  upstream-dev:
  name: upstream-dev
  runs-on: ubuntu-latest
- if: ${{ (contains(github.event.pull_request.labels.*.name, 'test-upstream') && github.event_name == 'pull_request') || github.event_name == 'workflow_dispatch' }}
+ if: ${{
+ (contains(github.event.pull_request.labels.*.name, 'test-upstream') && github.event_name == 'pull_request')
+ || github.event_name == 'workflow_dispatch'
+ || github.event_name == 'schedule'
+ }}
  defaults:
  run:
  shell: bash -l {0}
  strategy:
  fail-fast: false
  matrix:
- python-version: ["3.11"]
+ python-version: ["3.12"]
  steps:
  - uses: actions/checkout@v4
  with:
@@ -37,16 +45,49 @@ jobs:
  - name: Set up conda environment
  uses: mamba-org/setup-micromamba@v1
  with:
- environment-file: ci/upstream-dev-env.yml
  environment-name: flox-tests
  init-shell: bash
- cache-environment: true
+ # cache-environment: true
+ # micromamba list does not list pip dependencies, so install mamba
  create-args: >-
+ mamba
+ pip
  python=${{ matrix.python-version }}
  pytest-reportlog
+
+ - name: Install upstream dev dependencies
+ run: |
+ # install cython for building cftime without build isolation
+ micromamba install -f ci/upstream-dev-env.yml
+ micromamba remove --force numpy scipy pandas cftime
+ python -m pip install \
+ -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \
+ --no-deps \
+ --pre \
+ --upgrade \
+ numpy \
+ scipy \
+ pandas \
+ xarray
+ # without build isolation for packages compiling against numpy
+ # TODO: remove once there are `numpy>=2.0` builds for cftime
+ python -m pip install \
+ --no-deps \
+ --upgrade \
+ --no-build-isolation \
+ git+https://github.com/Unidata/cftime
+ python -m pip install \
+ git+https://github.com/dask/dask \
+ git+https://github.com/ml31415/numpy-groupies
+
  - name: Install flox
  run: |
  python -m pip install --no-deps -e .
+
+ - name: List deps
+ run: |
+ # micromamba list does not list pip dependencies
+ mamba list
  - name: Run Tests
  if: success()
  id: status

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -4,13 +4,13 @@ ci:
 repos:
  - repo: https://github.com/astral-sh/ruff-pre-commit
  # Ruff version.
- rev: "v0.0.292"
+ rev: "v0.3.5"
  hooks:
  - id: ruff
  args: ["--fix", "--show-fixes"]
 
  - repo: https://github.com/pre-commit/mirrors-prettier
- rev: "v3.0.3"
+ rev: "v4.0.0-alpha.8"
  hooks:
  - id: prettier
 
@@ -23,7 +23,7 @@ repos:
  - id: check-docstring-first
 
  - repo: https://github.com/psf/black-pre-commit-mirror
- rev: 23.9.1
+ rev: 24.3.0
  hooks:
  - id: black
 
@@ -36,14 +36,14 @@ repos:
  - mdformat-myst
 
  - repo: https://github.com/nbQA-dev/nbQA
- rev: 1.7.0
+ rev: 1.8.5
  hooks:
  - id: nbqa-black
  - id: nbqa-ruff
  args: [--fix]
 
  - repo: https://github.com/kynan/nbstripout
- rev: 0.6.1
+ rev: 0.7.1
  hooks:
  - id: nbstripout
  args: [--extra-keys=metadata.kernelspec metadata.language_info.version]
@@ -56,6 +56,25 @@ repos:
  - tomli
 
  - repo: https://github.com/abravalheri/validate-pyproject
- rev: v0.15
+ rev: v0.16
  hooks:
  - id: validate-pyproject
+
+ - repo: https://github.com/rhysd/actionlint
+ rev: v1.6.27
+ hooks:
+ - id: actionlint
+ files: ".github/workflows/"
+ args:
+ [
+ "-ignore",
+ "SC1090",
+ "-ignore",
+ "SC2046",
+ "-ignore",
+ "SC2086",
+ "-ignore",
+ "SC2129",
+ "-ignore",
+ "SC2155",
+ ]
diff --git a/asv_bench/asv.conf.json b/asv_bench/asv.conf.json
@@ -27,6 +27,11 @@
  // "python setup.py build",
  // "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}"
  // ],
+ //
+ "build_command": [
+ "python setup.py build",
+ "python -mpip wheel --no-deps --no-build-isolation --no-index -w {build_cache_dir} {build_dir}"
+ ],
 
  // List of branches to benchmark. If not provided, defaults to "master"
  // (for git) or "default" (for mercurial).