Merge branch 'main' into bugfix/1446

unionai-oss · May 8, 2024 · 2d1a3b2 · 2d1a3b2
2 parents dd8a0d7 + 0faae07
commit 2d1a3b2
Show file tree

Hide file tree

Showing 268 changed files with 33,108 additions and 15,284 deletions.
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
@@ -70,35 +70,39 @@ make docs
 
 #### Adding New Dependencies
 
-To add new dependencies to the project, make sure to alter the _environment.yml_ file. Then to sync the dependencies from the _environment.yml_ file to the _requirements-dev.txt_ run the following command
+This repo uses [mamba](https://github.com/mamba-org/mamba), which is a faster
+implementation of [miniconda](https://docs.conda.io/en/latest/miniconda.html),
+to run the `nox` test suite. Simply install it via conda-forge:
 
 ```bash
-make requirements-dev.txt
+conda install -c conda-forge mamba
+```
+
+To add new dependencies to the project, first alter the _environment.yml_ file. Then to sync the dependencies from the `environment.yml`` file to the `requirements.in` run the following command
+
+```bash
+make nox-requirements
 ```
 
 This will:
 
 - Invoke `python scripts/generate_pip_deps_from_conda.py` to convert `environment.yml`
   to a `requirements.in` file.
-- Use `pip-compile` to create `requirements-dev.txt` file that has a fully specified
-  set of dependencies.
+- Use `pip-compile` via the `uv` package to create requirements files in the
+  `ci` and `dev` directories. The `ci` requirements files are used by github
+   actions, while those in the `dev` directory should be used to create local
+   development enviornments.
 
-You can use the resulting `requirements-dev.txt` file to install your dependencies
+You can use the resulting `requirements-{3.x}.txt` file to install your dependencies
 with `pip`:
 
 ```bash
-pip install -r requirements-dev.txt
+pip install -r dev/requirements-{3.x}.txt  # replace {3.x} with desired python version
 ```
 
 Moreover to add new extra dependencies in setup.py, it is necessary to add it to
 the **_extras_require** dictionary.
 
-When you update dependencies also need to update the `pip-compile`d requirements
-files in the `ci` directory, which are used by the CI/CD process of this repo:
-
-```bash
-make nox-ci-requirements
-```
 
 #### Set up `pre-commit`
 
@@ -150,21 +154,6 @@ make nox-conda
 make nox
 ```
 
-Option 2 assumes that you have python environments for all of the versions
-that pandera supports.
-
-#### Using `mamba` (optional)
-
-You can also use [mamba](https://github.com/mamba-org/mamba), which is a faster
-implementation of [miniconda](https://docs.conda.io/en/latest/miniconda.html),
-to run the `nox` test suite. Simply install it via conda-forge, and
-`make nox-conda` should use it under the hood.
-
-```bash
-conda install -c conda-forge mamba
-make nox-conda
-```
-
 ### Project Releases
 
 Releases are organized under [milestones](https://github.com/pandera-dev/pandera/milestones),

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -12,7 +12,7 @@ A clear and concise description of what the bug is.
 
 - [ ] I have checked that this issue has not already been reported.
 - [ ] I have confirmed this bug exists on the latest version of pandera.
-- [ ] (optional) I have confirmed this bug exists on the master branch of pandera.
+- [ ] (optional) I have confirmed this bug exists on the main branch of pandera.
 
 **Note**: Please read [this guide](https://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports) detailing how to provide the necessary information for us to reproduce your bug.
 
@@ -29,8 +29,8 @@ A clear and concise description of what you expected to happen.
 #### Desktop (please complete the following information):
 
  - OS: [e.g. iOS]
- - Browser [e.g. chrome, safari]
- - Version [e.g. 22]
+ - Browser: [e.g. chrome, safari]
+ - Version: [e.g. 22]
 
 #### Screenshots
 If applicable, add screenshots to help explain your problem.

diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml
@@ -10,6 +10,7 @@ on:
     branches:
       - main
       - dev
+      - polars-dev
       - bugfix
       - "release/*"
 
@@ -40,10 +41,10 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: ~/.cache/pip  # ubuntu location
           key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }}
@@ -99,106 +100,102 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
+        os:
+        - ubuntu-latest
+        - windows-latest
+        - macos-13
+        # - macos-latest  # see: https://github.com/actions/setup-python/issues/696
         python-version: ["3.8", "3.9", "3.10", "3.11"]
-        pandas-version: ["1.5.3", "2.0.3"]
+        pandas-version: ["1.5.3", "2.0.3", "2.2.0"]
         pydantic-version: ["1.10.11", "2.3.0"]
         include:
         - os: ubuntu-latest
           pip-cache: ~/.cache/pip
-        - os: macos-latest
+        # - os: macos-latest
+        #   pip-cache: ~/Library/Caches/pip
+        - os: macos-13
           pip-cache: ~/Library/Caches/pip
         - os: windows-latest
           pip-cache: ~/AppData/Local/pip/Cache
+        exclude:
+        - python-version: "3.8"
+          pandas-version: "2.2.0"
+        - python-version: "3.11"
+          pandas-version: "1.5.3"
 
     steps:
       - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Cache conda
-        uses: actions/cache@v3
-        env:
-          # Increase this value to reset cache if etc/environment.yml has not changed
-          CACHE_NUMBER: 2
-        with:
-          path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment.yml') }}
-        id: cache
-
       - name: Cache pip
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ${{ matrix.pip-cache }}
           key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
 
       - name: Install deps
-        run: pip install -r ci/requirements-py${{ matrix.python-version }}-pandas${{ matrix.pandas-version }}-pydantic${{ matrix.pydantic-version }}.txt
+        run:
+          pip install uv
+          uv pip install -r ci/requirements-py${{ matrix.python-version }}-pandas${{ matrix.pandas-version }}-pydantic${{ matrix.pydantic-version }}.txt
 
       - run: |
           pip list
           printenv | sort
 
       - name: Unit Tests - Core
-        run: pytest tests/core ${{ env.PYTEST_FLAGS }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='core', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Unit Tests - Hypotheses
-        run: pytest tests/hypotheses ${{ env.PYTEST_FLAGS }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='hypotheses', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Unit Tests - IO
-        run: pytest tests/io ${{ env.PYTEST_FLAGS }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='io', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Unit Tests - Mypy
         # mypy tests hang on windows
         if: ${{ matrix.os != 'windows-latest' }}
-        run: pytest -v tests/mypy ${{ env.PYTEST_FLAGS }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='mypy', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Unit Tests - Strategies
-        run: pytest tests/strategies ${{ env.PYTEST_FLAGS }} ${{ env.HYPOTHESIS_FLAGS }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='strategies', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Unit Tests - FastAPI
         # there's an issue with the fastapi tests in CI that's not reproducible locally
         # when pydantic > v2
-        run: pytest tests/fastapi ${{ env.PYTEST_FLAGS }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='fastapi', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Unit Tests - GeoPandas
-        run: pytest tests/geopandas ${{ env.PYTEST_FLAGS }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='geopandas', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Unit Tests - Dask
-        if: ${{ matrix.pandas-version != '2.0.3' }}
-        run: pytest tests/dask ${{ env.PYTEST_FLAGS }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='dask', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Unit Tests - Pyspark
-        if: ${{ matrix.os != 'windows-latest' && matrix.pandas-version != '2.0.3' }}
-        run: pytest tests/pyspark ${{ env.PYTEST_FLAGS }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='pyspark', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Unit Tests - Modin-Dask
-        if: ${{ matrix.pandas-version != '2.0.3' }}
-        run: pytest tests/modin ${{ env.PYTEST_FLAGS }}
-        env:
-          CI_MODIN_ENGINES: dask
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='modin-dask', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Unit Tests - Modin-Ray
         # ray CI issues with the following:
         # - windows, python 3.10
         # Tracking issue: https://github.com/modin-project/modin/issues/5466
-        if: ${{ matrix.os != 'windows-latest' && matrix.pandas-version != '2.0.3' }}
-        run: pytest tests/modin ${{ env.PYTEST_FLAGS }}
-        env:
-          CI_MODIN_ENGINES: ray
+        if: ${{ matrix.os != 'windows-latest' }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='modin-ray', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
 
       - name: Check Docstrings
-        if: ${{ matrix.os != 'windows-latest' && matrix.python-version == '3.11' && matrix.pandas-version == '2.0.3' }}
+        if: ${{ matrix.os != 'windows-latest' && matrix.python-version == '3.11' && matrix.pandas-version == '2.2.0' }}
         run: nox ${{ env.NOX_FLAGS }} --session doctests
 
       - name: Check Docs
-        if: ${{ matrix.os != 'windows-latest' && matrix.python-version == '3.11' && matrix.pydantic-version == '2.0.3' }}
+        if: ${{ matrix.os != 'windows-latest' && matrix.python-version == '3.11' && matrix.pydantic-version == '2.2.0' }}
         run: nox ${{ env.NOX_FLAGS }} --session docs
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -12,7 +12,7 @@ jobs:
         with:
           fetch-depth: "0"
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.x"
       - name: Install dependencies

diff --git a/.gitignore b/.gitignore
@@ -3,6 +3,7 @@
 dask-worker-space
 spark-warehouse
 docs/source/_contents
+docs/jupyter_execute
 **.DS_Store
 
 # Byte-compiled / optimized / DLL files

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -45,12 +45,11 @@ repos:
         exclude: (^docs/|^scripts)
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.982
+    rev: v1.10.0
     hooks:
       - id: mypy
         additional_dependencies:
           - numpy
-          - pandas-stubs
           - types-click
           - types-pkg_resources
           - types-pytz

diff --git a/.pylintrc b/.pylintrc
@@ -48,4 +48,11 @@ disable=
     unnecessary-dunder-call,
     use-dict-literal,
     invalid-name,
-    import-outside-toplevel
+    import-outside-toplevel,
+    missing-class-docstring,
+    missing-function-docstring,
+    fixme,
+    too-many-locals,
+    redefined-outer-name,
+    logging-fstring-interpolation,
+    multiple-statements
diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -8,9 +8,18 @@ version: 2
 build:
   os: ubuntu-20.04
   tools:
-    python: "3.10"
+    python: "mambaforge-4.10"
 
 # Build documentation in the docs/ directory with Sphinx
+conda:
+  environment: environment.yml
+
+# Optionally set the version of Python and requirements required to build your docs
+python:
+  install:
+    - method: pip
+      path: .
+
 sphinx:
   configuration: docs/source/conf.py
 
@@ -20,10 +29,3 @@ sphinx:
 
 # Optionally build your docs in additional formats such as PDF and ePub
 formats: []
-
-# Optionally set the version of Python and requirements required to build your docs
-python:
-  install:
-    - requirements: requirements-docs.txt
-    - method: pip
-      path: .
diff --git a/Makefile b/Makefile
@@ -21,10 +21,10 @@ requirements:
 	pip install -r requirements-dev.txt
 
 docs-clean:
-	rm -rf docs/**/generated docs/**/methods docs/_build docs/source/_contents
+	rm -rf docs/source/reference/generated docs/**/generated docs/**/methods docs/_build docs/source/_contents
 
 docs: docs-clean
-	python -m sphinx -E "docs/source" "docs/_build" && make -C docs doctest
+	python -m sphinx -W -E "docs/source" "docs/_build" && make -C docs doctest
 
 quick-docs:
 	python -m sphinx -E "docs/source" "docs/_build" -W && \
@@ -38,17 +38,19 @@ nox:
 
 NOX_FLAGS ?= "-r"
 
-nox-conda:
-	nox -db conda --envdir .nox-conda ${NOX_FLAGS}
+nox-mamba:
+	nox -db mamba --envdir .nox-mamba ${NOX_FLAGS}
 
 deps-from-conda:
 	python scripts/generate_pip_deps_from_conda.py
 
 nox-ci-requirements: deps-from-conda
-	nox -db mamba --envdir .nox-mamba -s ci_requirements
+	nox -db mamba --envdir .nox-mamba -s ci_requirements ${NOX_FLAGS}
 
 nox-dev-requirements: deps-from-conda
-	nox -db mamba --envdir .nox-mamba -s dev_requirements
+	nox -db mamba --envdir .nox-mamba -s dev_requirements ${NOX_FLAGS}
 
-requirements-docs.txt: deps-from-conda
-	pip-compile requirements.in --no-emit-index-url --output-file requirements-docs.txt -v --resolver backtracking
+nox-requirements: nox-ci-requirements nox-dev-requirements
+
+nox-tests:
+	nox -db mamba --envdir .nox-mamba -s tests ${NOX_FLAGS}