[python] Initial work toward PyTorch data loaders

.github/workflows/python-tiledbsoma-ml.yml

@@ -0,0 +1,96 @@
+name: python-tiledbsoma-ml
+
+on:
+ pull_request:
+ branches: ["*"]
+ paths:
+ - "!**"
+ - "other_packages/python/tiledbsoma_ml/**"
+ - ".github/workflows/python-tiledbsoma-ml.yml"
+
+ push:
+ branches: [main]
+ paths:
+ - "!**"
+ - "other_packages/python/tiledbsoma_ml/**"
+ - ".github/workflows/python-tiledbsoma-ml.yml"
+
+ workflow_dispatch:
+
+jobs:
+ lint:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+
+ - name: Restore pre-commit cache
+ uses: actions/cache@v4
+ with:
+ path: ~/.cache/pre-commit
+ key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
+
+ - name: Install pre-commit
+ run: pip -v install pre-commit
+
+ - name: Run pre-commit hooks on all files
+ run: pre-commit run -v -a
+
+ tests:
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python-version }}
+ cache: pip
+ cache-dependency-path: python-spec/requirements-py${{ matrix.python-version }}.txt
+
+ - name: Install prereqs
+ working-directory: ./other_packages/python/tiledbsoma_ml/
+ run: |
+ pip install --upgrade pip wheel pytest pytest-cov setuptools
+ pip install .
+
+ - name: Run tests
+ run: |
+ PYTHONPATH=$(pwd)/other_packages/python/tiledbsoma_ml python -m pytest \
+ --cov=other_packages/python/tiledbsoma_ml/src \
+ --cov-report=xml other_packages/python/tiledbsoma_ml/tests \
+ -v
+
+ # - name: Report coverage to Codecov
+ # if: ${{ matrix.python-version == '3.11' }}
+ # uses: codecov/codecov-action@v4
+ # with:
+ # flags: python
+ # # Although Codecov isn't supposed to require an auth token for public repos like this one,
+ # # the uploader can be unreliable without one; see
+ # # https://github.com/codecov/codecov-action/issues/557#issuecomment-1216749652
+ # # As of this writing (8 Nov 2022) the CODECOV_TOKEN was generated by @aaronwolen in his
+ # # Codecov settings page for this repo, then filled into the GitHub Actions secrets.
+ # token: ${{ secrets.CODECOV_TOKEN }}
+
+ build:
+ # for now, just do a test build to ensure that it works
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+
+ - name: Do build
+ working-directory: ./other_packages/python/tiledbsoma_ml/
+ run: |
+ pip install --upgrade build pip wheel setuptools setuptools-scm
+ python -m build .

.gitignore

@@ -28,6 +28,7 @@ examples/obj/*
 scripts/deps-staging/*
 .idea
 *temp*
+**/tmp/*
 *.DS_Store
 doc/venv
 doc/source/__pycache__
@@ -52,6 +53,7 @@ apis/python/src/tiledbsoma/libtiledb.*
 apis/python/src/tiledbsoma/libtiledbsoma.so
 apis/python/src/tiledbsoma/libtiledbsoma.dylib
 apis/python/src/tiledbsoma/pytiledbsoma.*
+**/dist/
 
 /.quarto/
 

.pre-commit-config.yaml

@@ -1,24 +1,54 @@
+# note that many hooks are organized by the package to which they apply (files: param).
 exclude: ^doc/source/
 repos:
  - repo: https://github.com/psf/black
- rev: "24.4.2"
+ rev: "24.8.0"
  hooks:
- - id: black
+ - id: black
+
  - repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.5.5
+ rev: v0.6.2
  hooks:
- - id: ruff
- args: ["--config=apis/python/pyproject.toml"]
+ - id: ruff
+ name: "ruff for tiledbsoma"
+ files: "^apis/python/"
+ args: ["--config=apis/python/pyproject.toml"]
+
+ - id: ruff
+ name: "ruff for tiledbsoma_ml"
+ files: "^other_packages/python/tiledbsoma_ml/"
+ args: ["--config=other_packages/python/tiledbsoma_ml/pyproject.toml"]
+
  - repo: https://github.com/pre-commit/mirrors-mypy
  rev: v1.11.1
  hooks:
- - id: mypy
- additional_dependencies:
- # Pandas types changed between 1.x and 2.x. Our setup.py permits both, but for type-checking purposes we use the
- # Pandas 2.x types (e.g. `pd.Series[Any]`). See `_types.py` or https://github.com/single-cell-data/TileDB-SOMA/issues/2839
- # for more info.
- - "pandas-stubs>=2"
- - "somacore==1.0.14"
- - types-setuptools
- args: ["--config-file=apis/python/pyproject.toml", "apis/python/src", "apis/python/devtools"]
- pass_filenames: false
+ - id: mypy
+ name: "mypy for tiledbsoma"
+ files: "^apis/python/"
+ additional_dependencies:
+ # Pandas types changed between 1.x and 2.x. Our setup.py permits both, but for type-checking purposes we use the
+ # Pandas 2.x types (e.g. `pd.Series[Any]`). See `_types.py` or https://github.com/single-cell-data/TileDB-SOMA/issues/2839
+ # for more info.
+ - "pandas-stubs>=2"
+ - "somacore==1.0.14"
+ - types-setuptools
+ args:
+ [
+ "--config-file=apis/python/pyproject.toml",
+ "apis/python/src",
+ "apis/python/devtools",
+ ]
+ pass_filenames: false
+
+ - id: mypy
+ name: "mypy for tiledbsoma_ml"
+ files: "^other_packages/python/tiledbsoma_ml/"
+ args: ["--config=other_packages/python/tiledbsoma_ml/pyproject.toml"]
+ additional_dependencies:
+ - attrs
+ - types-requests
+ - pytest
+ - "pandas-stubs>=2"
+ - numpy
+ - typing_extensions
+ - types-setuptools