This repository has been archived by the owner on Mar 11, 2024. It is now read-only.
🐛 fix DeltaLake I/O with remote filesystems failing in some cases #129
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| workflow_dispatch: | |
| push: | |
| release: | |
| types: | |
| - created | |
| # Sequence of patterns matched against refs/tags | |
| tags: | |
| - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 | |
| jobs: | |
| test: | |
| name: test polars=${{ matrix.polars_version }} dagster=${{ matrix.dagster_version }} py=${{ matrix.py }} ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }}-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - Ubuntu | |
| # MacOs is commented out because: 1. For some reason it's very slow. 2. It never had OS-specific issues | |
| # - MacOs | |
| py: | |
| - "3.11" | |
| - "3.10" | |
| - "3.9" | |
| - "3.8" | |
| dagster_version: | |
| - "1.4.0" | |
| # - "1.5.0" disabled until https://github.com/dagster-io/dagster/issues/17436 is fixed | |
| polars_version: | |
| - "0.17.0" | |
| - "0.18.0" | |
| - "0.19.0" | |
| steps: | |
| - name: Setup python for test ${{ matrix.py }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.py }} | |
| - uses: actions/checkout@v2 | |
| - name: Install and configure Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| version: 1.4.2 | |
| virtualenvs-create: false | |
| virtualenvs-in-project: false | |
| installer-parallel: true | |
| - name: Install dependencies | |
| run: poetry install --all-extras --sync && pip install --force-reinstall polars~=${{ matrix.polars_version }} dagster~=${{ matrix.dagster_version }} | |
| - name: Print polars info | |
| run: python -c 'import polars; print(polars.show_versions())' | |
| - name: Print dagster info | |
| run: python -c 'import dagster; print(dagster.__version__)' | |
| - name: Run tests | |
| run: pytest -v . | |
| lint: | |
| name: lint polars=${{ matrix.polars_version }} dagster=${{ matrix.dagster_version }} py=${{ matrix.py }} ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }}-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - Ubuntu | |
| py: | |
| - "3.11" | |
| - "3.10" | |
| - "3.9" | |
| - "3.8" | |
| dagster_version: | |
| - "1.4.0" | |
| # - "1.5.0" disabled until https://github.com/dagster-io/dagster/issues/17436 is fixed | |
| polars_version: | |
| - "0.17.0" | |
| - "0.18.0" | |
| - "0.19.0" | |
| steps: | |
| - name: Setup python for test ${{ matrix.py }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.py }} | |
| - uses: actions/checkout@v2 | |
| - name: Install and configure Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| version: 1.4.2 | |
| virtualenvs-create: false | |
| virtualenvs-in-project: false | |
| installer-parallel: true | |
| - name: Install dependencies | |
| run: poetry install --all-extras --sync && pip install --force-reinstall polars~=${{ matrix.polars_version }} dagster~=${{ matrix.dagster_version }} | |
| - name: Run pre-commit hooks | |
| run: pre-commit run --all-files | |
| qa_success: | |
| name: QA Success | |
| needs: [test, lint] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Success | |
| run: echo "QA Succeeded!" | |
| publish: | |
| needs: qa_success | |
| if: contains(github.ref, 'refs/tags/') | |
| name: Publish to PyPI | |
| strategy: | |
| fail-fast: false | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Setup Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.9 | |
| - uses: actions/checkout@v2 | |
| - name: Install and configure Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| version: 1.4.2 | |
| virtualenvs-create: false | |
| virtualenvs-in-project: false | |
| installer-parallel: true | |
| - name: Install poetry-dynamic-versioning | |
| run: poetry self add [email protected] | |
| - name: Publish to PyPI | |
| env: | |
| PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} | |
| run: | | |
| poetry config pypi-token.pypi $PYPI_TOKEN | |
| poetry publish --build |