Use reflection to address the new parameter in LogicalRelation in Spark 4.0 #3027
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Test | |
| on: [push, pull_request, workflow_dispatch] | |
| jobs: | |
| build-and-test: | |
| runs-on: ubuntu-20.04 | |
| env: | |
| SPARK_LOCAL_IP: localhost | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AZURE_TEST_ACCOUNT_KEY: ${{ secrets.AZURE_TEST_ACCOUNT_KEY }} | |
| GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_service_account_key.json | |
| GOOGLE_SERVICE_ACCOUNT_KEY: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v2 | |
| - name: Cache Scala, SBT | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ~/.sbt | |
| ~/.ivy2 | |
| ~/.cache/coursier | |
| key: build-and-test-scala | |
| - name: Install Java 8 | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: '8' | |
| - run: ./build/sbt test | |
| python: | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| include: | |
| - pandas-version: 2.0.3 | |
| pyarrow-version: 16.1.0 | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| PANDAS_VERSION: ${{ matrix.pandas-version }} | |
| PYARROW_VERSION: ${{ matrix.pyarrow-version }} | |
| SPARK_LOCAL_IP: localhost | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AZURE_TEST_ACCOUNT_KEY: ${{ secrets.AZURE_TEST_ACCOUNT_KEY }} | |
| GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_service_account_key.json | |
| GOOGLE_SERVICE_ACCOUNT_KEY: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }} | |
| # Github Actions' default miniconda | |
| CONDA_PREFIX: /usr/share/miniconda | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v2 | |
| - name: Cache Scala, SBT | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ~/.sbt | |
| ~/.ivy2 | |
| ~/.cache/coursier | |
| key: build-and-test-python | |
| - name: Install Java 8 | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: '8' | |
| - name: Install dependencies | |
| run: | | |
| # See also https://github.com/conda/conda/issues/7980 | |
| source "$CONDA_PREFIX/etc/profile.d/conda.sh" | |
| conda update -q conda | |
| conda create -c conda-forge -q -n test-environment python=$PYTHON_VERSION | |
| conda activate test-environment | |
| conda config --env --add pinned_packages python=$PYTHON_VERSION | |
| conda config --env --add pinned_packages pandas==$PANDAS_VERSION | |
| conda config --env --add pinned_packages pyarrow==$PYARROW_VERSION | |
| conda install -c conda-forge --yes pandas==$PANDAS_VERSION pyarrow==$PYARROW_VERSION | |
| sed -i -e "/pandas/d" -e "/pyarrow/d" python/requirements-dev.txt | |
| conda install -c conda-forge --yes --file python/requirements-dev.txt | |
| conda list | |
| cd python | |
| cd delta-kernel-rust-sharing-wrapper | |
| maturin develop | |
| - name: Build Server | |
| run: ./build/sbt package | |
| - name: Run tests | |
| run: | | |
| # See also https://github.com/conda/conda/issues/7980 | |
| source "$CONDA_PREFIX/etc/profile.d/conda.sh" | |
| conda activate test-environment | |
| ./python/dev/lint-python | |
| ./python/dev/pytest |