Fix bug where to_pandas doesn't properly handle the case when column names between Delta and parquet metadata are different in casing #1900
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Test | |
| on: [push, pull_request] | |
| jobs: | |
| build-and-test: | |
| runs-on: ubuntu-20.04 | |
| env: | |
| SPARK_LOCAL_IP: localhost | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AZURE_TEST_ACCOUNT_KEY: ${{ secrets.AZURE_TEST_ACCOUNT_KEY }} | |
| GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_service_account_key.json | |
| GOOGLE_SERVICE_ACCOUNT_KEY: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v2 | |
| - name: Cache Scala, SBT | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ~/.sbt | |
| ~/.ivy2 | |
| ~/.cache/coursier | |
| key: build-and-test-scala | |
| - name: Install Java 8 | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: '8' | |
| - run: ./build/sbt test | |
| python: | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: [3.7, 3.8, 3.9] | |
| include: | |
| - pandas-version: 1.2.4 | |
| pyarrow-version: 4.0.0 | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| PANDAS_VERSION: ${{ matrix.pandas-version }} | |
| PYARROW_VERSION: ${{ matrix.pyarrow-version }} | |
| SPARK_LOCAL_IP: localhost | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AZURE_TEST_ACCOUNT_KEY: ${{ secrets.AZURE_TEST_ACCOUNT_KEY }} | |
| GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_service_account_key.json | |
| GOOGLE_SERVICE_ACCOUNT_KEY: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }} | |
| # Github Actions' default miniconda | |
| CONDA_PREFIX: /usr/share/miniconda | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v2 | |
| - name: Cache Scala, SBT | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ~/.sbt | |
| ~/.ivy2 | |
| ~/.cache/coursier | |
| key: build-and-test-python | |
| - name: Install Java 8 | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: '8' | |
| - name: Install dependencies | |
| run: | | |
| # See also https://github.com/conda/conda/issues/7980 | |
| source "$CONDA_PREFIX/etc/profile.d/conda.sh" | |
| conda update -q conda | |
| conda create -c conda-forge -q -n test-environment python=$PYTHON_VERSION | |
| conda activate test-environment | |
| conda config --env --add pinned_packages python=$PYTHON_VERSION | |
| conda config --env --add pinned_packages pandas==$PANDAS_VERSION | |
| conda config --env --add pinned_packages pyarrow==$PYARROW_VERSION | |
| conda install -c conda-forge --yes pandas==$PANDAS_VERSION pyarrow==$PYARROW_VERSION | |
| sed -i -e "/pandas/d" -e "/pyarrow/d" python/requirements-dev.txt | |
| conda install -c conda-forge --yes --file python/requirements-dev.txt | |
| conda list | |
| - name: Build Server | |
| run: ./build/sbt package | |
| - name: Run tests | |
| run: | | |
| # See also https://github.com/conda/conda/issues/7980 | |
| source "$CONDA_PREFIX/etc/profile.d/conda.sh" | |
| conda activate test-environment | |
| ./python/dev/lint-python | |
| ./python/dev/pytest |