Skip to content
Back to pull request #3715

Superset deployment #2717

Sign in to view logs

Superset deployment

Superset deployment #2717

Workflow file for this run

.github/workflows/pytest.yml at fbbfb2b
---
name: pytest
on:
pull_request:
merge_group:
types:
- checks_requested
workflow_dispatch:
env:
PUDL_OUTPUT: /home/runner/pudl-work/output/
PUDL_INPUT: /home/runner/pudl-work/input/
DAGSTER_HOME: /home/runner/pudl-work/dagster_home/
jobs:
ci-docs:
runs-on: ubuntu-latest
strategy:
fail-fast: false
defaults:
run:
shell: bash -l {0}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2
- name: Install conda-lock environment with micromamba
uses: mamba-org/setup-micromamba@v1
with:
environment-file: environments/conda-lock.yml
environment-name: pudl-dev
cache-environment: true
- name: Log environment details
run: |
conda info
conda list
conda config --show-sources
conda config --show
printenv | sort
- name: Make input, output and dagster dirs
run: mkdir -p ${{ env.PUDL_OUTPUT }} ${{ env.PUDL_INPUT}} ${{ env.DAGSTER_HOME }}
- name: Lint and build PUDL documentation with Sphinx
run: |
pip install --no-deps --editable .
make docs-build
- name: Upload docs coverage artifact
uses: actions/upload-artifact@v4
with:
name: coverage-docs
path: ./*coverage*
ci-unit:
runs-on: ubuntu-latest
strategy:
fail-fast: false
defaults:
run:
shell: bash -l {0}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2
- name: Install conda-lock environment with micromamba
uses: mamba-org/setup-micromamba@v1
with:
environment-file: environments/conda-lock.yml
environment-name: pudl-dev
cache-environment: true
- name: Log environment details
run: |
conda info
conda list
conda config --show-sources
conda config --show
printenv | sort
- name: Make input, output and dagster dirs
run: mkdir -p ${{ env.PUDL_OUTPUT }} ${{ env.PUDL_INPUT}} ${{ env.DAGSTER_HOME }}
- name: Log SQLite3 version
run: |
which sqlite3
sqlite3 --version
- name: Run PUDL unit tests and collect test coverage
run: |
pip install --no-deps --editable .
make pytest-unit
- name: Upload unit tests coverage artifact
uses: actions/upload-artifact@v4
with:
name: coverage-unit
path: ./*coverage*
ci-integration:
runs-on:
group: large-runner-group
labels: ubuntu-22.04-4core
if: github.event_name == 'workflow_dispatch' || github.event.merge_group
permissions:
contents: read
id-token: write
strategy:
fail-fast: false
defaults:
run:
shell: bash -l {0}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2
- name: Install Conda environment using mamba
uses: mamba-org/setup-micromamba@v1
with:
environment-file: environments/conda-lock.yml
environment-name: pudl-dev
cache-environment: true
- name: Log environment details
run: |
conda info
conda list
conda config --show-sources
conda config --show
printenv | sort
- name: Log SQLite3 version
run: |
which sqlite3
sqlite3 --version
- name: Compile Zenodo datastore DOIs for cache invalidation
run:
grep -e '.*10\.\(5281\|5072\)/zenodo\..*' src/pudl/workspace/datastore.py
| sed -e 's/",*$//g' | sed -e 's/^.*"//g' | sort > datastore-dois.txt
- name: Restore Zenodo datastore from cache if possible
uses: actions/cache@v4
id: cache-zenodo-datastore
with:
path: ${{ env.PUDL_INPUT }}
key: zenodo-datastore-${{ hashFiles('datastore-dois.txt') }}
- name: Make input, output and dagster dirs
run: mkdir -p ${{ env.PUDL_OUTPUT }} ${{ env.PUDL_INPUT}} ${{ env.DAGSTER_HOME }}
- name: List workspace contents
run: find /home/runner/pudl-work
- name: Set default GCP credentials
id: gcloud-auth
continue-on-error: true
uses: "google-github-actions/auth@v2"
with:
workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider"
service_account: "tox-pytest-github-action@catalyst-cooperative-pudl.iam.gserviceaccount.com"
- name: Run integration tests, trying to use GCS cache if possible
run: |
pip install --no-deps --editable .
pudl_datastore --dataset epacems --partition year_quarter=2022q1
make pytest-integration
- name: Upload integration test coverage artifact
uses: actions/upload-artifact@v4
with:
name: coverage-integration
path: ./*coverage*
- name: Log post-test Zenodo datastore contents
run: find ${{ env.PUDL_INPUT }}
ci-coverage:
runs-on: ubuntu-latest
needs:
- ci-docs
- ci-unit
- ci-integration
steps:
- uses: actions/checkout@v4
- name: Download coverage
id: download-coverage
uses: actions/download-artifact@v4
with:
path: coverage
- name: List downloaded files
run: |
find coverage -type f
- name: Upload test coverage report to CodeCov
uses: codecov/codecov-action@v4
with:
directory: coverage
token: ${{ secrets.CODECOV_TOKEN }}
- name: Install Micromamba
uses: mamba-org/setup-micromamba@v1
with:
init-shell: bash
environment-name: coverage
create-args: >-
python=3.12
coverage>=7.4.1
- name: Combine coverage data and check that we have required coverage
# Required coverage is set in pyproject.toml section [tool.coverage.report]
run: |
micromamba run -n coverage coverage combine coverage/*/.coverage
micromamba run -n coverage coverage xml --fail-under=0
micromamba run -n coverage coverage report