Merge remote-tracking branch 'origin/dev' into parallel-pytest

Additionally, switch integration tests to use tox under the hood.

.github/workflows/docker-build-test.yml

@@ -0,0 +1,36 @@
+---
+name: docker-build-test
+on:
+  workflow_dispatch:
+  pull_request:
+
+jobs:
+  pudl_docker_build:
+    name: Test building the PUDL ETL Docker image
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+
+      - name: Docker Metadata
+        id: docker_metadata
+        uses: docker/[email protected]
+        with:
+          images: catalystcoop/pudl-etl
+          flavor: |
+            latest=auto
+
+      - name: Set up Docker Buildx
+        uses: docker/[email protected]
+
+      - name: Build image but do not push to Docker Hub
+        uses: docker/[email protected]
+        with:
+          context: .
+          file: docker/Dockerfile
+          push: false
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/tox-pytest.yml

@@ -1,8 +1,14 @@
 ---
 name: tox-pytest
 
-on: [pull_request, workflow_dispatch]
-
+on:
+  workflow_dispatch:
+  pull_reqyest:
+    types:
+      - created
+      - opened
+      - synchronize
+      - ready_for_review
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -107,16 +113,13 @@ jobs:
           name: coverage-unit
           path: coverage.xml
 
-  # TODO(rousik): the following could simply run docker
-  # container mounted to specific directory.
-  # TODO(rousik): run this on larger runners, see dev for
-  # how to do that.
   ci-integration:
     needs:
       - ci-unit
     runs-on:
       group: large-runner-group
       labels: ubuntu-22.04-4core
+    if: github.event.pull_request.draft == false
     permissions:
       contents: read
       id-token: write
@@ -129,11 +132,6 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 2
-      # TODO(rousik): setup-python may be redundant with micromamba
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.11"
-          cache: "pip"
       - name: Install Conda environment using mamba
         uses: mamba-org/setup-micromamba@v1
         with:
@@ -145,9 +143,6 @@ jobs:
             - defaults
             channel_priority: strict
 
-      - name: Install PUDL and dependencies
-        run: pip install "./[test]"
-
       - name: Compile Zenodo datastore DOIs for cache invalidation
         run:
           grep -e '.*10\.\(5281\|5072\)/zenodo\..*' src/pudl/workspace/datastore.py
@@ -171,103 +166,21 @@ jobs:
           workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider"
           service_account: "tox-pytest-github-action@catalyst-cooperative-pudl.iam.gserviceaccount.com"
 
-      - name: Prepare for ETL execution
-        run: alembic upgrade head
       - name: Run ferc_to_sqlite
-        run: |
-          ferc_to_sqlite --gcs-cache-path=gs://zenodo-cache.catalyst.coop \
-            src/pudl/package_data/settings/etl_fast.yml
-      - name: Run pudl_etl
-        run: |
-          pudl_etl --gcs-cache-path=gs://zenodo-cache.catalyst.coop \
-            src/pudl/package_data/settings/etl_fast.yml
-      - name: Run integration tests
-        run: tox -e integration -- --live-dbs
+        run: tox -e ci_ferc_to_sqlite
 
-      # TODO(rousik): upload coverage reports, need to find out how to best do so
-  #      - name: Upload coverage
-  #        uses: actions/upload-artifact@v3
-  #        with:
-  #          name: coverage-integration
-  #          path: coverage.xml
+      - name: Run pudl_etl
+        run: tox -e ci_pudl_etl
 
-  # ci-integration:
-  #   runs-on: ubuntu-latest
-  #   needs:
-  #     - ci-integration-etl
-  #   permissions:
-  #     contents: read
-  #     id-token: write
-  #   strategy:
-  #     fail-fast: false
-  #   defaults:
-  #     run:
-  #       shell: bash -l {0}
+      - name: Run integration tests
+        run: tox -e ci_integration
 
-  #   steps:
-  #     - uses: actions/checkout@v3
-  #       with:
-  #         fetch-depth: 2
-  #
-  #      - name: Install Conda environment using mamba
-  #        uses: mamba-org/provision-with-micromamba@v16
-  #        with:
-  #          environment-file: test/test-environment.yml
-  #          cache-env: true
-  #          channels: conda-forge,defaults
-  #          channel-priority: strict
-  #
-  #      - name: Log environment details
-  #        run: |
-  #          conda info
-  #          conda list
-  #          conda config --show-sources
-  #          conda config --show
-  #          printenv | sort
-  #
-  #      - name: Log SQLite3 version
-  #        run: |
-  #          which sqlite3
-  #          sqlite3 --version
-  #
-  #      - name: Compile Zenodo datastore DOIs for cache invalidation
-  #        run:
-  #          grep -e '.*10\.\(5281\|5072\)/zenodo\..*' src/pudl/workspace/datastore.py
-  #          | sed -e 's/",*$//g' | sed -e 's/^.*"//g' | sort > datastore-dois.txt
-  #
-  #      - name: Restore Zenodo datastore from cache if possible
-  #        uses: actions/cache@v3
-  #        id: cache-zenodo-datastore
-  #        with:
-  #          path: ${{ env.PUDL_INPUT }}
-  #          key: zenodo-datastore-${{ hashFiles('datastore-dois.txt') }}
-  #
-  #      - name: Make input, output and dagster dirs
-  #        run: mkdir -p ${{ env.PUDL_OUTPUT }} ${{ env.PUDL_INPUT}} ${{ env.DAGSTER_HOME }}
-  #
-  #      - name: List workspace contents
-  #        run: find /home/runner/pudl-work
-  #
-  #      - name: Set default GCP credentials
-  #        id: gcloud-auth
-  #        continue-on-error: true
-  #        uses: "google-github-actions/auth@v1"
-  #        with:
-  #          workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider"
-  #          service_account: "tox-pytest-github-action@catalyst-cooperative-pudl.iam.gserviceaccount.com"
-  #
-  #      - name: Run integration tests, trying to use GCS cache if possible
-  #        run: |
-  #          tox -e integration -- --gcs-cache-path=gs://zenodo-cache.catalyst.coop --durations 0
-  #
-  #      - name: Upload coverage
-  #        uses: actions/upload-artifact@v3
-  #        with:
-  #          name: coverage-integration
-  #          path: coverage.xml
-  #
-  #      - name: Log post-test Zenodo datastore contents
-  #        run: find ${{ env.PUDL_INPUT }}
+      - name: Upload coverage
+        uses: actions/upload-artifact@v3
+        if: ${{ matrix.tox-env == 'docs' }}
+        with:
+          name: coverage-docs
+          path: coverage.xml
 
   ci-coverage:
     runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -5,8 +5,6 @@ repos:
     rev: v1.10.0
     hooks:
       - id: python-check-blanket-noqa # Prohibit overly broad QA exclusions.
-      - id: python-no-eval # Never use eval() it's dangerous.
-      - id: python-no-log-warn # logger.warning(), not old .warn()
       - id: rst-backticks # Find single rather than double backticks
       - id: rst-directive-colons # Missing double-colons after directives
       - id: rst-inline-touching-normal # Inline code should never touch normal text
@@ -21,21 +19,18 @@ repos:
       - id: check-yaml # Validate all YAML files.
       - id: check-case-conflict # Avoid case sensitivity in file names.
       - id: debug-statements # Watch for lingering debugger calls.
-      - id: end-of-file-fixer # Ensure there's a newline at EOF.
       - id: mixed-line-ending # Use Unix line-endings to avoid big no-op CSV diffs.
         args: ["--fix=lf"]
-      - id: trailing-whitespace # Remove trailing whitespace.
       - id: name-tests-test # Follow PyTest naming convention.
 
   ####################################################################################
   # Formatters: hooks that re-write Python & documentation files
   ####################################################################################
-  # Make sure import statements are sorted uniformly.
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.0.287
     hooks:
-      - id: isort
-        exclude: migrations/.*
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
 
   # Format the code
   - repo: https://github.com/psf/black
@@ -45,75 +40,28 @@ repos:
         language_version: python3.11
         exclude: migrations/.*
 
-  # Remove f-string prefix when there's nothing in the string to format.
-  - repo: https://github.com/dannysepler/rm_unneeded_f_str
-    rev: v0.2.0
-    hooks:
-      - id: rm-unneeded-f-str
-
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v3.0.3
     hooks:
       - id: prettier
         types_or: [yaml]
 
-  # Update Python language constructs to modern standards
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.10.1
-    hooks:
-      - id: pyupgrade
-        args: ["--py311-plus"]
-
   ####################################################################################
-  # Linters: hooks that check but don't alter Python and documentation files
+  # Linters: hooks that check but don't alter files
   ####################################################################################
 
-  # Check for PEP8 non-compliance, code complexity, style, errors, etc:
-  - repo: https://github.com/PyCQA/flake8
-    rev: 6.1.0
-    hooks:
-      - id: flake8
-        args: [--config, tox.ini]
-        exclude: migrations/.*
-        additional_dependencies:
-          - flake8-docstrings
-          - flake8-colors
-          - pydocstyle
-          - flake8-builtins
-          - mccabe
-          - pep8-naming
-          - pycodestyle
-          - pyflakes
-          - flake8-rst-docstrings
-          - flake8-use-fstring
-
-  # Check for known security vulnerabilities:
-  - repo: https://github.com/PyCQA/bandit
-    rev: 1.7.5
-    hooks:
-      - id: bandit
-        additional_dependencies: ["bandit[toml]"]
-        args: ["--configfile", ".bandit.yml"]
-
   # Lint Dockerfiles for errors and to ensure best practices
   - repo: https://github.com/AleksaC/hadolint-py
     rev: v2.12.0.2
     hooks:
       - id: hadolint
+
   # Check for errors in restructuredtext (.rst) files under the doc hierarchy
   - repo: https://github.com/PyCQA/doc8
     rev: v1.1.1
     hooks:
       - id: doc8
-        args: [--config, tox.ini]
-
-  # Lint any RST files and embedded code blocks for syntax / formatting errors
-  - repo: https://github.com/rstcheck/rstcheck
-    rev: v6.1.2
-    hooks:
-      - id: rstcheck
-        additional_dependencies: [sphinx]
-        args: [--config, tox.ini]
+        args: [--config, pyproject.toml]
 
   #####################################################################################
   # Our own pre-commit hooks, which don't come from the pre-commit project

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM condaforge/mambaforge:4.12.0-0
+FROM condaforge/mambaforge:23.3.1-1
 
 # Install curl and js
 # awscli requires unzip, less, groff and mandoc
@@ -27,7 +27,7 @@ WORKDIR ${CONTAINER_HOME}
 ENV CONDA_PREFIX=${CONTAINER_HOME}/env
 ENV PUDL_REPO=${CONTAINER_HOME}/pudl
 ENV CONDA_RUN="conda run --no-capture-output --prefix ${CONDA_PREFIX}"
-ENV PYTHON_VERSION="3.10"
+ENV PYTHON_VERSION="3.11"
 
 ENV CONTAINER_PUDL_WORKSPACE=${CONTAINER_HOME}/pudl_work
 ENV PUDL_INPUT=${CONTAINER_PUDL_WORKSPACE}/data

migrations/versions/8b3029915ab1_pandas_2_0.py

@@ -5,8 +5,8 @@
 Create Date: 2023-08-27 10:47:20.386231
 
 """
-from alembic import op
 import sqlalchemy as sa
+from alembic import op
 from sqlalchemy.dialects import sqlite
 
 # revision identifiers, used by Alembic.