Create pudl.duckdb from parquet files

docker/gcp_pudl_etl.sh

@@ -123,6 +123,13 @@ function distribute_parquet() {
     fi
 }
 
+function create_and_distribute_duckdb() {
+    echo "Creating DuckDB file from parquet files and copying it to GCS"
+    GCS_PATH="gs://superset.catalyst.coop"
+    parquet_to_duckdb "$PUDL_OUTPUT/parquet" "$PUDL_OUTPUT/pudl.duckdb" && \
+    gsutil -q -m -u "$GCP_BILLING_PROJECT" cp "$PUDL_OUTPUT/pudl.duckdb" "$GCS_PATH"
+}
+
 function copy_outputs_to_distribution_bucket() {
     # Only attempt to update outputs if we have a real value of BUILD_REF
     # This avoids accidentally blowing away the whole bucket if it's not set.
@@ -282,6 +289,9 @@ if [[ $ETL_SUCCESS == 0 ]]; then
         # Distribute Parquet outputs to a private bucket
         distribute_parquet 2>&1 | tee -a "$LOGFILE"
         DISTRIBUTE_PARQUET_SUCCESS=${PIPESTATUS[0]}
+        # Create duckdb file from parquet files
+        create_duckdb 2>&1 | tee -a "$LOGFILE"
+        CREATE_DUCKDB_SUCCESS=${PIPESTATUS[0]}
         # Remove some cruft from the builds that we don't want to distribute
         clean_up_outputs_for_distribution 2>&1 | tee -a "$LOGFILE"
         CLEAN_UP_OUTPUTS_SUCCESS=${PIPESTATUS[0]}
@@ -313,6 +323,7 @@ if [[ $ETL_SUCCESS == 0 && \
       $UPDATE_STABLE_SUCCESS == 0 && \
       $DATASETTE_SUCCESS == 0 && \
       $DISTRIBUTE_PARQUET_SUCCESS == 0 && \
+      $CREATE_DUCKDB_SUCCESS == 0 && \
       $CLEAN_UP_OUTPUTS_SUCCESS == 0 && \
       $DISTRIBUTION_BUCKET_SUCCESS == 0 && \
       $GCS_TEMPORARY_HOLD_SUCCESS == 0 && \

migrations/versions/33d66c56823a_.py

@@ -0,0 +1,24 @@
+"""empty message
+
+Revision ID: 33d66c56823a
+Revises: 52aa9f44b6a7, 7a16ce1fe774
+Create Date: 2024-07-29 20:33:23.329220
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '33d66c56823a'
+down_revision = ('52aa9f44b6a7', '7a16ce1fe774')
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    pass
+
+
+def downgrade() -> None:
+    pass