diff --git a/docker/Dockerfile b/docker/Dockerfile index 5dbdf17ce4..2682b18a58 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -35,7 +35,8 @@ RUN mkdir -p ${PUDL_INPUT} ${PUDL_OUTPUT} ${DAGSTER_HOME} ${PUDL_REPO} # Copy dagster configuration file COPY docker/dagster.yaml ${DAGSTER_HOME}/dagster.yaml -# Copy the cloned pudl repository into the user's home directory +# Copy the cloned pudl repository into the container +# This includes the .git directory, so it is a whole repo COPY --chown=${MAMBA_USER}:${MAMBA_USER} . ${PUDL_REPO} # Create a conda environment based on the specification in the repo @@ -60,6 +61,5 @@ RUN ${CONDA_RUN} bash -c 'curl -L https://fly.io/install.sh | sh' ENV PATH="${CONTAINER_HOME}/.fly/bin:$PATH" WORKDIR ${PUDL_REPO} -RUN git config -l # Run the unit tests: CMD ["micromamba", "run", "--prefix", "${CONDA_PREFIX}", "--attach", "''", "pytest", "test/unit"] diff --git a/docker/gcp_pudl_etl.sh b/docker/gcp_pudl_etl.sh index df4b587b64..6d5aabcd5e 100644 --- a/docker/gcp_pudl_etl.sh +++ b/docker/gcp_pudl_etl.sh @@ -48,16 +48,12 @@ function run_pudl_etl() { } function shutdown_vm() { - # Copy the outputs to the GCS bucket - upload_file_to_slack $LOGFILE "pudl_etl logs for $ACTION_SHA-$GITHUB_REF:" - + # Shut down the vm instance when the etl is done. echo "Shutting down VM." - # # Shut down the vm instance when the etl is done. ACCESS_TOKEN=`curl \ "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" \ -H "Metadata-Flavor: Google" | jq -r '.access_token'` - curl -X POST -H "Content-Length: 0" -H "Authorization: Bearer ${ACCESS_TOKEN}" https://compute.googleapis.com/compute/v1/projects/catalyst-cooperative-pudl/zones/$GCE_INSTANCE_ZONE/instances/$GCE_INSTANCE/stop } @@ -102,6 +98,14 @@ function notify_slack() { send_slack_msg "$message" } +if [ $GITHUB_ACTION_TRIGGER = "workflow_dispatch" ]; then + git config user.email "pudl@catalyst.coop" + git config user.name "pudlbot" + git remote set-url origin https://pudlbot:$PUDL_BOT_PAT@github.com/catalyst-cooperative/pudl.git + git tag -a -m "Nightly test tag" nightly-tag-test + git push origin nightly-tag-test + shutdown_vm + # # Run ETL. Copy outputs to GCS and shutdown VM if ETL succeeds or fails # 2>&1 redirects stderr to stdout. run_pudl_etl 2>&1 | tee $LOGFILE @@ -116,6 +120,7 @@ if [[ $ETL_SUCCESS == 0 ]]; then # Update the nightly branch to point at newly successful nightly build tag git config user.email "pudl@catalyst.coop" git config user.name "pudlbot" + git remote set-url origin https://pudlbot:$PUDL_BOT_PAT@github.com/catalyst-cooperative/pudl.git git checkout nightly git merge --ff-only $NIGHTLY_TAG git push