From 0d0154f465a1b06e128d791d8b6e9c6e3d947f2b Mon Sep 17 00:00:00 2001
From: Jover Lee <joverlee521@gmail.com>
Date: Mon, 30 Oct 2023 12:25:47 -0700
Subject: [PATCH] Remove set-branch-ingest-config
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Recently learned that Snakemake no longer overwrites the entire config
when providing nested config values via the `--config` option.¹ This
allows us to remove the set-branch-ingest-config script and directly set
branch config values via the command line option in our GitHub Action
workflows.

I've opted to use `jq` to interpolate the branch name into the S3_DST
because I didn't want to fiddle with escaping quotes in the JSON string.

¹ https://github.com/nextstrain/dengue/pull/13#discussion_r1375081279
---
 .../workflows/fetch-and-ingest-branch.yaml    | 11 ++++---
 phylogenetic/bin/set-branch-ingest-config     | 29 -------------------
 2 files changed, 5 insertions(+), 35 deletions(-)
 delete mode 100755 phylogenetic/bin/set-branch-ingest-config

diff --git a/.github/workflows/fetch-and-ingest-branch.yaml b/.github/workflows/fetch-and-ingest-branch.yaml
index ce834c4f..349b1886 100644
--- a/.github/workflows/fetch-and-ingest-branch.yaml
+++ b/.github/workflows/fetch-and-ingest-branch.yaml
@@ -26,11 +26,10 @@ jobs:
 
       - name: run_pipeline
         run: |
+          # Create JSON string for the nested upload config
           GITHUB_BRANCH=${GITHUB_REF#refs/heads/}
-          ./phylogenetic/bin/set-branch-ingest-config \
-            --config-yaml ingest/config/optional.yaml \
-            --s3-dst s3://nextstrain-data/files/workflows/monkeypox/branch/"${GITHUB_BRANCH}" \
-            > ingest/config/optional-branch.yaml
+          S3_DST="s3://nextstrain-data/files/workflows/monkeypox/branch/${GITHUB_BRANCH}"
+          UPLOAD_CONFIG=$(jq -n --arg S3_DST "$S3_DST" '{"s3": {"dst": $S3_DST }}')
 
           nextstrain build \
             --aws-batch \
@@ -43,8 +42,8 @@ jobs:
             --env AWS_SECRET_ACCESS_KEY \
             --env GITHUB_RUN_ID \
             ingest \
-              --configfiles config/config.yaml config/optional-branch.yaml \
-              --config trigger_rebuild=False
+              --configfiles config/config.yaml config/optional.yaml \
+              --config trigger_rebuild=False send_slack_notifications=False upload="$UPLOAD_CONFIG"
         env:
           AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
diff --git a/phylogenetic/bin/set-branch-ingest-config b/phylogenetic/bin/set-branch-ingest-config
deleted file mode 100755
index d1a2d627..00000000
--- a/phylogenetic/bin/set-branch-ingest-config
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env python3
-"""
-Edits the config YAML file and outputs the new config to stdout.
-"""
-import argparse
-import yaml
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(
-        description=__doc__,
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter
-    )
-    parser.add_argument("--config-yaml", default="ingest/config/optional.yaml",
-        help="Path to config YAML file to edit.")
-    parser.add_argument("--s3-dst", required=True,
-        help="The S3 destination to add to the config YAML file")
-    parser.add_argument("--send-slack-notifications", action="store_true",
-        help="Set the `send_slack_notifications` config to True")
-
-    args = parser.parse_args()
-
-    with open(args.config_yaml, 'r') as config_fh:
-        config = yaml.safe_load(config_fh)
-
-        config['upload']['s3']['dst'] = args.s3_dst
-        config['send_slack_notifications'] = args.send_slack_notifications
-
-        print(yaml.dump(config, default_flow_style=False, sort_keys=False))