diff --git a/.github/workflows/fetch-and-ingest-branch.yaml b/.github/workflows/fetch-and-ingest-branch.yaml index ce834c4f..310895b7 100644 --- a/.github/workflows/fetch-and-ingest-branch.yaml +++ b/.github/workflows/fetch-and-ingest-branch.yaml @@ -21,16 +21,12 @@ jobs: cli-version: ">=7.1.0" python-version: "3.10" - - name: install-pyyaml - run: python3 -m pip install pyyaml - - name: run_pipeline run: | + # Create JSON string for the nested upload config GITHUB_BRANCH=${GITHUB_REF#refs/heads/} - ./phylogenetic/bin/set-branch-ingest-config \ - --config-yaml ingest/config/optional.yaml \ - --s3-dst s3://nextstrain-data/files/workflows/monkeypox/branch/"${GITHUB_BRANCH}" \ - > ingest/config/optional-branch.yaml + S3_DST="s3://nextstrain-data/files/workflows/monkeypox/branch/${GITHUB_BRANCH}" + UPLOAD_CONFIG=$(jq -cn --arg S3_DST "$S3_DST" '{"s3": {"dst": $S3_DST }}') nextstrain build \ --aws-batch \ @@ -43,8 +39,8 @@ jobs: --env AWS_SECRET_ACCESS_KEY \ --env GITHUB_RUN_ID \ ingest \ - --configfiles config/config.yaml config/optional-branch.yaml \ - --config trigger_rebuild=False + --configfiles config/config.yaml config/optional.yaml \ + --config trigger_rebuild=False send_slack_notifications=False upload="$UPLOAD_CONFIG" env: AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/phylogenetic/bin/set-branch-ingest-config b/phylogenetic/bin/set-branch-ingest-config deleted file mode 100755 index d1a2d627..00000000 --- a/phylogenetic/bin/set-branch-ingest-config +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 -""" -Edits the config YAML file and outputs the new config to stdout. -""" -import argparse -import yaml - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument("--config-yaml", default="ingest/config/optional.yaml", - help="Path to config YAML file to edit.") - parser.add_argument("--s3-dst", required=True, - help="The S3 destination to add to the config YAML file") - parser.add_argument("--send-slack-notifications", action="store_true", - help="Set the `send_slack_notifications` config to True") - - args = parser.parse_args() - - with open(args.config_yaml, 'r') as config_fh: - config = yaml.safe_load(config_fh) - - config['upload']['s3']['dst'] = args.s3_dst - config['send_slack_notifications'] = args.send_slack_notifications - - print(yaml.dump(config, default_flow_style=False, sort_keys=False))