From 193c311cee05d4ca212dfc388cb0e2b4f313120d Mon Sep 17 00:00:00 2001 From: Jover Date: Fri, 14 Jul 2023 16:10:15 -0700 Subject: [PATCH 01/10] Copy notify-slack from ncov-ingest Copied from https://github.com/nextstrain/ncov-ingest/blob/8442ba80bc2ab16c345db1ad53233542fca343fe/bin/notify-slack Subsequent copies of this script that are functionally identical: - https://github.com/nextstrain/forecasts-ncov/blob/b7229bad08b047d480a2b40c4bafb1e33b7fe84b/ingest/bin/notify-slack - https://github.com/nextstrain/rsv/blob/ba171f4a43110382c38b6154be3febd50408d7bf/ingest/bin/notify-slack There was a one line difference in the rsv script that will be omitted in this repo: ``` @@ -1,4 +1,5 @@ #!/bin/bash +# Originally copied from nextstrain/ncov-ingest repo set -euo pipefail : "${SLACK_TOKEN:?The SLACK_TOKEN environment variable is required.}" ``` --- README.md | 1 + notify-slack | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100755 notify-slack diff --git a/README.md b/README.md index c7d1a39..af09812 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ approach to "ingest" has been discussed in various internal places, including: Scripts for supporting ingest workflow automation that don’t really belong in any of our existing tools. +- [notify-slack](notify-slack) - Send message or file to Slack - [s3-object-exists](s3-object-exists) - Used to prevent 404 errors during S3 file comparisons in the notify-* scripts - [trigger](trigger) - Triggers downstream GitHub Actions via the GitHub API using repository_dispatch events. diff --git a/notify-slack b/notify-slack new file mode 100755 index 0000000..6695d83 --- /dev/null +++ b/notify-slack @@ -0,0 +1,44 @@ +#!/bin/bash +set -euo pipefail + +: "${SLACK_TOKEN:?The SLACK_TOKEN environment variable is required.}" +: "${SLACK_CHANNELS:?The SLACK_CHANNELS environment variable is required.}" + +upload=0 +args=() + +for arg; do + case "$arg" in + --upload) + upload=1;; + *) + args+=("$arg");; + esac +done + +set -- "${args[@]}" + +text="${1:?Some message text is required.}" + +if [[ "$upload" == 1 ]]; then + echo "Uploading data to Slack with the message: $text" + curl https://slack.com/api/files.upload \ + --header "Authorization: Bearer $SLACK_TOKEN" \ + --form-string channels="$SLACK_CHANNELS" \ + --form-string title="$text" \ + --form-string filename="$text" \ + --form file=@/dev/stdin \ + --form filetype=text \ + --fail --silent --show-error \ + --http1.1 \ + --output /dev/null +else + echo "Posting Slack message: $text" + curl https://slack.com/api/chat.postMessage \ + --header "Authorization: Bearer $SLACK_TOKEN" \ + --form-string channel="$SLACK_CHANNELS" \ + --form-string text="$text" \ + --fail --silent --show-error \ + --http1.1 \ + --output /dev/null +fi From ba0769b1da6d9124c4ffbd93181cc128a59fa19d Mon Sep 17 00:00:00 2001 From: Jover Date: Fri, 29 Jul 2022 14:42:50 -0700 Subject: [PATCH 02/10] notify-slack: support threaded messages Changes copied from https://github.com/nextstrain/monkeypox/commit/875789012f09ae7c07db3648b5c8fd5b7d5af3ed Subsequent copies of this script that contain identical changes: - https://github.com/nextstrain/dengue/blob/247b2fd897361f2548627de1d97d45fae4115c5c/ingest/bin/notify-slack - https://github.com/nextstrain/zika/blob/4ac8d526f9f14be10b7e8858ad469a40b72a505e/ingest/bin/notify-slack These changes use optional args so they will not require changes to calls of the previous version of the script in ncov-ingest, forecasts-ncov, and rsv. --- notify-slack | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/notify-slack b/notify-slack index 6695d83..7fcb4f7 100755 --- a/notify-slack +++ b/notify-slack @@ -5,12 +5,21 @@ set -euo pipefail : "${SLACK_CHANNELS:?The SLACK_CHANNELS environment variable is required.}" upload=0 +output=/dev/null +thread_ts="" +broadcast=0 args=() for arg; do case "$arg" in --upload) upload=1;; + --output=*) + output="${arg#*=}";; + --thread-ts=*) + thread_ts="${arg#*=}";; + --broadcast) + broadcast=1;; *) args+=("$arg");; esac @@ -27,18 +36,22 @@ if [[ "$upload" == 1 ]]; then --form-string channels="$SLACK_CHANNELS" \ --form-string title="$text" \ --form-string filename="$text" \ + --form-string thread_ts="$thread_ts" \ + --form-string reply_broadcast="$broadcast" \ --form file=@/dev/stdin \ --form filetype=text \ --fail --silent --show-error \ --http1.1 \ - --output /dev/null + --output "$output" else echo "Posting Slack message: $text" curl https://slack.com/api/chat.postMessage \ --header "Authorization: Bearer $SLACK_TOKEN" \ --form-string channel="$SLACK_CHANNELS" \ --form-string text="$text" \ + --form-string thread_ts="$thread_ts" \ + --form-string reply_broadcast="$broadcast" \ --fail --silent --show-error \ --http1.1 \ - --output /dev/null + --output "$output" fi From 83e871d6558c5c6e2ea9dfbdbd14eba28698967b Mon Sep 17 00:00:00 2001 From: Jover Date: Fri, 14 Jul 2023 17:14:06 -0700 Subject: [PATCH 03/10] Copy notify-on-job-start from monkeypox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copied from https://github.com/nextstrain/monkeypox/blob/5c461dc7e90cd70c1f16b193f82fd1666d4c95e2/ingest/bin/notify-on-job-start I decided to copy the version in monkeypox because it is easier to update to be a generalized script than the ncov-ingest version which has unique changes to support the run-nextclade-full scripts.ΒΉ ΒΉ https://github.com/nextstrain/ncov-ingest/compare/88be153e02acc5f318c970dcf44e8745128d469f...e898400a0ebf599dcb9e69f1f93dc8c108c97796 --- README.md | 1 + notify-on-job-start | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100755 notify-on-job-start diff --git a/README.md b/README.md index af09812..51638c5 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ approach to "ingest" has been discussed in various internal places, including: Scripts for supporting ingest workflow automation that don’t really belong in any of our existing tools. +- [notify-on-job-start](notify-on-job-start) - Send Slack message with details about workflow job on GitHub Actions and/or AWS Batch - [notify-slack](notify-slack) - Send message or file to Slack - [s3-object-exists](s3-object-exists) - Used to prevent 404 errors during S3 file comparisons in the notify-* scripts - [trigger](trigger) - Triggers downstream GitHub Actions via the GitHub API using repository_dispatch events. diff --git a/notify-on-job-start b/notify-on-job-start new file mode 100755 index 0000000..9410fa3 --- /dev/null +++ b/notify-on-job-start @@ -0,0 +1,24 @@ +#!/bin/bash +set -euo pipefail + +: "${SLACK_TOKEN:?The SLACK_TOKEN environment variable is required.}" +: "${SLACK_CHANNELS:?The SLACK_CHANNELS environment variable is required.}" + +: "${AWS_BATCH_JOB_ID:=}" +: "${GITHUB_RUN_ID:=}" + +bin="$(dirname "$0")" + +echo "Notifying Slack about started ingest job." +message="🐡 Monkeypox ingest job has started." + +if [[ -n "${GITHUB_RUN_ID}" ]]; then + message+=" The job was submitted by GitHub Action ." +fi + +if [[ -n "${AWS_BATCH_JOB_ID}" ]]; then + message+=" The job was launched as AWS Batch job \`${AWS_BATCH_JOB_ID}\` ()." + message+=" Follow along in your local \`monkeypox\` repo with: "'```'"nextstrain build --aws-batch --no-download --attach ${AWS_BATCH_JOB_ID} ingest/"'```' +fi + +"$bin"/notify-slack "$message" From 98b235a70254db8f0a22ccfc0bbada8aff5bd86b Mon Sep 17 00:00:00 2001 From: Jover Date: Fri, 14 Jul 2023 17:49:04 -0700 Subject: [PATCH 04/10] notify-on-job-start: Add job_name and repo_name args Generalize the script by using new args to customize the Slack message and point to the appropriate GitHub Action URL. These changes were made based on diffs with subsequent copies of the script that edited the Slack message: - https://github.com/nextstrain/rsv/blob/ba171f4a43110382c38b6154be3febd50408d7bf/ingest/bin/notify-on-job-start - https://github.com/nextstrain/forecasts-ncov/blob/70bf78f459a3706dd817ae5f711af3b74887d7b1/bin/notify-on-job-start - https://github.com/nextstrain/dengue/blob/247b2fd897361f2548627de1d97d45fae4115c5c/ingest/bin/notify-on-job-start This will require all calls of the script to be updated since job_name and repo_name are required args. --- notify-on-job-start | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/notify-on-job-start b/notify-on-job-start index 9410fa3..8404311 100755 --- a/notify-on-job-start +++ b/notify-on-job-start @@ -8,17 +8,19 @@ set -euo pipefail : "${GITHUB_RUN_ID:=}" bin="$(dirname "$0")" +job_name="${1:?A job name is required as the first argument}" +repo_name="${2:?A repository name is required as the second argument}" -echo "Notifying Slack about started ingest job." -message="🐡 Monkeypox ingest job has started." +echo "Notifying Slack about started ${job_name} job." +message="${job_name} job has started." if [[ -n "${GITHUB_RUN_ID}" ]]; then - message+=" The job was submitted by GitHub Action ." + message+=" The job was submitted by GitHub Action ." fi if [[ -n "${AWS_BATCH_JOB_ID}" ]]; then message+=" The job was launched as AWS Batch job \`${AWS_BATCH_JOB_ID}\` ()." - message+=" Follow along in your local \`monkeypox\` repo with: "'```'"nextstrain build --aws-batch --no-download --attach ${AWS_BATCH_JOB_ID} ingest/"'```' + message+=" Follow along in your local \`${repo_name}\` repo with: "'```'"nextstrain build --aws-batch --no-download --attach ${AWS_BATCH_JOB_ID} ingest"'```' fi "$bin"/notify-slack "$message" From 00472555fb969899d4a8b4d5c2c5c32cdb06fe57 Mon Sep 17 00:00:00 2001 From: Jover Date: Tue, 18 Jul 2023 11:30:02 -0700 Subject: [PATCH 05/10] notify-on-job-start: Add optional build_dir arg The default value is "ingest" which is the expected ingest directory for standard pathogen repos. Adding the optional arg to support customizations for historical repos such as ncov-ingest that do not follow the new standardized repo structure. --- notify-on-job-start | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/notify-on-job-start b/notify-on-job-start index 8404311..3b8ae65 100755 --- a/notify-on-job-start +++ b/notify-on-job-start @@ -10,6 +10,7 @@ set -euo pipefail bin="$(dirname "$0")" job_name="${1:?A job name is required as the first argument}" repo_name="${2:?A repository name is required as the second argument}" +build_dir="${3:-ingest}" echo "Notifying Slack about started ${job_name} job." message="${job_name} job has started." @@ -20,7 +21,7 @@ fi if [[ -n "${AWS_BATCH_JOB_ID}" ]]; then message+=" The job was launched as AWS Batch job \`${AWS_BATCH_JOB_ID}\` ()." - message+=" Follow along in your local \`${repo_name}\` repo with: "'```'"nextstrain build --aws-batch --no-download --attach ${AWS_BATCH_JOB_ID} ingest"'```' + message+=" Follow along in your local \`${repo_name}\` repo with: "'```'"nextstrain build --aws-batch --no-download --attach ${AWS_BATCH_JOB_ID} ${build_dir}"'```' fi "$bin"/notify-slack "$message" From 192600841852273bd595c69775b075ce105dc5d7 Mon Sep 17 00:00:00 2001 From: Jover Date: Mon, 17 Jul 2023 16:33:14 -0700 Subject: [PATCH 06/10] Copy notify-on-job-fail from ncov-ingest Copied from https://github.com/nextstrain/ncov-ingest/blob/6fd5a9b1d87e59fab35173dbedf376632154943b/bin/notify-on-job-fail The following commits will update and generalize the script to incorporate changes made subsequent copies. --- README.md | 1 + notify-on-job-fail | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100755 notify-on-job-fail diff --git a/README.md b/README.md index 51638c5..84b855c 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ approach to "ingest" has been discussed in various internal places, including: Scripts for supporting ingest workflow automation that don’t really belong in any of our existing tools. +- [notify-on-job-fail](notify-on-job-fail) - Send Slack message with details about failed workflow job on GitHub Actions and/or AWS Batch - [notify-on-job-start](notify-on-job-start) - Send Slack message with details about workflow job on GitHub Actions and/or AWS Batch - [notify-slack](notify-slack) - Send message or file to Slack - [s3-object-exists](s3-object-exists) - Used to prevent 404 errors during S3 file comparisons in the notify-* scripts diff --git a/notify-on-job-fail b/notify-on-job-fail new file mode 100755 index 0000000..3d49b93 --- /dev/null +++ b/notify-on-job-fail @@ -0,0 +1,21 @@ +#!/bin/bash +set -euo pipefail + +: "${SLACK_TOKEN:?The SLACK_TOKEN environment variable is required.}" +: "${SLACK_CHANNELS:?The SLACK_CHANNELS environment variable is required.}" + +bin="$(dirname "$0")" + +aws_batch_job_id="${AWS_BATCH_JOB_ID:-}" +github_run_id="${GITHUB_RUN_ID:-}" + +echo "Notifying Slack about failed ingest job." +message="❌ Ingest job has FAILED 😞 " + +if [ -n "${aws_batch_job_id}" ]; then + message+="See AWS Batch job \`${aws_batch_job_id}\` () for error details. " +elif [ -n "${github_run_id}" ]; then + message+="See GitHub Action for error details. " +fi + +"$bin"/notify-slack "$message" From 309ebbf4688ed933d226db0f0803b38c0b0a6ae0 Mon Sep 17 00:00:00 2001 From: Jover Date: Mon, 17 Jul 2023 18:03:07 -0700 Subject: [PATCH 07/10] notify-on-job-fail: stylistic updates Follow patterns set in notify-on-job-start where the environment variables `AWS_BATCH_JOB_ID` and `GITHUB_RUN_ID` are used directly instead of reassigning them to local variables and use double square brackets. --- notify-on-job-fail | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/notify-on-job-fail b/notify-on-job-fail index 3d49b93..af3bd76 100755 --- a/notify-on-job-fail +++ b/notify-on-job-fail @@ -4,18 +4,18 @@ set -euo pipefail : "${SLACK_TOKEN:?The SLACK_TOKEN environment variable is required.}" : "${SLACK_CHANNELS:?The SLACK_CHANNELS environment variable is required.}" -bin="$(dirname "$0")" +: "${AWS_BATCH_JOB_ID:=}" +: "${GITHUB_RUN_ID:=}" -aws_batch_job_id="${AWS_BATCH_JOB_ID:-}" -github_run_id="${GITHUB_RUN_ID:-}" +bin="$(dirname "$0")" echo "Notifying Slack about failed ingest job." message="❌ Ingest job has FAILED 😞 " -if [ -n "${aws_batch_job_id}" ]; then - message+="See AWS Batch job \`${aws_batch_job_id}\` () for error details. " -elif [ -n "${github_run_id}" ]; then - message+="See GitHub Action for error details. " +if [[ -n "${AWS_BATCH_JOB_ID}" ]]; then + message+="See AWS Batch job \`${AWS_BATCH_JOB_ID}\` () for error details. " +elif [[ -n "${GITHUB_RUN_ID}" ]]; then + message+="See GitHub Action for error details. " fi "$bin"/notify-slack "$message" From b4b406fd6f96dfc15404721cf3dd1e6af9079248 Mon Sep 17 00:00:00 2001 From: Jover Date: Mon, 17 Jul 2023 18:12:57 -0700 Subject: [PATCH 08/10] notify-on-job-fail: Add job_name and repo_name args Generalize the script by using new args to customize the Slack message and point to the appropriate GitHub Action URL. The repo_name arg was added based on diffs with subsequent copies of the script that edited the Slack message: - https://github.com/nextstrain/monkeypox/blob/5c461dc7e90cd70c1f16b193f82fd1666d4c95e2/ingest/bin/notify-on-job-fail - https://github.com/nextstrain/forecasts-ncov/blob/70bf78f459a3706dd817ae5f711af3b74887d7b1/ingest/bin/notify-on-job-fail - https://github.com/nextstrain/rsv/blob/ba171f4a43110382c38b6154be3febd50408d7bf/ingest/bin/notify-on-job-fail - https://github.com/nextstrain/dengue/blob/247b2fd897361f2548627de1d97d45fae4115c5c/ingest/bin/notify-on-job-fail - https://github.com/nextstrain/zika/blob/4ac8d526f9f14be10b7e8858ad469a40b72a505e/ingest/bin/notify-on-job-fail Although the job_name arg is not necessary based on diffs with subsequent copies of the script, it's nice to be able to customize the Slack message for different ingest jobs. It's also a plus to mirror the args for notify-on-job-start. This will require all calls of the script to be update since job_name and repo_name are required args. --- notify-on-job-fail | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/notify-on-job-fail b/notify-on-job-fail index af3bd76..65aab6e 100755 --- a/notify-on-job-fail +++ b/notify-on-job-fail @@ -8,14 +8,16 @@ set -euo pipefail : "${GITHUB_RUN_ID:=}" bin="$(dirname "$0")" +job_name="${1:?A job name is required as the first argument}" +repo_name="${2:?A repository name is required as the second argument}" -echo "Notifying Slack about failed ingest job." -message="❌ Ingest job has FAILED 😞 " +echo "Notifying Slack about failed ${job_name} job." +message="❌ ${job_name} job has FAILED 😞 " if [[ -n "${AWS_BATCH_JOB_ID}" ]]; then message+="See AWS Batch job \`${AWS_BATCH_JOB_ID}\` () for error details. " elif [[ -n "${GITHUB_RUN_ID}" ]]; then - message+="See GitHub Action for error details. " + message+="See GitHub Action for error details. " fi "$bin"/notify-slack "$message" From b663e17211949c375ad2675abc50a9d5c24a715f Mon Sep 17 00:00:00 2001 From: Jover Date: Tue, 18 Jul 2023 14:56:04 -0700 Subject: [PATCH 09/10] notify-slack: remove reply_broadcast option for uploads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I realized in my testing of the script that broadcasting does not work for file uploads. Confirmed that this is not an argument for the files.upload API.ΒΉ Including the argument did not cause an error, but better to remove it so it doesn't confuse us in the future. ΒΉ https://api.slack.com/methods/files.upload --- notify-slack | 1 - 1 file changed, 1 deletion(-) diff --git a/notify-slack b/notify-slack index 7fcb4f7..db98bfb 100755 --- a/notify-slack +++ b/notify-slack @@ -37,7 +37,6 @@ if [[ "$upload" == 1 ]]; then --form-string title="$text" \ --form-string filename="$text" \ --form-string thread_ts="$thread_ts" \ - --form-string reply_broadcast="$broadcast" \ --form file=@/dev/stdin \ --form filetype=text \ --fail --silent --show-error \ From b2a0de70342cef1b2b27f3f139af262910466b24 Mon Sep 17 00:00:00 2001 From: Jover Date: Wed, 26 Jul 2023 15:31:15 -0700 Subject: [PATCH 10/10] notify-on-job-start/fail: replace repo_name with github_repo Using GitHub repos by org/name pair to make scripts more generalizable. This will also work really well with the `GITHUB_REPOSITORY` variable available for GitHub Action workflows. Co-authored-by: Victor Lin <13424970+victorlin@users.noreply.github.com> --- notify-on-job-fail | 4 ++-- notify-on-job-start | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/notify-on-job-fail b/notify-on-job-fail index 65aab6e..02cb6ba 100755 --- a/notify-on-job-fail +++ b/notify-on-job-fail @@ -9,7 +9,7 @@ set -euo pipefail bin="$(dirname "$0")" job_name="${1:?A job name is required as the first argument}" -repo_name="${2:?A repository name is required as the second argument}" +github_repo="${2:?A GitHub repository with owner and repository name is required as the second argument}" echo "Notifying Slack about failed ${job_name} job." message="❌ ${job_name} job has FAILED 😞 " @@ -17,7 +17,7 @@ message="❌ ${job_name} job has FAILED 😞 " if [[ -n "${AWS_BATCH_JOB_ID}" ]]; then message+="See AWS Batch job \`${AWS_BATCH_JOB_ID}\` () for error details. " elif [[ -n "${GITHUB_RUN_ID}" ]]; then - message+="See GitHub Action for error details. " + message+="See GitHub Action for error details. " fi "$bin"/notify-slack "$message" diff --git a/notify-on-job-start b/notify-on-job-start index 3b8ae65..3e44bb0 100755 --- a/notify-on-job-start +++ b/notify-on-job-start @@ -9,19 +9,19 @@ set -euo pipefail bin="$(dirname "$0")" job_name="${1:?A job name is required as the first argument}" -repo_name="${2:?A repository name is required as the second argument}" +github_repo="${2:?A GitHub repository with owner and repository name is required as the second argument}" build_dir="${3:-ingest}" echo "Notifying Slack about started ${job_name} job." message="${job_name} job has started." if [[ -n "${GITHUB_RUN_ID}" ]]; then - message+=" The job was submitted by GitHub Action ." + message+=" The job was submitted by GitHub Action ." fi if [[ -n "${AWS_BATCH_JOB_ID}" ]]; then message+=" The job was launched as AWS Batch job \`${AWS_BATCH_JOB_ID}\` ()." - message+=" Follow along in your local \`${repo_name}\` repo with: "'```'"nextstrain build --aws-batch --no-download --attach ${AWS_BATCH_JOB_ID} ${build_dir}"'```' + message+=" Follow along in your local clone of ${github_repo} with: "'```'"nextstrain build --aws-batch --no-download --attach ${AWS_BATCH_JOB_ID} ${build_dir}"'```' fi "$bin"/notify-slack "$message"