From 3bfa00a796f79dd266d165d9e4c1a29d3e12f83e Mon Sep 17 00:00:00 2001 From: Hannah Pho Date: Wed, 4 Dec 2024 17:24:07 -0500 Subject: [PATCH] Set up "cron" testing job in staging (#4769) * This is a cron testing job, but it will never run automatically. Instead it can be manually run as part of the release process. * Also, add a comment on the dev schedule string. --- deploy/helm_charts/envs/dev.yaml | 2 +- deploy/helm_charts/envs/staging.yaml | 6 ++++++ tools/delete_cron_outputs/run.sh | 12 +++++++++--- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/deploy/helm_charts/envs/dev.yaml b/deploy/helm_charts/envs/dev.yaml index b40b138058..8598f8263f 100644 --- a/deploy/helm_charts/envs/dev.yaml +++ b/deploy/helm_charts/envs/dev.yaml @@ -87,4 +87,4 @@ cronTesting: enabled: true screenshotDomain: "dev.datacommons.org" nodePool: "default-pool" - schedule: "* * 31 2 *" + schedule: "* * 31 2 *" # invalid day for month -> never run automatically diff --git a/deploy/helm_charts/envs/staging.yaml b/deploy/helm_charts/envs/staging.yaml index 112b00804d..e49ba1024f 100644 --- a/deploy/helm_charts/envs/staging.yaml +++ b/deploy/helm_charts/envs/staging.yaml @@ -73,3 +73,9 @@ serviceGroups: svg: blocklistFile: ["dc/g/Uncategorized", "oecd/g/OECD"] + +cronTesting: + enabled: true + screenshotDomain: "staging.datacommons.org" + nodePool: "default-pool" + schedule: "* * 31 2 *" # invalid day for month -> never run automatically diff --git a/tools/delete_cron_outputs/run.sh b/tools/delete_cron_outputs/run.sh index b5ea75baab..4500ba8d9a 100755 --- a/tools/delete_cron_outputs/run.sh +++ b/tools/delete_cron_outputs/run.sh @@ -17,7 +17,13 @@ set -e # Delete GCS folders that are older than this many days MAX_DAYS_OLD=90 -PARENT_FOLDERS=("gs://datcom-website-periodic-testing/bard" "gs://datcom-website-periodic-testing/autopush" "gs://datcom-website-screenshot/autopush.datacommons.org/") +PARENT_FOLDERS=( + "gs://datcom-website-periodic-testing/bard" + "gs://datcom-website-periodic-testing/autopush" + "gs://datcom-website-screenshot/autopush.datacommons.org/" + "gs://datcom-website-periodic-testing/staging" + "gs://datcom-website-screenshot/staging.datacommons.org/" +) CURRENT_TS=$(TZ="America/Los_Angeles" date +"%s") for parent_folder in "${PARENT_FOLDERS[@]}"; do @@ -29,10 +35,10 @@ for parent_folder in "${PARENT_FOLDERS[@]}"; do folder_timestamp=$(date -j -f "%Y_%m_%d_%H_%M_%S" "$folder_name" +"%s") # get the difference in days between current timestamp and folder timestamp timestamp_difference=$((CURRENT_TS - folder_timestamp)) - days_difference=$(( timestamp_difference / (60*60*24) )) + days_difference=$((timestamp_difference / (60 * 60 * 24))) # delete the folder if its older than MAX_DAYS_OLD if [[ "$days_difference" -gt "$MAX_DAYS_OLD" ]]; then - gsutil rm -r $folder + gsutil -m rm -r $folder echo "deleted $folder" fi done