From 493b7ad5ec63a8060b87adcb7d3cf305d8e52e06 Mon Sep 17 00:00:00 2001 From: Jordan Hoey Date: Wed, 18 Sep 2024 14:36:36 +0100 Subject: [PATCH 1/3] adding log --- scripts/common/common-functions.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/common/common-functions.sh b/scripts/common/common-functions.sh index dacf9206..f84926a1 100644 --- a/scripts/common/common-functions.sh +++ b/scripts/common/common-functions.sh @@ -224,6 +224,7 @@ function should_skip_start_stop () { echo "false" fi return + log "No exclusion request found" fi done < <(jq -c '.[]' issues_list.json) # If its onDemand and there are no issues matching above we should skip startup From 415f2fc067d5e54c36bee77989665156a90ff569 Mon Sep 17 00:00:00 2001 From: Jordan Hoey Date: Fri, 20 Sep 2024 10:06:11 +0100 Subject: [PATCH 2/3] updating Flex server status script to wotk with new graph query --- scripts/flexible-server/auto-start-stop.sh | 2 +- scripts/flexible-server/common-functions.sh | 4 +- scripts/flexible-server/flexserverstatus.sh | 102 +++++++++----------- 3 files changed, 50 insertions(+), 58 deletions(-) diff --git a/scripts/flexible-server/auto-start-stop.sh b/scripts/flexible-server/auto-start-stop.sh index cd8e4195..fb06e3dc 100755 --- a/scripts/flexible-server/auto-start-stop.sh +++ b/scripts/flexible-server/auto-start-stop.sh @@ -50,7 +50,7 @@ jq -c '.data[]' <<<$FLEXIBLE_SERVERS | while read flexibleserver; do if [[ $SKIP == "false" ]]; then if [[ $DEV_ENV != "true" ]]; then flexible_server_state_messages - az postgres flexible-server $MODE --resource-group $RESOURCE_GROUP --name $SERVER_NAME --subscription $SUBSCRIPTION --no-wait || echo Ignoring any errors while $MODE operation on sql server + az postgres flexible-server $MODE --resource-group $RESOURCE_GROUP --name $SERVER_NAME --subscription $SUBSCRIPTION_ID --no-wait || echo Ignoring any errors while $MODE operation on sql server else ts_echo_color BLUE "Development Env: simulating state commands only." flexible_server_state_messages diff --git a/scripts/flexible-server/common-functions.sh b/scripts/flexible-server/common-functions.sh index 7ef2f790..814eac5e 100755 --- a/scripts/flexible-server/common-functions.sh +++ b/scripts/flexible-server/common-functions.sh @@ -16,8 +16,8 @@ function get_flexible_sql_server_details() { log "$STARTUP_MODE" SERVER_STATE=$(jq -r '.properties_state' <<< $flexibleserver) log "$SERVER_STATE" - SUBSCRIPTION=$(jq -r '.subscriptionId' <<< $flexibleserver) - log "$SUBSCRIPTION" + SUBSCRIPTION_ID=$(jq -r '.subscriptionId' <<< $flexibleserver) + log "$SUBSCRIPTION_ID" } function flexible_server_state_messages() { diff --git a/scripts/flexible-server/flexserverstatus.sh b/scripts/flexible-server/flexserverstatus.sh index c8e9004d..c16b5d43 100755 --- a/scripts/flexible-server/flexserverstatus.sh +++ b/scripts/flexible-server/flexserverstatus.sh @@ -18,65 +18,57 @@ if [[ "$MODE" != "start" && "$MODE" != "stop" ]]; then exit 1 fi -# Find all subscriptions that are available to the credential used and saved to SUBSCRIPTIONS variable -SUBSCRIPTIONS=$(az account list -o json) +#MS az graph query to find and return a list of all PostgreSQL Flexible Servers tagged to be included in the auto-shutdown process. +log "Running az graph query..." +FLEXIBLE_SERVERS=$(az graph query -q "resources | where type =~ 'microsoft.dbforpostgresql/flexibleservers' | where tags.autoShutdown == 'true' | project name, resourceGroup, subscriptionId, ['tags'], properties.state, ['id']" --first 1000 -o json) +log "az graph query complete" -# For each subscription found, start the loop -jq -c '.[]' <<< $SUBSCRIPTIONS | while read subscription -do +# For each Flexible SQL Server found in the function `get_subscription_flexible_sql_servers` start another loop +jq -c '.[]' <<<$FLEXIBLE_SERVERS | while read flexibleserver; do + # Function that returns the Resource Group, Id and Name of the Flexible SQL Server and its current state as variables + get_flexible_sql_server_details - # Function that returns the Subscription Id and Name as variables, sets the subscription as the default then returns a json formatted variable of available Flexible SQL Servers with an autoshutdown tag - get_subscription_flexible_sql_servers - echo "Scanning $SUBSCRIPTION_NAME..." + # Set variables based on inputs which are used to decide when to SKIP an environment + if [[ $ENVIRONMENT == "stg" ]]; then + flexible_server_env=${ENVIRONMENT/stg/Staging} + elif [[ $ENVIRONMENT == "sbox" ]]; then + flexible_server_env=${ENVIRONMENT/sbox/Sandbox} + else + flexible_server_env=$ENVIRONMENT + fi - # For each Flexible SQL Server found in the function `get_subscription_flexible_sql_servers` start another loop - jq -c '.[]'<<< $FLEXIBLE_SERVERS | while read flexibleserver - do - # Function that returns the Resource Group, Id and Name of the Flexible SQL Server and its current state as variables - get_flexible_sql_server_details + flexible_server_business_area=$BUSINESS_AREA - # Set variables based on inputs which are used to decide when to SKIP an environment - if [[ $ENVIRONMENT == "stg" ]]; then - flexible_server_env=${ENVIRONMENT/stg/Staging} - elif [[ $ENVIRONMENT == "sbox" ]]; then - flexible_server_env=${ENVIRONMENT/sbox/Sandbox} - else - flexible_server_env=$ENVIRONMENT - fi + # SKIP variable updated based on the output of the `should_skip_start_stop` function which calculates its value + # based on the issues_list.json file which contains user requests to keep environments online after normal hours + SKIP=$(should_skip_start_stop $flexible_server_env $flexible_server_business_area $MODE) - flexible_server_business_area=$BUSINESS_AREA + # Setup message output templates for later use + logMessage="Flexible SQL Server: $SERVER_NAME in Subscription: $SUBSCRIPTION_ID ResourceGroup: $RESOURCE_GROUP is in $SERVER_STATE state after $MODE action" + slackMessage="Flexible SQL Server: *$SERVER_NAME* in Subscription: *$SUBSCRIPTION_ID* is in *$SERVER_STATE* state after *$MODE* action" - # SKIP variable updated based on the output of the `should_skip_start_stop` function which calculates its value - # based on the issues_list.json file which contains user requests to keep environments online after normal hours - SKIP=$(should_skip_start_stop $flexible_server_env $flexible_server_business_area $MODE) - - # Setup message output templates for later use - logMessage="Flexible SQL Server: $SERVER_NAME in Subscription: $SUBSCRIPTION_NAME ResourceGroup: $RESOURCE_GROUP is in $SERVER_STATE state after $MODE action" - slackMessage="Flexible SQL Server: *$SERVER_NAME* in Subscription: *$SUBSCRIPTION_NAME* is in *$SERVER_STATE* state after *$MODE* action" - - # If SKIP is false then we progress with the status check for the particular Flexible server in this loop run, if SKIP is true then do nothing - if [[ $SKIP == "false" ]]; then - # Check state of the Flexible SQL Server and print output as required - # Depending on the value of MODE a notification will also be sent - # - If MODE = Start then a stopped Flexible SQL Server is incorrect and we should notify - # - If MODE = Stop then a running Flexible SQL Server is incorrect and we should notify - # - If neither Running or Stopped is found then something else is going on and we should notify - case "$SERVER_STATE" in - *"Ready"*) - ts_echo_color $( [[ $MODE == "start" ]] && echo GREEN || echo RED ) "$logMessage" - [[ $MODE == "stop" ]] && auto_shutdown_notification ":red_circle: $slackMessage" - ;; - *"Stopped"*) - ts_echo_color $( [[ $MODE == "start" ]] && echo RED || echo GREEN ) "$logMessage" - [[ $MODE == "start" ]] && auto_shutdown_notification ":red_circle: $slackMessage" - ;; - *) - ts_echo_color AMBER "$logMessage" - auto_shutdown_notification ":yellow_circle: $slackMessage" - ;; - esac - else - ts_echo_color AMBER "Flexible SQL Server: $SERVER_NAME in ResourceGroup: $RESOURCE_GROUP has been skipped from today's $MODE operation schedule" - fi - done + # If SKIP is false then we progress with the status check for the particular Flexible server in this loop run, if SKIP is true then do nothing + if [[ $SKIP == "false" ]]; then + # Check state of the Flexible SQL Server and print output as required + # Depending on the value of MODE a notification will also be sent + # - If MODE = Start then a stopped Flexible SQL Server is incorrect and we should notify + # - If MODE = Stop then a running Flexible SQL Server is incorrect and we should notify + # - If neither Running or Stopped is found then something else is going on and we should notify + case "$SERVER_STATE" in + *"Ready"*) + ts_echo_color $([[ $MODE == "start" ]] && echo GREEN || echo RED) "$logMessage" + [[ $MODE == "stop" ]] && auto_shutdown_notification ":red_circle: $slackMessage" + ;; + *"Stopped"*) + ts_echo_color $([[ $MODE == "start" ]] && echo RED || echo GREEN) "$logMessage" + [[ $MODE == "start" ]] && auto_shutdown_notification ":red_circle: $slackMessage" + ;; + *) + ts_echo_color AMBER "$logMessage" + auto_shutdown_notification ":yellow_circle: $slackMessage" + ;; + esac + else + ts_echo_color AMBER "Flexible SQL Server: $SERVER_NAME in ResourceGroup: $RESOURCE_GROUP has been skipped from today's $MODE operation schedule" + fi done From 335f7064948067a96e32e9be52d05a75378e08d6 Mon Sep 17 00:00:00 2001 From: Jordan Hoey Date: Mon, 23 Sep 2024 15:48:57 +0100 Subject: [PATCH 3/3] splitting server jobs by enviornment & fixing status script --- .../flexibleserver-auto-shutdown.yaml | 45 ++++++++++++++++--- .../workflows/flexibleserver-auto-start.yaml | 43 ++++++++++++++++-- scripts/flexible-server/auto-start-stop.sh | 8 ++-- scripts/flexible-server/common-functions.sh | 32 +++++++++---- scripts/flexible-server/flexserverstatus.sh | 5 +-- 5 files changed, 108 insertions(+), 25 deletions(-) diff --git a/.github/workflows/flexibleserver-auto-shutdown.yaml b/.github/workflows/flexibleserver-auto-shutdown.yaml index ce7096e8..ffa140f1 100644 --- a/.github/workflows/flexibleserver-auto-shutdown.yaml +++ b/.github/workflows/flexibleserver-auto-shutdown.yaml @@ -3,6 +3,8 @@ on: workflow_dispatch: schedule: - cron: '0 19 * * *' # Every day at 8pm BST +env: + DEV_ENV: ${{ secrets.DEV_ENV }} permissions: id-token: write jobs: @@ -17,16 +19,49 @@ jobs: client-id: 2b6fa9d7-7dba-4600-a58a-5e25554997aa # DTS AKS Auto-Shutdown tenant-id: 531ff96d-0ae9-462a-8d2d-bec7c0b42082 # HMCTS.NET allow-no-subscriptions: true - - name: Postgres Flexible server Auto Shutdown - run: ./scripts/flexible-server/auto-start-stop.sh stop + + - name: Staging - Postgres Flexible server Auto Stop + run: ./scripts/flexible-server/auto-start-stop.sh stop staging + env: + DEV_ENV: ${{ env.DEV_ENV }} + + - name: Test - Postgres Flexible server Auto Stop + run: ./scripts/flexible-server/auto-start-stop.sh stop testing + env: + DEV_ENV: ${{ env.DEV_ENV }} + + - name: Demo - Postgres Flexible server Auto Stop + run: ./scripts/flexible-server/auto-start-stop.sh stop demo + env: + DEV_ENV: ${{ env.DEV_ENV }} + + - name: Development - Postgres Flexible server Auto Stop + run: ./scripts/flexible-server/auto-start-stop.sh stop development env: - DEV_ENV: ${{ secrets.DEV_ENV }} + DEV_ENV: ${{ env.DEV_ENV }} + + - name: Sandbox - Postgres Flexible server Auto Stop + run: ./scripts/flexible-server/auto-start-stop.sh stop sandbox + env: + DEV_ENV: ${{ env.DEV_ENV }} + + - name: ITHC - Postgres Flexible server Auto Stop + run: ./scripts/flexible-server/auto-start-stop.sh stop ithc + env: + DEV_ENV: ${{ env.DEV_ENV }} + + - name: Untagged - Postgres Flexible server Auto Stop + run: ./scripts/flexible-server/auto-start-stop.sh stop untagged + env: + DEV_ENV: ${{ env.DEV_ENV }} - name: Output log file run: ./scripts/common/log-output.sh - - name: Wait for Flexible servers to stop + - name: Wait for Flexible servers to start + if: env.DEV_ENV != 'true' run: sleep 600 - + - name: Postgres Flexible server status check + if: env.DEV_ENV != 'true' run: ./scripts/flexible-server/flexserverstatus.sh stop ${{ secrets.SHUTDOWN_NOTIFICATIONS_WEBHOOK }} diff --git a/.github/workflows/flexibleserver-auto-start.yaml b/.github/workflows/flexibleserver-auto-start.yaml index 32c6eea0..067aa9c5 100644 --- a/.github/workflows/flexibleserver-auto-start.yaml +++ b/.github/workflows/flexibleserver-auto-start.yaml @@ -3,6 +3,8 @@ on: workflow_dispatch: schedule: - cron: '30 5 * * 1-5' # Every weekday at 6:30am BST +env: + DEV_ENV: ${{ secrets.DEV_ENV }} permissions: id-token: write jobs: @@ -17,16 +19,49 @@ jobs: client-id: 2b6fa9d7-7dba-4600-a58a-5e25554997aa # DTS AKS Auto-Shutdown tenant-id: 531ff96d-0ae9-462a-8d2d-bec7c0b42082 # HMCTS.NET allow-no-subscriptions: true - - name: Postgres Flexible server Auto Start - run: ./scripts/flexible-server/auto-start-stop.sh start + + - name: Staging - Postgres Flexible server Auto Start + run: ./scripts/flexible-server/auto-start-stop.sh start staging env: - DEV_ENV: ${{ secrets.DEV_ENV }} + DEV_ENV: ${{ env.DEV_ENV }} + + - name: Test - Postgres Flexible server Auto Start + run: ./scripts/flexible-server/auto-start-stop.sh start testing + env: + DEV_ENV: ${{ env.DEV_ENV }} + + - name: Demo - Postgres Flexible server Auto Start + run: ./scripts/flexible-server/auto-start-stop.sh start demo + env: + DEV_ENV: ${{ env.DEV_ENV }} + + - name: Development - Postgres Flexible server Auto Start + run: ./scripts/flexible-server/auto-start-stop.sh start development + env: + DEV_ENV: ${{ env.DEV_ENV }} + + - name: Sandbox - Postgres Flexible server Auto Start + run: ./scripts/flexible-server/auto-start-stop.sh start sandbox + env: + DEV_ENV: ${{ env.DEV_ENV }} + + - name: ITHC - Postgres Flexible server Auto Start + run: ./scripts/flexible-server/auto-start-stop.sh start ithc + env: + DEV_ENV: ${{ env.DEV_ENV }} + + - name: Untagged - Postgres Flexible server Auto Start + run: ./scripts/flexible-server/auto-start-stop.sh start untagged + env: + DEV_ENV: ${{ env.DEV_ENV }} - name: Output log file run: ./scripts/common/log-output.sh - name: Wait for Flexible servers to start + if: env.DEV_ENV != 'true' run: sleep 600 - + - name: Postgres Flexible server status check + if: env.DEV_ENV != 'true' run: ./scripts/flexible-server/flexserverstatus.sh start ${{ secrets.SHUTDOWN_NOTIFICATIONS_WEBHOOK }} \ No newline at end of file diff --git a/scripts/flexible-server/auto-start-stop.sh b/scripts/flexible-server/auto-start-stop.sh index fb06e3dc..f9d955d0 100755 --- a/scripts/flexible-server/auto-start-stop.sh +++ b/scripts/flexible-server/auto-start-stop.sh @@ -16,10 +16,10 @@ if [[ "$MODE" != "start" && "$MODE" != "stop" ]]; then exit 1 fi -#MS az graph query to find and return a list of all PostgreSQL Flexible Servers tagged to be included in the auto-shutdown process. -log "Running az graph query..." -FLEXIBLE_SERVERS=$(az graph query -q "resources | where type =~ 'microsoft.dbforpostgresql/flexibleservers' | where tags.autoShutdown == 'true' | project name, resourceGroup, subscriptionId, ['tags'], properties.state, ['id']" --first 1000 -o json) -log "az graph query complete" +FLEXIBLE_SERVERS=$(get_flexible_sql_servers "$2") +flexible_server_count=$(jq -c -r '.count' <<< $FLEXIBLE_SERVERS) +log "$flexible_server_count Flexible Servers found" +log "----------------------------------------------" # For each PostgreSQL Flexible Server returned from the az graph query start another loop jq -c '.data[]' <<<$FLEXIBLE_SERVERS | while read flexibleserver; do diff --git a/scripts/flexible-server/common-functions.sh b/scripts/flexible-server/common-functions.sh index 814eac5e..9bf6850f 100755 --- a/scripts/flexible-server/common-functions.sh +++ b/scripts/flexible-server/common-functions.sh @@ -1,23 +1,39 @@ #!/bin/bash +function get_flexible_sql_servers() { + #MS az graph query to find and return a list of all PostgreSQL Flexible Servers tagged to be included in the auto-shutdown process. + log "----------------------------------------------" + log "Running az graph query..." + + if [ -z $1 ]; then + env_selector="" + elif [ $1 == "untagged" ]; then + env_selector="| where isnull(tags.environment)" + else + env_selector="| where tags.environment == '$1'" + fi + + az graph query -q " + resources + | where type =~ 'microsoft.dbforpostgresql/flexibleservers' + | where tags.autoShutdown == 'true' + $env_selector + | project name, resourceGroup, subscriptionId, ['tags'], properties.state, ['id'] + " --first 1000 -o json + + log "az graph query complete" +} + # Function that accepts the PostgreSQL flexible server json as input and sets variables for later use to stop or start as required. function get_flexible_sql_server_details() { RESOURCE_GROUP=$(jq -r '.resourceGroup' <<< $flexibleserver) - log "$RESOURCE_GROUP" SERVER_ID=$(jq -r '.id' <<< $flexibleserver) - log "$SERVER_ID" SERVER_NAME=$(jq -r '.name' <<< $flexibleserver) - log "$SERVER_NAME" ENVIRONMENT=$(echo $SERVER_NAME | rev | cut -d'-' -f 1 | rev ) - log "$ENVIRONMENT" BUSINESS_AREA=$( jq -r 'if (.tags.businessArea | ascii_downcase) == "ss" then "cross-cutting" else .tags.businessArea | ascii_downcase end' <<< $flexibleserver) - log "$BUSINESS_AREA" STARTUP_MODE=$(jq -r '.tags.startupMode' <<< $flexibleserver) - log "$STARTUP_MODE" SERVER_STATE=$(jq -r '.properties_state' <<< $flexibleserver) - log "$SERVER_STATE" SUBSCRIPTION_ID=$(jq -r '.subscriptionId' <<< $flexibleserver) - log "$SUBSCRIPTION_ID" } function flexible_server_state_messages() { diff --git a/scripts/flexible-server/flexserverstatus.sh b/scripts/flexible-server/flexserverstatus.sh index c16b5d43..1d66aad9 100755 --- a/scripts/flexible-server/flexserverstatus.sh +++ b/scripts/flexible-server/flexserverstatus.sh @@ -18,10 +18,7 @@ if [[ "$MODE" != "start" && "$MODE" != "stop" ]]; then exit 1 fi -#MS az graph query to find and return a list of all PostgreSQL Flexible Servers tagged to be included in the auto-shutdown process. -log "Running az graph query..." -FLEXIBLE_SERVERS=$(az graph query -q "resources | where type =~ 'microsoft.dbforpostgresql/flexibleservers' | where tags.autoShutdown == 'true' | project name, resourceGroup, subscriptionId, ['tags'], properties.state, ['id']" --first 1000 -o json) -log "az graph query complete" +FLEXIBLE_SERVERS=$(get_flexible_sql_servers) # For each Flexible SQL Server found in the function `get_subscription_flexible_sql_servers` start another loop jq -c '.[]' <<<$FLEXIBLE_SERVERS | while read flexibleserver; do