Merge pull request #753 from hmcts/DTSPO-18841

DTSPO-18841 - start / stop by env
hmcts · Sep 23, 2024 · 6f9e0a1 · 6f9e0a1
2 parents 4cae4c3 + 335f706
commit 6f9e0a1
Show file tree

Hide file tree

Showing 6 changed files with 154 additions and 78 deletions.
diff --git a/.github/workflows/flexibleserver-auto-shutdown.yaml b/.github/workflows/flexibleserver-auto-shutdown.yaml
@@ -3,6 +3,8 @@ on:
   workflow_dispatch:
   schedule:
     - cron: '0 19 * * *' # Every day at 8pm BST
+env:
+  DEV_ENV: ${{ secrets.DEV_ENV }}
 permissions:
   id-token: write
 jobs:
@@ -17,16 +19,49 @@ jobs:
           client-id: 2b6fa9d7-7dba-4600-a58a-5e25554997aa # DTS AKS Auto-Shutdown
           tenant-id: 531ff96d-0ae9-462a-8d2d-bec7c0b42082 # HMCTS.NET
           allow-no-subscriptions: true
-      - name: Postgres Flexible server  Auto Shutdown
-        run: ./scripts/flexible-server/auto-start-stop.sh stop
+
+      - name: Staging - Postgres Flexible server Auto Stop 
+        run: ./scripts/flexible-server/auto-start-stop.sh stop staging
+        env:
+          DEV_ENV: ${{ env.DEV_ENV }}
+
+      - name: Test - Postgres Flexible server Auto Stop 
+        run: ./scripts/flexible-server/auto-start-stop.sh stop testing
+        env:
+          DEV_ENV: ${{ env.DEV_ENV }}
+
+      - name: Demo - Postgres Flexible server Auto Stop 
+        run: ./scripts/flexible-server/auto-start-stop.sh stop demo
+        env:
+          DEV_ENV: ${{ env.DEV_ENV }}
+
+      - name: Development - Postgres Flexible server Auto Stop 
+        run: ./scripts/flexible-server/auto-start-stop.sh stop development
         env:
-          DEV_ENV: ${{ secrets.DEV_ENV }}
+          DEV_ENV: ${{ env.DEV_ENV }}
+
+      - name: Sandbox - Postgres Flexible server Auto Stop 
+        run: ./scripts/flexible-server/auto-start-stop.sh stop sandbox
+        env:
+          DEV_ENV: ${{ env.DEV_ENV }}
+
+      - name: ITHC - Postgres Flexible server Auto Stop 
+        run: ./scripts/flexible-server/auto-start-stop.sh stop ithc
+        env:
+          DEV_ENV: ${{ env.DEV_ENV }}
+
+      - name: Untagged - Postgres Flexible server Auto Stop 
+        run: ./scripts/flexible-server/auto-start-stop.sh stop untagged
+        env:
+          DEV_ENV: ${{ env.DEV_ENV }}
 
       - name: Output log file
         run: ./scripts/common/log-output.sh
 
-      - name: Wait for Flexible servers to stop
+      - name: Wait for Flexible servers to start
+        if: env.DEV_ENV != 'true'
         run: sleep 600
-        
+
       - name: Postgres Flexible server status check
+        if: env.DEV_ENV != 'true'
         run: ./scripts/flexible-server/flexserverstatus.sh stop ${{ secrets.SHUTDOWN_NOTIFICATIONS_WEBHOOK }}
diff --git a/.github/workflows/flexibleserver-auto-start.yaml b/.github/workflows/flexibleserver-auto-start.yaml
@@ -3,6 +3,8 @@ on:
   workflow_dispatch:
   schedule:
     - cron: '30 5 * * 1-5' # Every weekday at 6:30am BST
+env:
+  DEV_ENV: ${{ secrets.DEV_ENV }}
 permissions:
   id-token: write
 jobs:
@@ -17,16 +19,49 @@ jobs:
           client-id: 2b6fa9d7-7dba-4600-a58a-5e25554997aa # DTS AKS Auto-Shutdown
           tenant-id: 531ff96d-0ae9-462a-8d2d-bec7c0b42082 # HMCTS.NET
           allow-no-subscriptions: true
-      - name: Postgres Flexible server  Auto Start
-        run: ./scripts/flexible-server/auto-start-stop.sh start
+
+      - name: Staging - Postgres Flexible server Auto Start
+        run: ./scripts/flexible-server/auto-start-stop.sh start staging
         env:
-          DEV_ENV: ${{ secrets.DEV_ENV }}
+          DEV_ENV: ${{ env.DEV_ENV }}
+
+      - name: Test - Postgres Flexible server Auto Start
+        run: ./scripts/flexible-server/auto-start-stop.sh start testing
+        env:
+          DEV_ENV: ${{ env.DEV_ENV }}
+
+      - name: Demo - Postgres Flexible server Auto Start
+        run: ./scripts/flexible-server/auto-start-stop.sh start demo
+        env:
+          DEV_ENV: ${{ env.DEV_ENV }}
+
+      - name: Development - Postgres Flexible server Auto Start
+        run: ./scripts/flexible-server/auto-start-stop.sh start development
+        env:
+          DEV_ENV: ${{ env.DEV_ENV }}
+
+      - name: Sandbox - Postgres Flexible server Auto Start
+        run: ./scripts/flexible-server/auto-start-stop.sh start sandbox
+        env:
+          DEV_ENV: ${{ env.DEV_ENV }}
+
+      - name: ITHC - Postgres Flexible server Auto Start
+        run: ./scripts/flexible-server/auto-start-stop.sh start ithc
+        env:
+          DEV_ENV: ${{ env.DEV_ENV }}
+
+      - name: Untagged - Postgres Flexible server Auto Start
+        run: ./scripts/flexible-server/auto-start-stop.sh start untagged
+        env:
+          DEV_ENV: ${{ env.DEV_ENV }}
 
       - name: Output log file
         run: ./scripts/common/log-output.sh
 
       - name: Wait for Flexible servers to start
+        if: env.DEV_ENV != 'true'
         run: sleep 600
-        
+
       - name: Postgres Flexible server status check
+        if: env.DEV_ENV != 'true'
         run: ./scripts/flexible-server/flexserverstatus.sh start ${{ secrets.SHUTDOWN_NOTIFICATIONS_WEBHOOK }}
diff --git a/scripts/common/common-functions.sh b/scripts/common/common-functions.sh
@@ -224,6 +224,7 @@ function should_skip_start_stop () {
         echo "false"
       fi
       return
+    log "No exclusion request found"
     fi
   done < <(jq -c '.[]' issues_list.json)
 # If its onDemand and there are no issues matching above we should skip startup

diff --git a/scripts/flexible-server/auto-start-stop.sh b/scripts/flexible-server/auto-start-stop.sh
@@ -16,10 +16,10 @@ if [[ "$MODE" != "start" && "$MODE" != "stop" ]]; then
     exit 1
 fi
 
-#MS az graph query to find and return a list of all PostgreSQL Flexible Servers tagged to be included in the auto-shutdown process.
-log "Running az graph query..."
-FLEXIBLE_SERVERS=$(az graph query -q "resources | where type =~ 'microsoft.dbforpostgresql/flexibleservers' | where tags.autoShutdown == 'true' | project name, resourceGroup, subscriptionId, ['tags'], properties.state, ['id']" --first 1000 -o json)
-log "az graph query complete"
+FLEXIBLE_SERVERS=$(get_flexible_sql_servers "$2")
+flexible_server_count=$(jq -c -r '.count' <<< $FLEXIBLE_SERVERS)
+log "$flexible_server_count Flexible Servers found"
+log "----------------------------------------------"
 
 # For each PostgreSQL Flexible Server returned from the az graph query start another loop
 jq -c '.data[]' <<<$FLEXIBLE_SERVERS | while read flexibleserver; do
@@ -50,7 +50,7 @@ jq -c '.data[]' <<<$FLEXIBLE_SERVERS | while read flexibleserver; do
     if [[ $SKIP == "false" ]]; then
         if [[ $DEV_ENV != "true" ]]; then
             flexible_server_state_messages
-            az postgres flexible-server $MODE --resource-group $RESOURCE_GROUP --name $SERVER_NAME --subscription $SUBSCRIPTION --no-wait || echo Ignoring any errors while $MODE operation on sql server
+            az postgres flexible-server $MODE --resource-group $RESOURCE_GROUP --name $SERVER_NAME --subscription $SUBSCRIPTION_ID --no-wait || echo Ignoring any errors while $MODE operation on sql server
         else
             ts_echo_color BLUE "Development Env: simulating state commands only."
             flexible_server_state_messages

diff --git a/scripts/flexible-server/common-functions.sh b/scripts/flexible-server/common-functions.sh
@@ -1,23 +1,39 @@
 #!/bin/bash
+function get_flexible_sql_servers() {
+  #MS az graph query to find and return a list of all PostgreSQL Flexible Servers tagged to be included in the auto-shutdown process.
+  log "----------------------------------------------"
+  log "Running az graph query..."
+
+  if [ -z $1 ]; then
+    env_selector=""
+  elif [ $1 == "untagged" ]; then
+    env_selector="| where isnull(tags.environment)"
+  else
+    env_selector="| where tags.environment == '$1'"
+  fi
+
+  az graph query -q "
+    resources
+    | where type =~ 'microsoft.dbforpostgresql/flexibleservers'
+    | where tags.autoShutdown == 'true'
+    $env_selector
+    | project name, resourceGroup, subscriptionId, ['tags'], properties.state, ['id']
+    " --first 1000 -o json
+
+  log "az graph query complete"
+}
+
 
 # Function that accepts the PostgreSQL flexible server json as input and sets variables for later use to stop or start as required.
 function get_flexible_sql_server_details() {
   RESOURCE_GROUP=$(jq -r '.resourceGroup' <<< $flexibleserver)
-  log "$RESOURCE_GROUP"
   SERVER_ID=$(jq -r '.id' <<< $flexibleserver)
-  log "$SERVER_ID"
   SERVER_NAME=$(jq -r '.name' <<< $flexibleserver)
-  log "$SERVER_NAME"
   ENVIRONMENT=$(echo $SERVER_NAME | rev | cut -d'-' -f 1 | rev )
-  log "$ENVIRONMENT"
   BUSINESS_AREA=$( jq -r 'if (.tags.businessArea | ascii_downcase) == "ss" then "cross-cutting" else .tags.businessArea | ascii_downcase end' <<< $flexibleserver)
-  log "$BUSINESS_AREA"
   STARTUP_MODE=$(jq -r '.tags.startupMode' <<< $flexibleserver)
-  log "$STARTUP_MODE"
   SERVER_STATE=$(jq -r '.properties_state' <<< $flexibleserver)
-  log "$SERVER_STATE"
-  SUBSCRIPTION=$(jq -r '.subscriptionId' <<< $flexibleserver)
-  log "$SUBSCRIPTION"
+  SUBSCRIPTION_ID=$(jq -r '.subscriptionId' <<< $flexibleserver)
 }
 
 function flexible_server_state_messages() {

diff --git a/scripts/flexible-server/flexserverstatus.sh b/scripts/flexible-server/flexserverstatus.sh
@@ -18,65 +18,54 @@ if [[ "$MODE" != "start" && "$MODE" != "stop" ]]; then
     exit 1
 fi
 
-# Find all subscriptions that are available to the credential used and saved to SUBSCRIPTIONS variable
-SUBSCRIPTIONS=$(az account list -o json)
+FLEXIBLE_SERVERS=$(get_flexible_sql_servers)
 
-# For each subscription found, start the loop
-jq -c '.[]' <<< $SUBSCRIPTIONS | while read subscription
-do
+# For each Flexible SQL Server found in the function `get_subscription_flexible_sql_servers` start another loop
+jq -c '.[]' <<<$FLEXIBLE_SERVERS | while read flexibleserver; do
+    # Function that returns the Resource Group, Id and Name of the Flexible SQL Server and its current state as variables
+    get_flexible_sql_server_details
 
-    # Function that returns the Subscription Id and Name as variables, sets the subscription as the default then returns a json formatted variable of available Flexible SQL Servers with an autoshutdown tag
-    get_subscription_flexible_sql_servers
-    echo "Scanning $SUBSCRIPTION_NAME..."
+    # Set variables based on inputs which are used to decide when to SKIP an environment
+    if [[ $ENVIRONMENT == "stg" ]]; then
+        flexible_server_env=${ENVIRONMENT/stg/Staging}
+    elif [[ $ENVIRONMENT == "sbox" ]]; then
+        flexible_server_env=${ENVIRONMENT/sbox/Sandbox}
+    else
+        flexible_server_env=$ENVIRONMENT
+    fi
 
-    # For each Flexible SQL Server found in the function `get_subscription_flexible_sql_servers` start another loop
-    jq -c '.[]'<<< $FLEXIBLE_SERVERS | while read flexibleserver
-    do
-        # Function that returns the Resource Group, Id and Name of the Flexible SQL Server and its current state as variables
-        get_flexible_sql_server_details
+    flexible_server_business_area=$BUSINESS_AREA
 
-        # Set variables based on inputs which are used to decide when to SKIP an environment
-        if [[  $ENVIRONMENT == "stg" ]]; then
-            flexible_server_env=${ENVIRONMENT/stg/Staging}
-        elif [[ $ENVIRONMENT == "sbox" ]]; then
-            flexible_server_env=${ENVIRONMENT/sbox/Sandbox}
-        else
-            flexible_server_env=$ENVIRONMENT
-        fi
+    # SKIP variable updated based on the output of the `should_skip_start_stop` function which calculates its value
+    # based on the issues_list.json file which contains user requests to keep environments online after normal hours
+    SKIP=$(should_skip_start_stop $flexible_server_env $flexible_server_business_area $MODE)
 
-        flexible_server_business_area=$BUSINESS_AREA
+    # Setup message output templates for later use
+    logMessage="Flexible SQL Server: $SERVER_NAME in Subscription: $SUBSCRIPTION_ID  ResourceGroup: $RESOURCE_GROUP is in $SERVER_STATE state after $MODE action"
+    slackMessage="Flexible SQL Server: *$SERVER_NAME* in Subscription: *$SUBSCRIPTION_ID* is in *$SERVER_STATE* state after *$MODE* action"
 
-        # SKIP variable updated based on the output of the `should_skip_start_stop` function which calculates its value
-        # based on the issues_list.json file which contains user requests to keep environments online after normal hours
-        SKIP=$(should_skip_start_stop $flexible_server_env $flexible_server_business_area $MODE)
-
-        # Setup message output templates for later use
-        logMessage="Flexible SQL Server: $SERVER_NAME in Subscription: $SUBSCRIPTION_NAME  ResourceGroup: $RESOURCE_GROUP is in $SERVER_STATE state after $MODE action"
-        slackMessage="Flexible SQL Server: *$SERVER_NAME* in Subscription: *$SUBSCRIPTION_NAME* is in *$SERVER_STATE* state after *$MODE* action"
-
-        # If SKIP is false then we progress with the status check for the particular Flexible server in this loop run, if SKIP is true then do nothing
-        if [[ $SKIP == "false" ]]; then
-            # Check state of the Flexible SQL Server and print output as required
-            # Depending on the value of MODE a notification will also be sent
-            #    - If MODE = Start then a stopped Flexible SQL Server is incorrect and we should notify
-            #    - If MODE = Stop then a running Flexible SQL Server is incorrect and we should notify
-            #    - If neither Running or Stopped is found then something else is going on and we should notify
-            case "$SERVER_STATE" in
-                *"Ready"*)
-                    ts_echo_color $( [[ $MODE == "start" ]] && echo GREEN || echo RED ) "$logMessage"
-                    [[ $MODE == "stop" ]] && auto_shutdown_notification ":red_circle: $slackMessage"
-                    ;;
-                *"Stopped"*)
-                    ts_echo_color $( [[ $MODE == "start" ]] && echo RED || echo GREEN ) "$logMessage"
-                    [[ $MODE == "start" ]] && auto_shutdown_notification ":red_circle: $slackMessage"
-                    ;;
-                *)
-                    ts_echo_color AMBER "$logMessage"
-                    auto_shutdown_notification ":yellow_circle: $slackMessage"
-                    ;;
-            esac
-        else
-            ts_echo_color AMBER "Flexible SQL Server: $SERVER_NAME in ResourceGroup: $RESOURCE_GROUP has been skipped from today's $MODE operation schedule"
-        fi
-    done
+    # If SKIP is false then we progress with the status check for the particular Flexible server in this loop run, if SKIP is true then do nothing
+    if [[ $SKIP == "false" ]]; then
+        # Check state of the Flexible SQL Server and print output as required
+        # Depending on the value of MODE a notification will also be sent
+        #    - If MODE = Start then a stopped Flexible SQL Server is incorrect and we should notify
+        #    - If MODE = Stop then a running Flexible SQL Server is incorrect and we should notify
+        #    - If neither Running or Stopped is found then something else is going on and we should notify
+        case "$SERVER_STATE" in
+        *"Ready"*)
+            ts_echo_color $([[ $MODE == "start" ]] && echo GREEN || echo RED) "$logMessage"
+            [[ $MODE == "stop" ]] && auto_shutdown_notification ":red_circle: $slackMessage"
+            ;;
+        *"Stopped"*)
+            ts_echo_color $([[ $MODE == "start" ]] && echo RED || echo GREEN) "$logMessage"
+            [[ $MODE == "start" ]] && auto_shutdown_notification ":red_circle: $slackMessage"
+            ;;
+        *)
+            ts_echo_color AMBER "$logMessage"
+            auto_shutdown_notification ":yellow_circle: $slackMessage"
+            ;;
+        esac
+    else
+        ts_echo_color AMBER "Flexible SQL Server: $SERVER_NAME in ResourceGroup: $RESOURCE_GROUP has been skipped from today's $MODE operation schedule"
+    fi
 done