testing

.github/workflows/deploy-stack.yml

@@ -49,7 +49,7 @@ jobs:
 
   down-services:
     needs:
-      - set-latest-versions
+      - set-service-versions
       - set-maintenance-mode
     strategy:
      fail-fast: false
@@ -63,7 +63,7 @@ jobs:
 
   upgrade-web:
     needs:
-      - set-latest-versions
+      - set-service-versions
       - set-maintenance-mode
     uses: ./.github/workflows/upgrade-web.yml
     with:
@@ -72,7 +72,7 @@ jobs:
 
   upgrade-veritech:
     needs:
-      - set-latest-versions
+      - set-service-versions
       - set-maintenance-mode
     uses: ./.github/workflows/upgrade-veritech.yml
     with:
@@ -110,6 +110,7 @@ jobs:
       - upgrade-web
       - upgrade-veritech
       - upgrade-services
+    if: always()
     uses: ./.github/workflows/e2e-validation.yml
     with:
       environment: ${{ inputs.environment }}

.github/workflows/instance-refresh.yml

@@ -17,9 +17,18 @@ on:
 jobs:
   replace:
     environment: ${{ inputs.environment }}
+    concurrency:
+      group: instance-refresh-${{ inputs.environment }}-${{ inputs.service }}
+      cancel-in-progress: true
     name: Instance refresh
     runs-on: ubuntu-latest
     steps:
+      - name: Configure AWS credentials for ${{ inputs.environment }}
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }}
+          aws-region: us-east-1
       - name: Instance refresh
         run: |
             poll_instance_refresh() {

.github/workflows/set-service-version.yml

@@ -24,7 +24,7 @@ jobs:
     environment: ${{ inputs.environment }}
     runs-on: ubuntu-latest
     steps:
-      - name: Configure AWS credentials for production
+      - name: Configure AWS credentials for ${{ inputs.environment }}
         uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}

.github/workflows/upgrade-veritech.yml

@@ -18,6 +18,7 @@ jobs:
     secrets: inherit
 
   up-veritech:
+    needs: upgrade-veritech
     uses: ./.github/workflows/up-service.yml
     with:
       environment: ${{ inputs.environment }}

component/init/configs/service.toml

@@ -1,3 +1,4 @@
+migration_mode = "skip"
 pkgs_path = "/tmp"
 create_workspace_permissions = "$SI_WORKSPACE_PERMISSIONS"
 create_workspace_allowlist = [ "$SI_WORKSPACE_ALLOW_LIST" ]

component/toolbox/awsi.sh

@@ -14,4 +14,4 @@ docker run --rm "${terminal}" \
   -e AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \
   -e AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \
   -e AWS_SESSION_TOKEN="${AWS_SESSION_TOKEN}" \
-  systeminit/toolbox:stable "$*"
+  systeminit/toolbox:20240828.153013.0-sha.44c7781c0-amd64 "$*"

component/toolbox/scripts/migrate

@@ -110,7 +110,7 @@ mkdir -p "$results_directory/"
 # get the first SDF and go do the thing
 while read -r line; do
     instance_id=$(echo "$line" | awk '{print $2}')
-    start_and_track_ssm_session "$instance_id" "$sdf_migrate_script" "sdf" "migrate" "$results_directory" &
+    start_and_track_ssm_session "$instance_id" "$sdf_migrate_script" "InstanceId=$instance_id"
     break
 done <<< "$instances"
 

component/toolbox/scripts/service-state

@@ -41,7 +41,7 @@ if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then
 fi
 
 # Parse flags
-while getopts ":p:r:a:s:" opt; do
+while getopts ":p:r:a:s:S" opt; do
     case ${opt} in
         p)
             profile=$OPTARG
@@ -120,7 +120,7 @@ i=1
 while read -r line; do
     instance_id=$(echo "$line" | awk '{print $2}')
     service_name=$(echo "$line" | awk '{print $1}' | awk -F- '{print $2}')
-    start_and_track_ssm_session "$instance_id" "$service_state_script" "$service_name" "$state" "$results_directory" # Serially
+    start_and_track_ssm_session "$instance_id" "$service_state_script" "$results_directory" "InstanceId=$instance_id,Service=$service_name,Action=$state"  # Serially
     ((i++))
 done <<< "$instances"
 

component/toolbox/scripts/ssm-scripts/si-migrate-sdf

@@ -2,19 +2,10 @@
 schemaVersion: "2.2"
 description: "Run a oneshot SDF with MigrationMode=RunAndQuit"
 parameters:
-  Service:
-    type: "String"
-    description: "Service to Run on Node"
-    default: "N/A"
   InstanceId:
     type: "String"
     description: "InstanceId of the executing node"
     default: "N/A"
-  Action:
-    type: "String"
-    description: "Action to execute [not yet used]"
-    default: "N/A"
-
 mainSteps:
 - action: "aws:runShellScript"
   name: "example"

component/toolbox/scripts/ssm-scripts/si-service-state

@@ -24,7 +24,7 @@ mainSteps:
   inputs:
     runCommand:
     - |
-        case {{ Action }} in
+        case "{{ Action }}" in
             "down")
                 service {{ Service }} stop
                 ;;
@@ -42,7 +42,7 @@ mainSteps:
                 docker-compose -f /run/app/docker-compose.yaml --profile $SI_SERVICE up --wait
 
                 wget https://artifacts.systeminit.com/{{ Service }}/${SI_VERSION}/omnibus/linux/$(arch)/{{ Service }}-${SI_VERSION}-omnibus-linux-$(arch).tar.gz -O - | tar -xzf - -C /
-                METADATA==$(sudo find / -wholename '/etc/nix-omnibus/{{ Service }}/**/metadata.json' | tail -n 1 | xargs cat | jq)
+                METADATA=$(sudo find / -wholename '/etc/nix-omnibus/{{ Service }}/**/metadata.json' | tail -n 1 | xargs cat | jq)
                 COMMIT=$(echo $METADATA | jq -r '.commit')
                 RUNNING_VERSION=$(echo $METADATA | jq -r '.version')
 

component/toolbox/scripts/supporting-funcs/ssm-funcs.sh

@@ -13,11 +13,10 @@ start_and_track_ssm_session() {
 
   instance_id=$1
   script=$2
-  service=$3
-  action=$4
-  results_directory=$5
+  results_directory=$3
+  params=$4
 
-  output=$(aws ssm send-command --instance-ids "$instance_id" --document-name "$script" --parameters "Service=$service,InstanceId=$instance_id,Action=$action" 2>&1)
+  output=$(aws ssm send-command --instance-ids "$instance_id" --document-name "$script" --parameters "$params" 2>&1)
 
   status=$?
 

component/toolbox/scripts/toggle-maintenance

@@ -1,7 +1,7 @@
 #!/bin/bash
 # ---------------------------------------------------------------------------------------------------
 # Identify all the service replicas/machines and permits you to toggle maintenance mode on or off via
-# an SSM document execution on the host. If the json output from the SSM executions is not enough to 
+# an SSM document execution on the host. If the json output from the SSM executions is not enough to
 # debug just look in AWS and you'll see the whole execution history in SSM Command Execution History.
 # ---------------------------------------------------------------------------------------------------
 
@@ -10,7 +10,7 @@
 set -eo pipefail
 
 # Find & Import all the supporting functions from the supporting folder
-# Get the directory of the current script to figure out where the 
+# Get the directory of the current script to figure out where the
 # Supporting funcs are
 IMPORT_DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)
 
@@ -126,6 +126,7 @@ while read -r line; do
     instance_id=$(echo "$line" | awk '{print $2}')
     service_name=$(echo "$line" | awk '{print $1}' | awk -F- '{print $2}')
     start_and_track_ssm_session "$instance_id" "$service_maintenance_script" "$service_name" "$maintenance" "$results_directory" &
+    start_and_track_ssm_session "$instance_id" "$service_maintenance_script" "$results_directory" "InstanceId=$instance_id,Action=$maintenance" &
     ((i++))
 done <<< "$instances"
 

component/toolbox/scripts/upgrade

@@ -51,7 +51,7 @@ if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then
 fi
 
 # Parse flags
-while getopts ":p:r:a:s:" opt; do
+while getopts ":p:r:a:s:e" opt; do
     case ${opt} in
         p)
             profile=$OPTARG
@@ -130,7 +130,7 @@ i=1
 while read -r line; do
     instance_id=$(echo "$line" | awk '{print $2}')
     service_name=$(echo "$line" | awk '{print $1}' | awk -F- '{print $2}')
-    start_and_track_ssm_session "$instance_id" "$upgrade_check_script" "$service_name" "check" "$results_directory" &
+    start_and_track_ssm_session "$instance_id" "$upgrade_check_script" "$results_directory" "InstanceId=$instance_id,Service=$service_name" &
     ((i++))
 done <<< "$instances"
 
@@ -173,7 +173,7 @@ upgrade_hosts_num=$(jq 'map(select(.service != "veritech")) | .[]' <<< $upgrade_
 jq 'map(select(.service != "veritech")) | .[]' <<< $upgrade_candidates_json | jq -c '.' | while read -r line; do
   instance_id=$(echo "$line" | jq -r '.instance_id')
   service_name=$(echo "$line" | jq -r '.service')
-  start_and_track_ssm_session "$instance_id" "$service_state_script" "$service_name" "upgrade" "$results_directory" -e "$environment" & # In Parallel
+  start_and_track_ssm_session "$instance_id" "$service_state_script" "$results_directory" "InstanceId=$instance_id,Service=$service_name,Action=upgrade,Environment=$environment" &
   ((i++))
 done