test: trying a different approach

.github/workflows/run-api-test.yml

@@ -79,11 +79,6 @@ jobs:
         with:
           deno-version: v1.x
 
-      - name: Push failure of exit 53 to test code randomly
-        run: |
-          [[ "$(( ( RANDOM % 5 )  + 1 ))" == "3" ]] && exit 53
-          exit 0
-
       - name: Run the deno exec with retry
         run: |
           cd bin/si-api-test
@@ -117,7 +112,14 @@ jobs:
             
             # Check the exit code
             if [ -z "$exit_code" ]; then
-              echo "Deno task succeeded!"
+              echo "Deno task succeeded [ or this failed for a totally non-valid reason ]!"
+
+              # Randomly make it fail to test logic
+              if [[ "$(( ( RANDOM % 5 )  + 1 ))" == "3" ]]; then
+                echo "last_exit_code=$exit_code" >> "$GITHUB_OUTPUT"
+                exit 53
+              fi 
+
               break
             fi
 
@@ -128,29 +130,48 @@ jobs:
 
           if [ $n -ge $max_retries ]; then
             echo "All $max_retries attempts failed."
-            exit_code=0
+            exit_code=1
           fi
 
           echo "last_exit_code=$exit_code" >> "$GITHUB_OUTPUT"
           exit $last_exit_code
 
+      - name: Upload artifact if exit code 53
+        if: ${{ failure() && env.last_exit_code == '53' }}
+        run: |
+          echo "Uploading marker for test ${{ matrix.tests.name }}"
+          mkdir -p artifacts/${{ matrix.tests.name }}
+          echo "failure-marker" > artifacts/${{ matrix.tests.name }}/failure-marker
+
+      - name: Upload artifacts
+        if: ${{ failure() && env.last_exit_code == '53' }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: debug-${{ matrix.tests.name }}
+          path: artifacts/${{ matrix.tests.name }}
+
   on-failure:
     runs-on: ubuntu-latest
     needs: api-test
     environment: ${{ inputs.environment }}
     if: ${{ failure() }} #&& github.ref == 'refs/heads/main' }} #TODO(johnrwatson): disable comment before merging
     steps:
-      - run: |
-          VALID_FAILURE_EXIT_CODES="53"
-          echo "consolidated:"
-          echo "${{ needs.api-test.outputs.exit_code }}"
-          echo "alt:"
-          echo "${{ needs.api-test.outputs.exit_code }} | jq"
-          exit_codes='[${{ needs.api-test.outputs.exit_code}}]  -r '.[]''
-          # Check if any exit code is "bad"
-          for code in $exit_codes; do
-            if echo "$VALID_FAILURE_EXIT_CODES" | grep -qw "$code"; then
-              echo "Bad exit code detected: $code"
+      - name: Download all artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: artifacts
+      - run: |     
+          has_artifacts=false
+          # Check for marker files
+          for marker in artifacts/*/failure-marker; do
+            if [ -f "$marker" ]; then
+              echo "Artifact detected for a failed test: $marker"
+              has_artifacts=true
+              break
+            fi
+          done
+          # If valid failure markers are present page
+          if [ "$has_artifacts" = true ]; then
               curl --location "${{ secrets.FIREHYDRANT_WEBHOOK_URL }}" \
               --header "Content-Type: application/json" \
               --data "{
@@ -166,9 +187,9 @@ jobs:
                   \"service:github\"
                 ]
               }" 
-            fi
-          done
+          fi
       - run: |
+          # Always send the Internal Slack Notification if failure detected, even if we didn't page
           curl --location "${{ secrets.SLACK_WEBHOOK_URL }}" -X POST \
             --header 'Content-type: application/json' \
             --data "{\"text\": \":si: Failed API Tests for ${{ inputs.environment }}: <https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID|:test_tube: Link>\"}"