fix: e2e tests weren't failing properly, amends it

systeminit · Oct 2, 2024 · 1d367b5 · 1d367b5
1 parent 7201bf7
commit 1d367b5
Show file tree

Hide file tree

Showing 2 changed files with 68 additions and 34 deletions.
diff --git a/.github/workflows/e2e-validation.yml b/.github/workflows/e2e-validation.yml
@@ -71,6 +71,7 @@ jobs:
           sudo apt install uuid -y
 
       - name: Run Cypress Tests
+        continue-on-error: true
         run: |
           cd app/web
           export VITE_AUTH0_USERNAME="${{ secrets.VITE_AUTH0_USERNAME }}"
@@ -83,8 +84,33 @@ jobs:
           export VITE_UUID
           export VITE_AUTH_API_URL="https://auth-api.systeminit.com"
           export VITE_AUTH_PORTAL_URL="https://auth.systeminit.com"
-          npx cypress run --spec "cypress/e2e/${{ matrix.tests }}/**"
-        continue-on-error: true
+
+          # Retry loop with 3 attempts
+          n=0
+          max_retries=3
+          
+          until [ $n -ge $max_retries ]
+          do
+
+            unset exit_code || echo "exit_code not set"
+
+            # Run the npx task and store exit code in a variable
+            npx cypress run --spec "cypress/e2e/${{ matrix.tests }}/**" || exit_code=$?
+            
+            # Check the exit code
+            if [ -z "$exit_code" ]; then
+              echo "Cypress Test task succeeded!"
+              break
+            fi
+
+            n=$((n+1))
+            echo "Attempt $n/$max_retries failed with exit code $exit_code! Retrying..."
+          done
+
+          if [ $n -ge $max_retries ]; then
+            echo "All $max_retries attempts failed."
+            exit 1
+          fi
 
       - name: 'Upload Cypress Recordings to Github'
         uses: actions/upload-artifact@v4
@@ -98,18 +124,10 @@ jobs:
         if: failure()
         run: exit 1
 
-  retry:
-    runs-on: ubuntu-latest
-    needs: cypress-tests
-    if: failure() && github.run_attempt < 3
-    steps:
-      - name: Retry Cypress Tests
-        run: echo "Retry attempt ${{ github.run_attempt }}/3"
-
   on-failure:
     runs-on: ubuntu-latest
     needs: cypress-tests
-    if: ${{ failure() && github.ref == 'refs/heads/main' && github.run_attempt == 3 }}
+    if: ${{ failure() && github.ref == 'refs/heads/main' }}
     steps:
       - run: |
           curl --location "${{ secrets.FIREHYDRANT_WEBHOOK_URL }}" \
@@ -130,6 +148,6 @@ jobs:
 
       - run: |
           curl -X POST \
-            -H 'Content-type: application/json' \
-            --data "{\"text\": \":si: Failed Cypress E2E Test for ${{ inputs.environment }}: <https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID|:test_tube: Link>\"}" \
-            ${{ secrets.SLACK_WEBHOOK_URL }}
+          --header 'Content-type: application/json' \
+          --data "{\"text\": \":si: Failed Cypress E2E Test for ${{ inputs.environment }}: <https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID|:test_tube: Link>\"}" \
+          ${{ secrets.SLACK_WEBHOOK_URL }}
diff --git a/.github/workflows/run-api-test.yml b/.github/workflows/run-api-test.yml
@@ -46,7 +46,6 @@ jobs:
       fail-fast: false
       matrix:
         tests: ${{ fromJSON(needs.define-test-matrix.outputs.tests) }}
-
     env:
       SDF_API_URL: ${{ vars.SDF_API_URL }}
       AUTH_API_URL: ${{ vars.AUTH_API_URL }}
@@ -59,28 +58,45 @@ jobs:
         with:
           deno-version: v1.x
 
-      - name: Run the deno exec
+      - name: Run the deno exec with retry
+        continue-on-error: true
         run: |
           cd bin/si-api-test
-          echo ${{ github.ref }}
-          deno task run \
-          --workspaceId ${{ vars.API_TEST_WORKSPACE_ID }} \
-          --userId ${{ secrets.API_TEST_EMAIL }} \
-          --password ${{ secrets.API_TEST_PASSWORD }} \
-          --tests ${{ matrix.tests }}
+          
+          # Retry loop with 3 attempts
+          n=0
+          max_retries=3
+          
+          until [ $n -ge $max_retries ]
+          do
+            unset exit_code || echo "exit_code not set"
+
+            # Run the deno task and store exit code in a variable
+            deno task run \
+            --workspaceId ${{ vars.API_TEST_WORKSPACE_ID }} \
+            --userId ${{ secrets.API_TEST_EMAIL }} \
+            --password ${{ secrets.API_TEST_PASSWORD }} \
+            --tests ${{ matrix.tests }} || exit_code=$?
+            
+            # Check the exit code
+            if [ -z "$exit_code" ]; then
+              echo "Deno task succeeded!"
+              break
+            fi
+
+            n=$((n+1))
+            echo "Attempt $n/$max_retries failed with exit code $exit_code! Retrying..."
+          done
+
+          if [ $n -ge $max_retries ]; then
+            echo "All $max_retries attempts failed."
+            exit 1
+          fi
 
-  retry:
-    runs-on: ubuntu-latest
-    needs: api-test
-    if: failure() && github.run_attempt < 3
-    steps:
-      - name: Retry API Tests
-        run: echo "Retry attempt ${{ github.run_attempt }}/3"
-
   on-failure:
     runs-on: ubuntu-latest
     needs: api-test
-    if: ${{ failure() && github.ref == 'refs/heads/main' && github.run_attempt == 3 }}
+    if: ${{ failure() && github.ref == 'refs/heads/main' }}
     steps:
       - run: |
             curl --location "${{ secrets.FIREHYDRANT_WEBHOOK_URL }}" \
@@ -100,6 +116,6 @@ jobs:
             }"
       - run: |
           curl -X POST \
-            -H 'Content-type: application/json' \
-            --data "{\"text\": \":si: Failed API Tests for ${{ inputs.environment }}: <https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID|:test_tube: Link>\"}" \
-            ${{ secrets.SLACK_WEBHOOK_URL }}
+          --header 'Content-type: application/json' \
+          --data "{\"text\": \":si: Failed API Tests for ${{ inputs.environment }}: <https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID|:test_tube: Link>\"}" \
+          ${{ secrets.SLACK_WEBHOOK_URL }}