Run E2E tests on production deployments (#1253)

* Temp run on test branches

* Run e2e on prod deployments

* temp remove production deployment workflow trigger

* add fixtures cache key

* fetch prod fixtures

* remove local frontend starting

* set working directory

* load fixtures cache

* remove temp testing logic

.github/workflows/deploy.yml

@@ -7,12 +7,15 @@ on:
       - main
     tags:
       - 'v*'
-
+ 
 # https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-amazon-web-services
 permissions:
   id-token: write
   contents: read
 
+env:
+  FIXTURES_CACHE_KEY: e2e-fixtures-${{ github.run_id }}-${{ github.run_attempt }}
+
 jobs:
   prod:
     name: deploy to production
@@ -91,3 +94,145 @@ jobs:
         with:
           name: lighthouse-reports
           path: ./.lighthouseci/
+
+  prod-e2e-setup:
+    name: Production E2E Tests Setup
+    defaults:
+      run:
+        working-directory: frontend/
+    runs-on: ubuntu-latest
+    needs: prod # Needs the production deploy to happen first before E2E tests can be ran
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20.5.x
+
+      - name: Get Yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
+
+      # cache both yarn-lock and node_modules
+      - name: Setup Yarn cache
+        uses: actions/cache@v3
+        id: yarn-cache
+        with:
+          path: |
+            **/node_modules
+            ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      # --prefer-offline forces yarn to use cache if available
+      - name: Install dependencies with yarn
+        # if: steps.yarn-cache.outputs.cache-hit != 'true'
+        run: yarn install --prefer-offline --frozen-lockfile
+
+      - name: Cache playwright binaries
+        uses: actions/cache@v3
+        id: playwright-cache
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-${{ hashFiles('**/yarn.lock') }}
+
+      - name: Install playwright
+        # if: steps.playwright-cache.outputs.cache-hit != 'true'
+        run: npx playwright install --with-deps
+
+      - name: Setting up local build cache
+        uses: actions/cache@v3
+        with:
+          path: frontend/.next
+          key: local-build-${{ hashFiles('**/next.config.js') }}-${{ hashFiles('frontend/src') }}-${{ hashFiles('**/yarn.lock') }}
+
+      - name: Setting up fixture cache
+        uses: actions/cache@v3
+        with:
+          path: frontend/e2e/fixtures
+          key: ${{ env.FIXTURES_CACHE_KEY }}
+
+      - name: Fetch fixture data
+        env:
+          ENV: prod
+        run: node scripts/fetch-e2e-fixtures
+
+  prod-e2e:
+    name: E2E tests ${{ matrix.browser }} ${{ matrix.shardCurrent }} / ${{ matrix.shardTotal }}
+    defaults:
+      run:
+        working-directory: frontend/
+    runs-on: ubuntu-latest
+    needs: prod-e2e-setup
+
+    concurrency:
+      group: ${{ github.ref }}-${{ matrix.browser }}-${{ matrix.shardCurrent }}-${{ matrix.shardTotal }}
+      cancel-in-progress: true
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # browser: [chrome, firefox, safari]
+        browser: [chrome]
+        shardCurrent: [1, 2, 3, 4, 5]
+        shardTotal: [5]
+
+    steps:
+      - name: 25 minutes wait # Wait 25 mins before running E2E tests to let deployment changes on AWS propigate
+        run: sleep 1500 # Sleep time in seconds (25 minutes * 60 seconds = 1500 seconds)
+
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20.5.x
+
+      - name: Get Yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
+
+      # cache both yarn-lock and node_modules
+      - name: Load yarn cache
+        uses: actions/cache@v3
+        id: yarn-cache
+        with:
+          path: |
+            **/node_modules
+            ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Load cached playwright binaries
+        uses: actions/cache@v3
+        id: playwright-cache
+        with:
+          path: |
+            ~/.cache/ms-playwright
+          key: playwright-${{ hashFiles('**/yarn.lock') }}
+
+      - name: Load fixtures cache
+        uses: actions/cache@v3
+        with:
+          path: frontend/e2e/fixtures
+          key: ${{ env.FIXTURES_CACHE_KEY }}
+
+      - name: Run E2E Tests
+        env:
+          BROWSER: ${{ matrix.browser }}
+          ENV: prod
+        run: yarn e2e:ci --shard ${{ matrix.shardCurrent }}/${{ matrix.shardTotal }}
+
+      - name: Upload artifacts
+        if: ${{ failure() }}
+        uses: actions/[email protected]
+        with:
+          name: e2e-artifacts
+          path: frontend/e2e/report
+          retention-days: 1