Adding system tests (#1086)

* feat: add first test

* feat: change to pageObject

* feat: rewrite on playwright

* feat: rewrite on playwright

* feat: add more cases

* feat: matrix login test

* fix: dockercompose files

* chore: fix package.json

* chore: fix package.json

* chore: resolve conflicts

* fix: linter problems

* fix: add info and remove redundant browsers

* fix: increase matrix timeout

* feat: add matrix login in multisig creation flow

* fix: improve tests for matrix auth

* fix: comments

* fix: update to current dev

* chore: clean up code

fix: readme

* feat: adding publication

* fix: increase timeout

* feat: save many reports

* fix: url

* fix: set only for main branch

.github/workflows/system_tests.yml

@@ -0,0 +1,81 @@
+name: Run system tests
+
+on:
+  pull_request:
+    branches:
+      - 'main'
+
+env:
+  CI: true
+
+jobs:
+  system-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build compose
+        run: docker-compose up -d
+      - name: ⚙️ Install dependencies
+        uses: ./.github/workflows/install-pnpm
+      - name: Install Playwright Browsers
+        run: npx playwright install --with-deps
+      - name: Run Playwright tests
+        run: npx playwright test
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30
+
+  publish_report:
+    name: Publish HTML Report
+    if: success() || needs.system-tests.result == 'failure'
+    needs: [system-tests]
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    env:
+      # Unique URL path for each workflow run attempt
+      HTML_REPORT_URL_PATH: reports/${{ github.ref_name }}/${{ github.run_id }}/${{ github.run_attempt }}
+    steps:
+      - name: Checkout GitHub Pages Branch
+        uses: actions/checkout@v2
+        with:
+          repository: novasamatech/spektr-reports
+          ref: main
+          token: ${{ secrets.CREATE_TAG_PAT }}
+      - name: Set Git User
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+      - name: Download zipped HTML report
+        uses: actions/download-artifact@v2
+        with:
+          name: playwright-report
+          path: ${{ env.HTML_REPORT_URL_PATH }}
+      - name: Push HTML Report
+        timeout-minutes: 3
+        # commit report, then try push-rebase-loop until it's able to merge the HTML report to the gh-pages branch
+        # this is necessary when this job is running at least twice at the same time (e.g. through two pushes at the same time)
+        run: |
+          git add .
+          git commit -m "workflow: add HTML report for run-id ${{ github.run_id }} (attempt:  ${{ github.run_attempt }})"
+  
+          while true; do
+            git pull --rebase
+            if [ $? -ne 0 ]; then
+              echo "Failed to rebase. Please review manually."
+              exit 1
+            fi
+
+            git push
+            if [ $? -eq 0 ]; then
+              echo "Successfully pushed HTML report to repo."
+              exit 0
+            fi
+          done
+      - name: Output Report URL as Worfklow Annotation
+        run: |
+          FULL_HTML_REPORT_URL=https://novasamatech.github.io/spektr-reports/$HTML_REPORT_URL_PATH
+  
+          echo "::notice title=📋 Published Playwright Test Report::$FULL_HTML_REPORT_URL"

.gitignore

@@ -31,3 +31,6 @@ yarn-error.log*
 schema.graphql
 
 *-results.json
+/test-results/
+/playwright-report/
+/playwright/.cache/

docker-compose.yml

@@ -0,0 +1,9 @@
+version: '3.8'
+
+services:
+  web-app:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile
+    ports:
+      - '3000:3000'

docker/Dockerfile

@@ -0,0 +1,13 @@
+FROM node:16
+
+WORKDIR /app
+
+COPY package*.json ./
+
+RUN npm install -g pnpm
+RUN pnpm install
+RUN pnpm install sucrase @babel/register esbuild-register @swc/register
+
+COPY . .
+
+CMD [ "pnpm", "start:renderer" ]

docker/Dockerfile.tests

@@ -0,0 +1,13 @@
+FROM mcr.microsoft.com/playwright:v1.38.0-jammy
+
+RUN mkdir -p /tests
+WORKDIR /tests
+
+COPY package*.json ./
+
+RUN npm install -g pnpm
+RUN pnpm install
+
+COPY . .
+
+CMD [ "npx", "playwright", "test" ]

package.json

@@ -46,6 +46,11 @@
     "test:coverage": "jest --config=jest.config.ts --coverage --passWithNoTests | tee ./coverage.txt",
     "test:coverage-new-files": "jest --config=jest.config.pr.ts --coverage --changedSince=dev ; exitCode=$? ; echo $exitCode | tee ./coverage.txt ; exit $exitCode",
 
+    "pretest:system": "npx playwright install",
+    "test:system": "npx playwright test -j 4",
+    "test:system-generator": "npx playwright codegen --ignore-https-errors https://localhost:3000",
+    "test:system:ui-mod": "npx playwright test --ui",
+
     "lint": "eslint . --ext=js,ts,tsx,json",
     "lint:fix": "pnpm lint --fix",
     "lint:i18n-locale": "cross-env I18N=true eslint src/renderer/shared/api/translation/locales/ --ext=json --plugin i18n-json",
@@ -129,6 +134,8 @@
     "@commitlint/config-conventional": "^17.0.3",
     "@electron/notarize": "^1.2.3",
     "@headlessui/tailwindcss": "^0.1.2",
+    "@jest/types": "^29.6.3",
+    "@playwright/test": "^1.38.0",
     "@pmmmwh/react-refresh-webpack-plugin": "^0.5.7",
     "@storybook/addon-actions": "^6.5.9",
     "@storybook/addon-essentials": "^6.5.9",
@@ -196,6 +203,7 @@
     "mini-css-extract-plugin": "^2.6.1",
     "nodemon": "^2.0.18",
     "npm-run-all": "^4.1.5",
+    "playwright": "^1.38.0",
     "pnpm": "^8.6.11",
     "postcss": "^8.4.31",
     "postcss-loader": "^7.3.3",

playwright.config.ts

@@ -0,0 +1,34 @@
+import { defineConfig, devices } from '@playwright/test';
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './tests/system',
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: Boolean(process.env.CI),
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    baseURL: 'https://127.0.0.1:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+  ],
+});