Merge pull request #11 from lightning-js/feature/backstopjs

Backstopjs Visual Regression Tests

.gitignore

@@ -3,3 +3,5 @@ node_modules
 build
 dist
 .env
+tests/backstopjs/bitmaps_test/
+tests/backstopjs/html_report/

README.md

@@ -10,6 +10,74 @@ This Blits L3 Example App contains a demo of a TMBD integration. Head over to ht
 
 You can also take a look at the [hosted version](http://blits-demo.lightningjs.io) of this App.
 
+## Visual Regression Test
+
+
+[BackstopJS](https://github.com/garris/BackstopJS) is a tool that facilitates automated visual regression testing for responsive web user interfaces.
+It accomplishes this by comparing DOM screenshots over time.
+
+### Setup
+
+To get started with BackstopJS, follow these steps:
+
+1. Run the App via the command: `npm run start:test` or `NODE_ENV=testing npm run dev`
+2. Run `npm test` to test the App against the reference bitmaps.
+
+
+> Note: To ensure consistency regarding the performance of the app while creating the reference bitmaps or running the tests, `asyncCaptureLimit` should be set to `1` in the `backstop.cjs` file. This will ensure that the tests are run sequentially and not in parallel (so concurrency won't affect the performance of the app).
+
+> Note: The default host and port of the dev server is `http://localhost:5173`. If you are running the app on a different host or port, you should update `TEST_HOST` and `TEST_PORT` in the `.env` file. Or you can directly modify the `backstop.cjs` file.
+
+### Creating Reference Bitmaps
+
+To create reference bitmaps for visual regression testing, use the following command in the terminal:
+
+```bash
+npm run test:reference
+```
+
+This command will remove any existing reference snapshots and generate new ones based on the provided configuration `URL` in the previous section.
+
+
+### Creating Test Bitmaps
+
+Generate test bitmaps by using the following command in the terminal:
+
+```bash
+$ npm test
+```
+
+This command will produce a new set of bitmaps in the `bitmap_test/<timestamp>` folder. After bitmap generation is complete,
+a report comparing the most recent bitmaps against the reference will be displayed.
+
+
+### Browser Options
+
+BackstopJS supports both Puppeteer and Playwright. The existing reference bitmaps were created using Puppeteer with its default browser (Chrome headless). Changing browser settings may change page load and animation timings so after a change reference bitmaps might not match with test bitmaps. In that case, reference bitmaps should be generated again.
+
+
+If you want to modify browser configuration, use the following settings in your `backstop.cjs` file:
+
+To use webkit:
+
+
+```json
+"engine": "playwright",
+"engineOptions": {
+    "args": ["--no-sandbox"],
+    "browser": "webkit"
+},
+
+```
+
+### Debugging
+
+In case you prefer to visually monitor your application's state during the tests, you can enable the debug window using the following setting:
+
+```json
+"debugWindow": true
+```
+
 ## Bugs or issues?
 
 If you find any bugs or issues, please feel free to file a GitHub issue or open a PR.

backstop.cjs

@@ -0,0 +1,199 @@
+const host = process.env.TEST_HOST || 'localhost'
+const port = process.env.TEST_PORT || '5173'
+
+module.exports = {
+  id: 'blits_example_app',
+  viewports: [
+    {
+      label: 'tv',
+      width: 1920,
+      height: 1080,
+    },
+  ],
+  scenarios: [
+    {
+      label: 'Blits - Main Page',
+      url: `http://${host}:${port}/#/`,
+      delay: 4000,
+    },
+    {
+      label: 'Blits - Loading',
+      url: `http://${host}:${port}/#/demos/loading`,
+      readyEvent: 'backstopjs:ready',
+    },
+    {
+      label: 'Blits - Intro',
+      url: `http://${host}:${port}/#/demos/intro`,
+      delay: 8000,
+    },
+    {
+      label: 'Blits - Theming',
+      url: `http://${host}:${port}/#/demos/theming`,
+      delay: 1000,
+    },
+    {
+      label: 'Blits - Theming - Arrow Right Keypress',
+      url: `http://${host}:${port}/#/demos/theming`,
+      onReadyScript: 'theming-arrow-right-keypress.cjs',
+    },
+    {
+      label: 'Blits - Theming - Arrow Up Keypress',
+      url: `http://${host}:${port}/#/demos/theming`,
+      onReadyScript: 'theming-arrow-up-keypress.cjs',
+    },
+    {
+      label: 'Blits - Tmdb',
+      url: `http://${host}:${port}/#/demos/tmdb`,
+      delay: 1000,
+    },
+    {
+      label: 'Blits - Sprites',
+      url: `http://${host}:${port}/#/demos/sprites`,
+      delay: 1000,
+    },
+    {
+      label: 'Blits - Focushandling',
+      url: `http://${host}:${port}/#/demos/focushandling`,
+      delay: 1000,
+    },
+    {
+      label: 'Blits - Focushandling - Arrow Right Keypress x 3',
+      url: `http://${host}:${port}/#/demos/focushandling`,
+      onReadyScript: 'focushandling-arrow-right-keypress.cjs',
+    },
+    {
+      label: 'Blits - Focushandling - Enter Keypress',
+      url: `http://${host}:${port}/#/demos/focushandling`,
+      onReadyScript: 'focushandling-enter-keypress.cjs',
+    },
+    {
+      label: 'Blits - Memory Game',
+      url: `http://${host}:${port}/#/demos/memory-game`,
+      delay: 1000,
+    },
+    {
+      label: 'Blits - Memory Game - Arrow Right Keypress x 2',
+      url: `http://${host}:${port}/#/demos/memory-game`,
+      onReadyScript: 'memory-game-arrow-right-keypress.cjs',
+    },
+    {
+      label: 'Blits - Positioning',
+      url: `http://${host}:${port}/#/examples/positioning`,
+      readyEvent: 'backstopjs:ready',
+    },
+    {
+      label: 'Blits - Colors',
+      url: `http://${host}:${port}/#/examples/colors`,
+      readyEvent: 'backstopjs:ready',
+    },
+    {
+      label: 'Blits - Colors - Enter Keypress',
+      url: `http://${host}:${port}/#/examples/colors`,
+      onReadyScript: 'colors-enter-keypress.cjs',
+    },
+    {
+      label: 'Blits - Gradients',
+      url: `http://${host}:${port}/#/examples/gradients`,
+      delay: 500,
+    },
+    {
+      label: 'Blits - Transitions',
+      url: `http://${host}:${port}/#/examples/transitions`,
+      delay: 5000,
+    },
+    {
+      label: 'Blits - Components',
+      url: `http://${host}:${port}/#/examples/components`,
+      delay: 1200,
+    },
+    {
+      label: 'Blits - Alpha',
+      url: `http://${host}:${port}/#/examples/alpha`,
+      readyEvent: 'backstopjs:ready',
+      delay: 1000,
+    },
+    {
+      label: 'Blits - Scaling',
+      url: `http://${host}:${port}/#/examples/scaling`,
+      delay: 3000,
+    },
+    {
+      label: 'Blits - Rotation',
+      url: `http://${host}:${port}/#/examples/rotation`,
+      delay: 2500,
+    },
+    {
+      label: 'Blits - Keyinput',
+      url: `http://${host}:${port}/#/examples/keyinput`,
+      delay: 750,
+    },
+    {
+      label: 'Blits - Keyinput - Arrow Right + Down Keypress',
+      url: `http://${host}:${port}/#/examples/keyinput`,
+      onReadyScript: 'keyinput-arrow-right-down-keypress.cjs',
+    },
+    {
+      label: 'Blits - Keyinput - Enter Keypress',
+      url: `http://${host}:${port}/#/examples/keyinput`,
+      onReadyScript: 'keyinput-enter-keypress.cjs',
+    },
+    {
+      label: 'Blits - Forloop',
+      url: `http://${host}:${port}/#/examples/forloop`,
+      delay: 6000,
+    },
+    {
+      label: 'Blits - Effects',
+      url: `http://${host}:${port}/#/examples/effects`,
+      delay: 500,
+    },
+    {
+      label: 'Blits - Showif',
+      url: `http://${host}:${port}/#/examples/showif`,
+      readyEvent: 'backstopjs:ready',
+    },
+    {
+      label: 'Blits - Events',
+      url: `http://${host}:${port}/#/examples/events`,
+      delay: 500,
+    },
+    {
+      label: 'Blits - Events - Arrow Right Keypress x 2',
+      url: `http://${host}:${port}/#/examples/events`,
+      onReadyScript: 'events-arrow-right-keypress.cjs',
+    },
+    {
+      label: 'Blits - Images',
+      url: `http://${host}:${port}/#/examples/images`,
+      readyEvent: 'backstopjs:ready',
+    },
+    {
+      label: 'Blits - Texts',
+      url: `http://${host}:${port}/#/examples/texts`,
+      readyEvent: 'backstopjs:ready',
+    },
+    {
+      label: 'Blits - Slots',
+      url: `http://${host}:${port}/#/examples/slots`,
+      readyEvent: 'backstopjs:ready',
+    },
+  ],
+  paths: {
+    bitmaps_reference: 'tests/backstopjs/bitmaps_reference',
+    bitmaps_test: 'tests/backstopjs/bitmaps_test',
+    engine_scripts: 'tests/backstopjs/engine_scripts',
+    html_report: 'tests/backstopjs/html_report',
+    ci_report: 'tests/backstopjs/ci_report',
+  },
+  report: ['browser'],
+  engine: 'puppeteer',
+  engineOptions: {
+    args: ['--no-sandbox', '--disable-setuid-sandbox'],
+  },
+  headless: true,
+  asyncCaptureLimit: 1,
+  asyncCompareLimit: 50,
+  debug: false,
+  debugWindow: false,
+  misMatchThreshold: 0.1,
+}