From 3d47d403bc47fabcf6358ee3ab5c4521ed9fc513 Mon Sep 17 00:00:00 2001 From: DudaGod Date: Mon, 5 Aug 2024 09:43:12 +0300 Subject: [PATCH] feat: implement new cli command to read tests --- README.md | 2 +- docs/cli.md | 244 ------------ docs/component-testing.md | 139 ------- docs/events/index.md | 2 +- docs/programmatic-api.md | 5 + docs/writing-tests.md | 1 + package-lock.json | 18 +- package.json | 8 +- src/bundle/test-transformer.ts | 1 + src/cli/commands/list-tests/index.ts | 65 ++++ src/cli/constants.ts | 3 + src/cli/index.ts | 44 +-- src/index.ts | 9 +- src/test-collection/constants.ts | 6 + src/test-collection/formatters/list.ts | 33 ++ src/test-collection/formatters/tree.ts | 126 +++++++ .../index.ts} | 64 +++- src/test-reader/index.ts | 4 +- src/test-reader/mocha-reader/index.js | 103 ++++- .../mocha-reader/tree-builder-decorator.js | 4 +- src/test-reader/mocha-reader/utils.js | 12 + .../test-object/configurable-test-object.ts | 10 +- src/test-reader/test-object/suite.ts | 6 +- src/test-reader/test-object/test.ts | 7 +- src/test-reader/test-object/types.ts | 6 + src/test-reader/test-parser.ts | 11 +- src/testplane.ts | 9 +- src/types/helpers.ts | 1 + src/utils/cli.ts | 45 +++ src/worker/runner/simple-test-parser.ts | 2 +- test/src/cli/commands/list-tests/index.ts | 171 +++++++++ test/src/cli/index.js | 15 +- test/src/test-collection/formatters/list.ts | 97 +++++ test/src/test-collection/formatters/tree.ts | 351 ++++++++++++++++++ .../index.ts} | 61 ++- test/src/test-reader/index.js | 5 +- test/src/test-reader/mocha-reader/index.js | 149 ++++++++ test/src/test-reader/test-parser.js | 14 +- test/src/test-reader/test-transformer.ts | 4 +- test/src/testplane.js | 6 + .../browser-env/runner/test-runner/index.ts | 1 + test/src/worker/runner/simple-test-parser.js | 2 +- 42 files changed, 1400 insertions(+), 466 deletions(-) delete mode 100644 docs/cli.md delete mode 100644 docs/component-testing.md create mode 100644 src/cli/commands/list-tests/index.ts create mode 100644 src/cli/constants.ts create mode 100644 src/test-collection/constants.ts create mode 100644 src/test-collection/formatters/list.ts create mode 100644 src/test-collection/formatters/tree.ts rename src/{test-collection.ts => test-collection/index.ts} (76%) create mode 100644 src/types/helpers.ts create mode 100644 src/utils/cli.ts create mode 100644 test/src/cli/commands/list-tests/index.ts create mode 100644 test/src/test-collection/formatters/list.ts create mode 100644 test/src/test-collection/formatters/tree.ts rename test/src/{test-collection.ts => test-collection/index.ts} (89%) diff --git a/README.md b/README.md index 2c162929a..8c97711ad 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ Feel free to visit these pages for a brief overview of some of Testplane feature - [Browser Commands Reference](docs/commands.md) - [Testplane Config Reference](https://testplane.io/docs/v8/config/main/) - [Testplane × Typescript](docs/typescript.md) -- [Testplane CLI](docs/cli.md) +- [Testplane CLI](https://testplane.io/docs/v8/command-line/) - [Testplane Events](docs/events.md) - [Testplane Programmatic API](docs/programmatic-api.md) - [Testplane Component Testing (experimental)](docs/component-testing.md) diff --git a/docs/cli.md b/docs/cli.md deleted file mode 100644 index 19d176865..000000000 --- a/docs/cli.md +++ /dev/null @@ -1,244 +0,0 @@ -## Testplane CLI - - - -### Contents - -- [Overview](#overview) -- [Reporters](#reporters) -- [Require modules](#require-modules) -- [Overriding settings](#overriding-settings) -- [Debug mode](#debug-mode) -- [REPL mode](#repl-mode) - - [switchToRepl](#switchtorepl) - - [Test development in runtime](#test-development-in-runtime) - - [How to set up using VSCode](#how-to-set-up-using-vscode) - - [How to set up using Webstorm](#how-to-set-up-using-webstorm) -- [Environment variables](#environment-variables) - - [TESTPLANE_SKIP_BROWSERS](#testplane_skip_browsers) - - [TESTPLANE_SETS](#testplane_sets) - - - -### Overview - -``` -testplane --help -``` - -shows the following - -``` - Usage: testplane [options] [paths...] - - Options: - - -V, --version output the version number - -c, --config path to configuration file - -b, --browser run tests only in specified browser - -s, --set run tests only in the specified set - -r, --require require a module before running `testplane` - --reporter test reporters - --grep run only tests matching the pattern - --update-refs update screenshot references or gather them if they do not exist ("assertView" command) - --inspect [inspect] nodejs inspector on [=[host:]port] - --inspect-brk [inspect-brk] nodejs inspector with break at the start - --repl [type] run one test, call `browser.switchToRepl` in test code to open repl interface (default: false) - --repl-before-test [type] open repl interface before test run (default: false) - --repl-on-fail [type] open repl interface on test fail only (default: false) - -h, --help output usage information -``` - -For example, -``` -testplane --config ./config.js --reporter flat --browser firefox --grep name -``` - -**Note.** All CLI options override config values. - -### Reporters - -You can choose `flat`, `plain` or `jsonl` reporter by option `--reporter`. Default is `flat`. -Information about test results is displayed to the command line by default. But there is an ability to redirect the output to a file, for example: -``` -testplane --reporter '{"type": "jsonl", "path": "./some-path/result.jsonl"}' -``` - -In that example specified file path and all directories will be created automatically. Moreover you can use few reporters: -``` -testplane --reporter '{"type": "jsonl", "path": "./some-path/result.jsonl"}' --reporter flat -``` - -Information about each report type: -* `flat` – all information about failed and retried tests would be grouped by browsers at the end of the report; -* `plain` – information about fails and retries would be placed after each test; -* `jsonl` - displays detailed information about each test result in [jsonl](https://jsonlines.org/) format. - -### Require modules - -Using `-r` or `--require` option you can load external modules, which exists in your local machine, before running `testplane`. This is useful for: - -- compilers such as TypeScript via [ts-node](https://www.npmjs.com/package/ts-node) (using `--require ts-node/register`) or Babel via [@babel/register](https://www.npmjs.com/package/@babel/register) (using `--require @babel/register`); -- loaders such as ECMAScript modules via [esm](https://www.npmjs.com/package/esm). - -### Overriding settings - -All options can also be overridden via command-line flags or environment variables. Priorities are the following: - -* A command-line option has the highest priority. It overrides the environment variable and config file value. - -* An environment variable has second priority. It overrides the config file value. - -* A config file value has the lowest priority. - -* If there isn't a command-line option, environment variable or config file option specified, the default is used. - -To override a config setting with a CLI option, convert the full option path to `--kebab-case`. For example, if you want to run tests against a different base URL, call: - -``` -testplane path/to/mytest.js --base-url http://example.com -``` - -To change the number of sessions for Firefox (assuming you have a browser with the `firefox` id in the config): - -``` -testplane path/to/mytest.js --browsers-firefox-sessions-per-browser 7 -``` - -To override a setting with an environment variable, convert its full path to `snake_case` and add the `testplane_` prefix. The above examples can be rewritten to use environment variables instead of CLI options: - -``` -testplane_base_url=http://example.com testplane path/to/mytest.js -testplane_browsers_firefox_sessions_per_browser=7 testplane path/to/mytest.js -``` - -### Debug mode - -In order to understand what is going on in the test step by step, there is a debug mode. You can run tests in this mode using these options: `--inspect` and `--inspect-brk`. The difference between them is that the second one stops before executing the code. - -Example: -``` -testplane path/to/mytest.js --inspect -``` - -**Note**: In the debugging mode, only one worker is started and all tests are performed only in it. -Use this mode with option `sessionsPerBrowser=1` in order to debug tests one at a time. - -### REPL mode - -Testplane provides a [REPL](https://en.wikipedia.org/wiki/Read–eval–print_loop) implementation that helps you not only to learn the framework API, but also to debug and inspect your tests. In this mode, there is no timeout for the duration of the test (it means that there will be enough time to debug the test). It can be used by specifying the CLI options: - -- `--repl` - in this mode, only one test in one browser should be run, otherwise an error is thrown. REPL interface does not start automatically, so you need to call [switchToRepl](#switchtorepl) command in the test code. Disabled by default; -- `--repl-before-test` - the same as `--repl` option except that REPL interface opens automatically before run test. Disabled by default; -- `--repl-on-fail` - the same as `--repl` option except that REPL interface opens automatically on test fail. Disabled by default. - -#### switchToRepl - -Browser command that stops the test execution and opens REPL interface in order to communicate with browser. For example: - -```js -it('foo', async ({browser}) => { - console.log('before open repl'); - - await browser.switchToRepl(); - - console.log('after open repl'); -}); -``` - -And run it using the command: - -```bash -npx testplane --repl --grep "foo" -b "chrome" -``` - -In this case, we are running only one test in one browser (or you can use `testplane.only.in('chrome')` before `it` + `it.only`). -When executing the test, the text `before open repl` will be displayed in the console first, then test execution stops, REPL interface is opened and waits your commands. So we can write some command in the terminal: - -```js -await browser.getUrl(); -// about:blank -``` - -In the case when you need to execute a block of code, for example: - -```js -for (const item of [...Array(3).keys]) { - await browser.$(`.selector_${item}`).isDisplayed(); -} -``` - -You need to switch to editor mode by running the `.editor` command in REPL and insert the desired a block of code. Then execute it by pressing `Ctrl+D`. -It is worth considering that some of code can be executed without editor mode: -- one-line code like `await browser.getUrl().then(console.log)`; -- few lines of code without using block scope or chaining, for example: - ```js - await browser.url('http://localhost:3000'); - await browser.getUrl(); - // http://localhost:3000 - ``` - -After user closes the server, the test will continue to run (text `after open repl` will be displayed in the console and browser will close). - -Another command features: -- all `const` and `let` declarations called in REPL mode are modified to `var` in runtime. This is done in order to be able to redefine created variables; -- before switching to the REPL mode `process.cwd` is replaced with the path to the folder of the executed test. After exiting from the REPL mode `process.cwd` is restored. This feature allows you to import modules relative to the test correctly; -- ability to pass the context to the REPL interface. For example: - - ```js - it('foo', async ({browser}) => { - const foo = 1; - - await browser.switchToRepl({foo}); - }); - ``` - - And now `foo` variable is available in REPL: - - ```bash - console.log("foo:", foo); - // foo: 1 - ``` - -#### Test development in runtime - -For quick test development without restarting the test or the browser, you can run the test in the terminal of IDE with enabled REPL mode: - -```bash -npx testplane --repl-before-test --grep "foo" -b "chrome" -``` - -After that, you need to configure the hotkey in IDE to run the selected one or more lines of code in the terminal. As a result, each new written line can be sent to the terminal using a hotkey and due to this, you can write a test much faster. - -Also, during the test development process, it may be necessary to execute commands in a clean environment (without side effects from already executed commands). You can achieve this with the following commands: -- [clearSession](#clearsession) - clears session state (deletes cookies, clears local and session storages). In some cases, the environment may contain side effects from already executed commands; -- [reloadSession](https://webdriver.io/docs/api/browser/reloadSession/) - creates a new session with a completely clean environment. - -##### How to set up using VSCode - -1. Open `Code` -> `Settings...` -> `Keyboard Shortcuts` and print `run selected text` to search input. After that, you can specify the desired key combination -2. Run `testplane` in repl mode (examples were above) -3. Select one or mode lines of code and press created hotkey - -##### How to set up using Webstorm - -Ability to run selected text in terminal will be available after this [issue](https://youtrack.jetbrains.com/issue/WEB-49916/Debug-JS-file-selection) will be resolved. - -### Environment variables - -#### TESTPLANE_SKIP_BROWSERS -Skip the browsers specified in the config by passing the browser IDs. Multiple browser IDs should be separated by commas -(spaces after commas are allowed). - -For example, -``` -TESTPLANE_SKIP_BROWSERS=ie10,ie11 testplane -``` - -#### TESTPLANE_SETS -Specify sets to run using the environment variable as an alternative to using the CLI option `--set`. - -For example, -``` -TESTPLANE_SETS=desktop,touch testplane -``` diff --git a/docs/component-testing.md b/docs/component-testing.md deleted file mode 100644 index 50b0adea4..000000000 --- a/docs/component-testing.md +++ /dev/null @@ -1,139 +0,0 @@ -## Testplane Component Testing (experimental) - -Almost every modern web interfaces -Almost all modern web interfaces are written using frameworks (React, Vue, Svelte, ...) to simplify the creation, reuse and composition of web components. It is important to test such components in isolation from each other to be sure each component is doing its job correctly. Just like we write unit tests separately from integration tests. Testplane already supports testing components using [Storybook](https://storybook.js.org/) (via [@testplane/storybook](https://github.com/gemini-testing/testplane-storybook) plugin), but this tool is not relevant for all projects. Therefore, we have developed another component testing option that does not require the use of Storybook. - -### Implementation options for component testing - -Component testing is a type of testing in which the logic of a web component is tested in isolation from the web page in which it is used. In order to perform such a test, you need to be able to render the component correctly. [JSDom](https://github.com/jsdom/jsdom) is often used for this task (it is also used inside Jest), which renders web components using the Node.js virtual renderer, that is without using a real browser. On the one hand, it works faster (the browser is not being launched), and on the other hand, it is less stable, since the checks are not performed in a real browser. The second popular solution is to use a very fast dev server [Vite](https://vitejs.dev/), which supports many frameworks (React, Vue, Svelte, ...) and is responsible for rendering components in isolation. - -We chose the option using Vite, as this approach ensures the page is tested more closely to reality (as if the user had opened it). At the same time, the tests themselves run a little longer than in the JSDom. But the most important thing for us is the stability and reproducibility of the test results, so the choice was obvious. - -### How to use it? - -We will set up testing of react components written in Typescript. Therefore, first of all, we will install the necessary dependencies: - -```bash -npm i testplane vite @vitejs/plugin-react @testing-library/react --save-dev -npm i react --save -``` - -Now let's create a Vite config in which we will connect the plugin to support React. Example: - -```typescript -// vite.config.ts -import { defineConfig } from 'vite'; -import react from '@vitejs/plugin-react'; - -export default defineConfig({ - plugins: [ - react(), - ] -}); -``` - -After that, we will configure the tests to run in the browser. To do this, specify the [testRunEnv](https://testplane.io/docs/v8/config/system/#testrunenv) option. Example: - -```typescript -// .testplane.conf.ts -export const { - // ... - system: { - // ... - testRunEnv: ['browser', { viteConfig: './vite.config.ts' }], - }, - sets: { - linux: { - files: [ - 'src/tests/**/*.testplane.tsx' - ], - browsers: [ - 'chrome' - ] - }, - }, -} -``` - -And in the end, we can write a test in which we simply output the `document` value to the console without using the [browser.execute](https://webdriver.io/docs/api/browser/execute) command: - -```typescript -// src/tests/test.testplane.tsx -it('should log document', async () => { - console.log(document); -}); -``` - -If such a test were performed in a Node.js environment, then it would have fallen with the error: `ReferenceError: document is not defined`. But in our case, it will be executed directly in the browser, where the global variable `document` is available. Therefore, in the browser and terminal log (we will tell you about this feature below) we will see the following: - -``` -{ - location: { - ancestorOrigins: {}, - href: 'http://localhost:56292/run-uuids/23d2af81-4259-425c-8214-c9e770d75ea4', - origin: 'http://localhost:56292', - protocol: 'http:', - host: 'localhost:56292', - hostname: 'localhost', - port: '56292', - pathname: '/run-uuids/23d2af81-4259-425c-8214-c9e770d75ea4', - search: '', - hash: '' - } -} -``` - -Let's write a more complex test with a render of the react component: - -```typescript -// src/tests/test.testplane.tsx -import { useState } from 'react'; -import { render } from '@testing-library/react'; - -// A simple component with a title and a counter button -function Component() { - const [count, setCount] = useState(0); - - return ( -
-

Testplane Component Testing

- -
- ); -} - -it('should render react button', async ({browser}) => { - render(); // rendering the component on the generated Vite page - - const button = await browser.$("button"); - - await button.click(); - await button.click(); - - await expect(button).toHaveText("count is 2"); -}); -``` - -A fully working examples can be found [here](../examples/component-testing/). - -> ⚠️ Currently, there are the following restrictions: -> - only components written in React in files `.jsx` and `.tsx` are supported. Ability to write tests in `.js` files will be implemented soon. We will also support the Vue framework in the near future; -> - there is no access to `currentTest` from `it`, `beforeEach` and `afterEach`. It will appear in the near future; -> - the [@testplane/global-hook](https://github.com/gemini-testing/testplane-global-hook) plugin is temporarily not supported. - - -### What additional features are supported? - -#### Hot Module Replacement (HMR) - -[HMR](https://vitejs.dev/guide/api-hmr.html) is supported in Vite. It means if you change the loaded file, either the component will be remounted, or the page will be completely preloaded. If the component is described in a separate file (i.e. not in the same file as the test), a remount will be performed, but the test will not be restarted. And if you change the test file, the page will be reloaded, which will cause Testplane to interrupt the execution of the current test and start it again. Due to this feature, you can quickly develop components in Vite and write tests for them. It is recommended to use it together with the [REPL mode](./cli.md#repl-mode). - -#### Using the browser and expect instances directly in the browser DevTools - -Instances of the `browser` and `expect` are available inside of the browser's global scope. It is quite convenient to use it when debugging the test. - -#### Logs from the browser console in the terminal - -Calling the `log`, `info`, `warn`, `error`, `debug` and `table` commands on the `console` object in the browser causes information to be displayed not only in the browser's DevTools, but also in the terminal from which Testplane was launched. I.e., you can call `console.log` in the test/component and you will be able to see the result of it execution in the terminal. This is especially handy when debugging the test. diff --git a/docs/events/index.md b/docs/events/index.md index caed544a8..6c6bb928c 100644 --- a/docs/events/index.md +++ b/docs/events/index.md @@ -25,7 +25,7 @@ TBD ### Events scheme description {#events-scheme-description} -Testplane can be launched either via the [CLI (command line)](../cli.md) or via its API: from a script using the `run` command. +Testplane can be launched either via the [CLI (command line)](https://testplane.io/docs/v8/command-line) or via its API: from a script using the `run` command. After launching, Testplane loads all plugins and proceeds to CLI parsing if it was launched via the CLI, or directly to the initialization stage if it was launched via the API. diff --git a/docs/programmatic-api.md b/docs/programmatic-api.md index e04db90a2..2cadc038e 100644 --- a/docs/programmatic-api.md +++ b/docs/programmatic-api.md @@ -255,6 +255,9 @@ await testplane.readTests(testPaths, options); * **ignore** (optional) `String|Glob|Array` - patterns to exclude paths from the test search. * **sets** (optional) `String[]`– Sets to run tests in. * **grep** (optional) `RegExp` – Pattern that defines which tests to run. + * **replMode** (optional) `{enabled: boolean; beforeTest: boolean; onFail: boolean;}` - [Test development mode using REPL](https://testplane.io/docs/v8/command-line/#repl). When reading the tests, it checks that only one test is running in one browser. + * **runnableOpts** (optional): + * **saveLocations** (optional) `Boolean` - flag to save `location` (`line` and `column`) to suites and tests. Allows to determine where the suite or test is declared in the file. ### isFailed @@ -317,6 +320,8 @@ TestCollection API: * `eachRootSuite((root, browserId) => ...)` - iterates over all root suites in collection which have some tests. +* `format(formatterType)` - formats the tests in one of the available formatting types (`list` or `tree`). You can read more about the available formatting types [here](https://testplane.io/docs/v8/command-line). + ### Test Parser API `TestParserAPI` object is emitted on `BEFORE_FILE_READ` event. It provides the ability to customize test parsing process. diff --git a/docs/writing-tests.md b/docs/writing-tests.md index 0d0fafae7..0587d0c4a 100644 --- a/docs/writing-tests.md +++ b/docs/writing-tests.md @@ -8,6 +8,7 @@ - [Hooks](#hooks) - [Skip](#skip) - [Only](#only) +- [Also](#also) - [Config overriding](#config-overriding) - [testTimeout](#testtimeout) - [WebdriverIO extensions](#webdriverio-extensions) diff --git a/package-lock.json b/package-lock.json index 586c9b276..d82fa67a7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,7 +25,6 @@ "debug": "2.6.9", "devtools": "8.39.0", "error-stack-parser": "2.1.4", - "escape-string-regexp": "1.0.5", "expect-webdriverio": "3.6.0", "fastq": "1.13.0", "fs-extra": "5.0.0", @@ -68,6 +67,7 @@ "@babel/preset-typescript": "7.24.1", "@commitlint/cli": "^19.0.3", "@commitlint/config-conventional": "^19.0.3", + "@cspotcode/source-map-support": "0.8.0", "@sinonjs/fake-timers": "10.3.0", "@swc/core": "1.3.40", "@types/babel__code-frame": "7.0.6", @@ -123,10 +123,14 @@ "node": ">= 18.0.0" }, "peerDependencies": { + "@cspotcode/source-map-support": ">=0.7.0", "@swc/core": ">=1.3.96", "ts-node": ">=10.5.0" }, "peerDependenciesMeta": { + "@cspotcode/source-map-support": { + "optional": true + }, "@swc/core": { "optional": true }, @@ -1228,9 +1232,9 @@ } }, "node_modules/@cspotcode/source-map-support": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", - "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.0.tgz", + "integrity": "sha512-pOQRG+w/T1KogjiuO4uqqa+dw/IIb8kDY0ctYfiJstWv7TOTmtuAkx8ZB4YgauDNn2huHR33oruOgi45VcatOg==", "dev": true, "dependencies": { "@jridgewell/trace-mapping": "0.3.9" @@ -17812,9 +17816,9 @@ } }, "@cspotcode/source-map-support": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", - "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.0.tgz", + "integrity": "sha512-pOQRG+w/T1KogjiuO4uqqa+dw/IIb8kDY0ctYfiJstWv7TOTmtuAkx8ZB4YgauDNn2huHR33oruOgi45VcatOg==", "dev": true, "requires": { "@jridgewell/trace-mapping": "0.3.9" diff --git a/package.json b/package.json index 406077c36..57f5ce306 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,6 @@ "debug": "2.6.9", "devtools": "8.39.0", "error-stack-parser": "2.1.4", - "escape-string-regexp": "1.0.5", "expect-webdriverio": "3.6.0", "fastq": "1.13.0", "fs-extra": "5.0.0", @@ -106,6 +105,7 @@ "@babel/preset-typescript": "7.24.1", "@commitlint/cli": "^19.0.3", "@commitlint/config-conventional": "^19.0.3", + "@cspotcode/source-map-support": "0.8.0", "@sinonjs/fake-timers": "10.3.0", "@swc/core": "1.3.40", "@types/babel__code-frame": "7.0.6", @@ -158,6 +158,7 @@ "uglifyify": "3.0.4" }, "peerDependencies": { + "@cspotcode/source-map-support": ">=0.7.0", "@swc/core": ">=1.3.96", "ts-node": ">=10.5.0" }, @@ -165,8 +166,11 @@ "ts-node": { "optional": true }, - "@swc/core": { + "@cspotcode/source-map-support": { "optional": true + }, + "@swc/core": { + "optional": true } } } diff --git a/src/bundle/test-transformer.ts b/src/bundle/test-transformer.ts index 2c5af56f5..b134b26f5 100644 --- a/src/bundle/test-transformer.ts +++ b/src/bundle/test-transformer.ts @@ -13,6 +13,7 @@ export const setupTransformHook = (opts: { removeNonJsImports?: boolean } = {}): configFile: false, compact: false, presets: [require("@babel/preset-typescript")], + sourceMaps: "inline", plugins: [ [ require("@babel/plugin-transform-react-jsx"), diff --git a/src/cli/commands/list-tests/index.ts b/src/cli/commands/list-tests/index.ts new file mode 100644 index 000000000..232ac8b89 --- /dev/null +++ b/src/cli/commands/list-tests/index.ts @@ -0,0 +1,65 @@ +import path from "node:path"; +import fs from "fs-extra"; + +import { Testplane } from "../../../testplane"; +import { Formatters, validateFormatter } from "../../../test-collection"; +import { CliCommands } from "../../constants"; +import { withCommonCliOptions, collectCliValues, handleRequires, type CommonCmdOpts } from "../../../utils/cli"; +import logger from "../../../utils/logger"; + +import type { ValueOf } from "../../../types/helpers"; + +const { LIST_TESTS: commandName } = CliCommands; + +type ListTestsCmdOpts = { + ignore?: Array; + silent?: boolean; + outputFile?: string; + formatter: ValueOf; +}; + +export type ListTestsCmd = typeof commander & CommonCmdOpts; + +export const registerCmd = (cliTool: ListTestsCmd, testplane: Testplane): void => { + withCommonCliOptions({ cmd: cliTool.command(`${commandName}`), actionName: "list" }) + .description("Lists all tests info in one of available formats") + .option("--ignore ", "exclude paths from tests read", collectCliValues) + .option("--silent [type]", "flag to disable events emitting while reading tests", Boolean, false) + .option("--output-file ", "save results to specified file") + .option("--formatter [name]", "return tests in specified format", String, Formatters.LIST) + .arguments("[paths...]") + .action(async (paths: string[], options: ListTestsCmdOpts) => { + const { grep, browser: browsers, set: sets, require: requireModules } = cliTool; + const { ignore, silent, outputFile, formatter } = options; + + try { + validateFormatter(formatter); + handleRequires(requireModules); + + const testCollection = await testplane.readTests(paths, { + browsers, + sets, + grep, + ignore, + silent, + runnableOpts: { + saveLocations: formatter === Formatters.TREE, + }, + }); + + const result = testCollection.format(formatter); + + if (outputFile) { + await fs.ensureDir(path.dirname(outputFile)); + await fs.writeJson(outputFile, result); + } else { + console.info(JSON.stringify(result)); + } + + process.exit(0); + } catch (err) { + logger.error((err as Error).stack || err); + process.exit(1); + } + }); +}; diff --git a/src/cli/constants.ts b/src/cli/constants.ts new file mode 100644 index 000000000..22fa8ec7a --- /dev/null +++ b/src/cli/constants.ts @@ -0,0 +1,3 @@ +export const CliCommands = { + LIST_TESTS: "list-tests", +} as const; diff --git a/src/cli/index.ts b/src/cli/index.ts index de5fee94b..53812a12d 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1,14 +1,15 @@ -import util from "util"; +import path from "node:path"; +import util from "node:util"; import { Command } from "@gemini-testing/commander"; -import escapeRe from "escape-string-regexp"; import defaults from "../config/defaults"; import { configOverriding } from "./info"; import { Testplane } from "../testplane"; import pkg from "../../package.json"; import logger from "../utils/logger"; -import { requireModule } from "../utils/module"; import { shouldIgnoreUnhandledRejection } from "../utils/errors"; +import { withCommonCliOptions, collectCliValues, handleRequires } from "../utils/cli"; +import { CliCommands } from "./constants"; export type TestplaneRunOpts = { cliName?: string }; @@ -47,13 +48,10 @@ export const run = (opts: TestplaneRunOpts = {}): void => { const configPath = preparseOption(program, "config") as string; testplane = Testplane.create(configPath); - program - .on("--help", () => logger.log(configOverriding(opts))) - .option("-b, --browser ", "run tests only in specified browser", collect) - .option("-s, --set ", "run tests only in the specified set", collect) - .option("-r, --require ", "require module", collect) - .option("--reporter ", "test reporters", collect) - .option("--grep ", "run only tests matching the pattern", compileGrep) + withCommonCliOptions({ cmd: program, actionName: "run" }) + .on("--help", () => console.log(configOverriding(opts))) + .description("Run tests") + .option("--reporter ", "test reporters", collectCliValues) .option( "--update-refs", 'update screenshot references or gather them if they do not exist ("assertView" command)', @@ -112,15 +110,18 @@ export const run = (opts: TestplaneRunOpts = {}): void => { } }); + for (const commandName of Object.values(CliCommands)) { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { registerCmd } = require(path.resolve(__dirname, "./commands", commandName)); + + registerCmd(program, testplane); + } + testplane.extendCli(program); program.parse(process.argv); }; -function collect(newValue: string | string[], array: string[] = []): string[] { - return array.concat(newValue); -} - function preparseOption(program: Command, option: string): unknown { // do not display any help, do not exit const configFileParser = Object.create(program); @@ -130,18 +131,3 @@ function preparseOption(program: Command, option: string): unknown { configFileParser.parse(process.argv); return configFileParser[option]; } - -function compileGrep(grep: string): RegExp { - try { - return new RegExp(`(${grep})|(${escapeRe(grep)})`); - } catch (error) { - logger.warn(`Invalid regexp provided to grep, searching by its string representation. ${error}`); - return new RegExp(escapeRe(grep)); - } -} - -async function handleRequires(requires: string[] = []): Promise { - for (const modulePath of requires) { - await requireModule(modulePath); - } -} diff --git a/src/index.ts b/src/index.ts index 3b5b4127d..c2346ed05 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,7 +24,14 @@ export type { } from "./types"; export type { Config } from "./config"; export type { ConfigInput } from "./config/types"; -export type { TestCollection } from "./test-collection"; +export type { + TestCollection, + FormatterTreeSuite, + FormatterTreeTest, + FormatterTreeMainRunnable, + FormatterListTest, +} from "./test-collection"; +export type { StatsResult } from "./stats"; import type { TestDefinition, SuiteDefinition, TestHookDefinition } from "./test-reader/test-object/types"; diff --git a/src/test-collection/constants.ts b/src/test-collection/constants.ts new file mode 100644 index 000000000..5cb0dd80b --- /dev/null +++ b/src/test-collection/constants.ts @@ -0,0 +1,6 @@ +export const Formatters = { + LIST: "list", + TREE: "tree", +} as const; + +export const AVAILABLE_FORMATTERS = Object.values(Formatters); diff --git a/src/test-collection/formatters/list.ts b/src/test-collection/formatters/list.ts new file mode 100644 index 000000000..52ac17edd --- /dev/null +++ b/src/test-collection/formatters/list.ts @@ -0,0 +1,33 @@ +import path from "node:path"; +import type { TestCollection, TestDisabled, FormatterListTest } from ".."; + +export const format = (testCollection: TestCollection): FormatterListTest[] => { + const allTestsById = new Map(); + + testCollection.eachTest((test, browserId) => { + if ((test as TestDisabled).disabled) { + return; + } + + if (allTestsById.has(test.id)) { + const foundTest = allTestsById.get(test.id)!; + + if (!foundTest.browserIds.includes(browserId)) { + foundTest.browserIds.push(browserId); + } + + return; + } + + allTestsById.set(test.id, { + id: test.id, + titlePath: test.titlePath(), + browserIds: [browserId], + file: path.relative(process.cwd(), test.file as string), + pending: test.pending, + skipReason: test.skipReason, + }); + }); + + return [...allTestsById.values()]; +}; diff --git a/src/test-collection/formatters/tree.ts b/src/test-collection/formatters/tree.ts new file mode 100644 index 000000000..1cc2386b7 --- /dev/null +++ b/src/test-collection/formatters/tree.ts @@ -0,0 +1,126 @@ +import path from "node:path"; + +import type { + TestCollection, + TestDisabled, + FormatterTreeMainRunnable, + FormatterTreeTest, + FormatterTreeSuite, +} from ".."; +import type { Suite, Test } from "../../types"; + +export const format = (testCollection: TestCollection): FormatterTreeMainRunnable[] => { + const allSuitesById = new Map(); + const allTestsById = new Map(); + + testCollection.eachTest((test, browserId) => { + if ((test as TestDisabled).disabled) { + return; + } + + if (allTestsById.has(test.id)) { + const treeTest = allTestsById.get(test.id)!; + + if (!treeTest.browserIds.includes(browserId)) { + treeTest.browserIds.push(browserId); + } + + return; + } + + const treeTest = createTreeTest(test, browserId); + allTestsById.set(treeTest.id, treeTest); + + collectSuites(test.parent!, treeTest, allSuitesById); + }); + + return getTreeRunnables(allSuitesById, allTestsById); +}; + +function collectSuites( + suite: Suite, + child: FormatterTreeTest | FormatterTreeSuite, + allSuitesById: Map, +): void { + if (allSuitesById.has(suite.id)) { + const treeSuite = allSuitesById.get(suite.id)!; + addChild(treeSuite, child); + + return; + } + + if (!suite.parent) { + return; + } + + const treeSuite = createTreeSuite(suite); + addChild(treeSuite, child); + + allSuitesById.set(treeSuite.id, treeSuite); + + collectSuites(suite.parent, treeSuite, allSuitesById); +} + +function isTreeTest(runnable: unknown): runnable is FormatterTreeTest { + return Boolean((runnable as FormatterTreeTest).browserIds); +} + +function createTreeTest(test: Test, browserId: string): FormatterTreeTest { + return { + id: test.id, + title: test.title, + pending: test.pending, + skipReason: test.skipReason, + ...test.location!, + browserIds: [browserId], + ...getMainRunanbleFields(test), + }; +} + +function createTreeSuite(suite: Suite): FormatterTreeSuite { + return { + id: suite.id, + title: suite.title, + pending: suite.pending, + skipReason: suite.skipReason, + ...suite.location!, + ...getMainRunanbleFields(suite), + suites: [], + tests: [], + }; +} + +function addChild(treeSuite: FormatterTreeSuite, child: FormatterTreeTest | FormatterTreeSuite): void { + const fieldName = isTreeTest(child) ? "tests" : "suites"; + const foundRunnable = treeSuite[fieldName].find(test => test.id === child.id); + + if (!foundRunnable) { + isTreeTest(child) ? addTest(treeSuite, child) : addSuite(treeSuite, child); + } +} + +function addTest(treeSuite: FormatterTreeSuite, child: FormatterTreeTest): void { + treeSuite.tests.push(child); +} + +function addSuite(treeSuite: FormatterTreeSuite, child: FormatterTreeSuite): void { + treeSuite.suites.push(child); +} + +function getMainRunanbleFields(runanble: Suite | Test): Partial> { + const isMain = runanble.parent && runanble.parent.root; + + return { + // "file" field must exists only in topmost runnables + ...(isMain ? { file: path.relative(process.cwd(), runanble.file) } : {}), + }; +} + +function getTreeRunnables( + allSuitesById: Map, + allTestsById: Map, +): FormatterTreeMainRunnable[] { + return [...allSuitesById.values(), ...allTestsById.values()].filter( + suite => (suite as FormatterTreeMainRunnable).file, + ) as FormatterTreeMainRunnable[]; +} diff --git a/src/test-collection.ts b/src/test-collection/index.ts similarity index 76% rename from src/test-collection.ts rename to src/test-collection/index.ts index c26f41f78..295de89d9 100644 --- a/src/test-collection.ts +++ b/src/test-collection/index.ts @@ -1,8 +1,42 @@ +import path from "node:path"; import _ from "lodash"; - -import type { Suite, RootSuite, Test } from "./types"; - -type TestDisabled = Test & { disabled: true }; +import { Formatters, AVAILABLE_FORMATTERS } from "./constants"; + +import type { ValueOf } from "../types/helpers"; +import type { Suite, RootSuite, Test } from "../types"; + +export * from "./constants"; + +export type FormatterTreeSuite = { + id: string; + title: string; + line: number; + column: number; + suites: FormatterTreeSuite[]; + // eslint-disable-next-line no-use-before-define + tests: FormatterTreeTest[]; + pending: boolean; + skipReason: string; +}; + +export type FormatterTreeTest = Omit & { + browserIds: string[]; +}; + +export type FormatterTreeMainRunnable = (FormatterTreeSuite | FormatterTreeTest) & { + file: string; +}; + +export type FormatterListTest = { + id: string; + titlePath: string[]; + file: string; + browserIds: string[]; + pending: boolean; + skipReason: string; +}; + +export type TestDisabled = Test & { disabled: true }; type TestsCallback = (test: Test, browserId: string) => T; type SortTestsCallback = (test1: Test, test2: Test) => number; @@ -22,6 +56,10 @@ export class TestCollection { this.#specs = _.mapValues(specs, _.clone); } + get formatters(): typeof Formatters { + return Formatters; + } + getRootSuite(browserId: string): RootSuite | null { const test = this.#originalSpecs[browserId][0]; return test && test.parent && this.#getRoot(test.parent); @@ -172,4 +210,22 @@ export class TestCollection { return this; } + + format(formatterType: ValueOf): (FormatterListTest | FormatterTreeMainRunnable)[] { + validateFormatter(formatterType); + + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { format } = require(path.resolve(__dirname, "./formatters", formatterType)) as + | typeof import("./formatters/list") + | typeof import("./formatters/tree"); + return format(this); + } +} + +export function validateFormatter(formatterType: ValueOf): void { + if (!AVAILABLE_FORMATTERS.includes(formatterType)) { + throw new Error( + `"formatter" option must be one of: ${AVAILABLE_FORMATTERS.join(", ")}, but got ${formatterType}`, + ); + } } diff --git a/src/test-reader/index.ts b/src/test-reader/index.ts index 1425248b2..3d983761f 100644 --- a/src/test-reader/index.ts +++ b/src/test-reader/index.ts @@ -29,7 +29,7 @@ export class TestReader extends EventEmitter { } async read(options: TestReaderOpts): Promise> { - const { paths, browsers, ignore, sets, grep } = options; + const { paths, browsers, ignore, sets, grep, runnableOpts } = options; const { fileExtensions } = this.#config.system; const envSets = env.parseCommaSeparatedValue(["TESTPLANE_SETS", "HERMIONE_SETS"]).value; @@ -46,7 +46,7 @@ export class TestReader extends EventEmitter { const parser = new TestParser({ testRunEnv }); passthroughEvent(parser, this, [MasterEvents.BEFORE_FILE_READ, MasterEvents.AFTER_FILE_READ]); - await parser.loadFiles(setCollection.getAllFiles(), this.#config); + await parser.loadFiles(setCollection.getAllFiles(), { config: this.#config, runnableOpts }); const filesByBro = setCollection.groupByBrowser(); const testsByBro = _.mapValues(filesByBro, (files, browserId) => diff --git a/src/test-reader/mocha-reader/index.js b/src/test-reader/mocha-reader/index.js index 7ff5ea5ee..3a46c7909 100644 --- a/src/test-reader/mocha-reader/index.js +++ b/src/test-reader/mocha-reader/index.js @@ -1,17 +1,20 @@ "use strict"; +const _ = require("lodash"); +const Mocha = require("mocha"); + const { MochaEventBus } = require("./mocha-event-bus"); const { TreeBuilderDecorator } = require("./tree-builder-decorator"); const { TestReaderEvents } = require("../../events"); const { MasterEvents } = require("../../events"); -const Mocha = require("mocha"); +const { getMethodsByInterface } = require("./utils"); -async function readFiles(files, { esmDecorator, config, eventBus }) { +async function readFiles(files, { esmDecorator, config, eventBus, runnableOpts }) { const mocha = new Mocha(config); mocha.fullTrace(); initBuildContext(eventBus); - initEventListeners(mocha.suite, eventBus); + initEventListeners({ rootSuite: mocha.suite, outBus: eventBus, config, runnableOpts }); files.forEach(f => mocha.addFile(f)); await mocha.loadFilesAsync({ esmDecorator }); @@ -25,11 +28,12 @@ function initBuildContext(outBus) { }); } -function initEventListeners(rootSuite, outBus) { +function initEventListeners({ rootSuite, outBus, config, runnableOpts }) { const inBus = MochaEventBus.create(rootSuite); forbidSuiteHooks(inBus); passthroughFileEvents(inBus, outBus); + addLocationToRunnables(inBus, config, runnableOpts); registerTestObjects(inBus, outBus); inBus.emit(MochaEventBus.events.EVENT_SUITE_ADD_SUITE, rootSuite); @@ -95,6 +99,97 @@ function applyOnly(rootSuite, eventBus) { }); } +function addLocationToRunnables(inBus, config, runnableOpts) { + if (!runnableOpts || !runnableOpts.saveLocations) { + return; + } + + const sourceMapSupport = tryToRequireSourceMapSupport(); + const { suiteMethods, testMethods } = getMethodsByInterface(config.ui); + + inBus.on(MochaEventBus.events.EVENT_FILE_PRE_REQUIRE, ctx => { + [ + { + methods: suiteMethods, + eventName: MochaEventBus.events.EVENT_SUITE_ADD_SUITE, + }, + { + methods: testMethods, + eventName: MochaEventBus.events.EVENT_SUITE_ADD_TEST, + }, + ].forEach(({ methods, eventName }) => { + methods.forEach(methodName => { + ctx[methodName] = withLocation(ctx[methodName], { inBus, eventName, sourceMapSupport }); + + if (ctx[methodName]) { + ctx[methodName].only = withLocation(ctx[methodName].only, { inBus, eventName, sourceMapSupport }); + ctx[methodName].skip = withLocation(ctx[methodName].skip, { inBus, eventName, sourceMapSupport }); + } + + if (!config.ui || config.ui === "bdd") { + const pendingMethodName = `x${methodName}`; + ctx[pendingMethodName] = withLocation(ctx[pendingMethodName], { + inBus, + eventName, + sourceMapSupport, + }); + } + }); + }); + }); +} + +function withLocation(origFn, { inBus, eventName, sourceMapSupport }) { + if (!_.isFunction(origFn)) { + return origFn; + } + + const wrappedFn = (...args) => { + const origStackTraceLimit = Error.stackTraceLimit; + const origPrepareStackTrace = Error.prepareStackTrace; + + Error.stackTraceLimit = 2; + Error.prepareStackTrace = (error, stackFrames) => { + const frame = sourceMapSupport ? sourceMapSupport.wrapCallSite(stackFrames[1]) : stackFrames[1]; + + return { + line: frame.getLineNumber(), + column: frame.getColumnNumber(), + }; + }; + + const obj = {}; + Error.captureStackTrace(obj); + + const location = obj.stack; + Error.stackTraceLimit = origStackTraceLimit; + Error.prepareStackTrace = origPrepareStackTrace; + + inBus.once(eventName, runnable => { + if (!runnable.location) { + runnable.location = location; + } + }); + + return origFn(...args); + }; + + for (const key of Object.keys(origFn)) { + wrappedFn[key] = origFn[key]; + } + + return wrappedFn; +} + +function tryToRequireSourceMapSupport() { + try { + const module = require("@cspotcode/source-map-support"); + module.install({ hookRequire: true }); + + return module; + } catch {} // eslint-disable-line no-empty +} + module.exports = { readFiles, }; diff --git a/src/test-reader/mocha-reader/tree-builder-decorator.js b/src/test-reader/mocha-reader/tree-builder-decorator.js index 27cad49de..1cdd14b67 100644 --- a/src/test-reader/mocha-reader/tree-builder-decorator.js +++ b/src/test-reader/mocha-reader/tree-builder-decorator.js @@ -62,8 +62,8 @@ class TreeBuilderDecorator { } #mkTestObject(Constructor, mochaObject, customOpts) { - const { title, file } = mochaObject; - return Constructor.create({ title, file, ...customOpts }); + const { title, file, location } = mochaObject; + return Constructor.create({ title, file, location, ...customOpts }); } #applyConfig(testObject, mochaObject) { diff --git a/src/test-reader/mocha-reader/utils.js b/src/test-reader/mocha-reader/utils.js index eac1aadce..b9478c474 100644 --- a/src/test-reader/mocha-reader/utils.js +++ b/src/test-reader/mocha-reader/utils.js @@ -59,6 +59,18 @@ const computeFile = mochaSuite => { return null; }; +const getMethodsByInterface = (mochaInterface = "bdd") => { + switch (mochaInterface) { + case "tdd": + case "qunit": + return { suiteMethods: ["suite"], testMethods: ["test"] }; + case "bdd": + default: + return { suiteMethods: ["describe", "context"], testMethods: ["it", "specify"] }; + } +}; + module.exports = { computeFile, + getMethodsByInterface, }; diff --git a/src/test-reader/test-object/configurable-test-object.ts b/src/test-reader/test-object/configurable-test-object.ts index f72f92f24..ca6819996 100644 --- a/src/test-reader/test-object/configurable-test-object.ts +++ b/src/test-reader/test-object/configurable-test-object.ts @@ -2,7 +2,7 @@ import _ from "lodash"; import { TestObject } from "./test-object"; import type { ConfigurableTestObjectData, TestObjectData } from "./types"; -type ConfigurableTestObjectOpts = Pick & TestObjectData; +type ConfigurableTestObjectOpts = Pick & TestObjectData; type SkipData = { reason: string; @@ -11,10 +11,10 @@ type SkipData = { export class ConfigurableTestObject extends TestObject { #data: ConfigurableTestObjectData; - constructor({ title, file, id }: ConfigurableTestObjectOpts) { + constructor({ title, file, id, location }: ConfigurableTestObjectOpts) { super({ title }); - this.#data = { id, file } as ConfigurableTestObjectData; + this.#data = { id, file, location } as ConfigurableTestObjectData; } assign(src: this): this { @@ -109,4 +109,8 @@ export class ConfigurableTestObject extends TestObject { #getInheritedProperty(name: keyof ConfigurableTestObjectData, defaultValue: T): T { return name in this.#data ? (this.#data[name] as T) : (_.get(this.parent, name, defaultValue) as T); } + + get location(): ConfigurableTestObjectData["location"] { + return this.#data.location; + } } diff --git a/src/test-reader/test-object/suite.ts b/src/test-reader/test-object/suite.ts index 90a673c73..78b570bd9 100644 --- a/src/test-reader/test-object/suite.ts +++ b/src/test-reader/test-object/suite.ts @@ -4,7 +4,7 @@ import { Hook } from "./hook"; import { Test } from "./test"; import type { TestObjectData, ConfigurableTestObjectData, TestFunction, TestFunctionCtx } from "./types"; -type SuiteOpts = Pick & TestObjectData; +type SuiteOpts = Pick & TestObjectData; export class Suite extends ConfigurableTestObject { #suites: this[]; @@ -17,8 +17,8 @@ export class Suite extends ConfigurableTestObject { } // used inside test - constructor({ title, file, id }: SuiteOpts = {} as SuiteOpts) { - super({ title, file, id }); + constructor({ title, file, id, location }: SuiteOpts = {} as SuiteOpts) { + super({ title, file, id, location }); this.#suites = []; this.#tests = []; diff --git a/src/test-reader/test-object/test.ts b/src/test-reader/test-object/test.ts index 0e682297a..6641d4194 100644 --- a/src/test-reader/test-object/test.ts +++ b/src/test-reader/test-object/test.ts @@ -2,7 +2,7 @@ import { ConfigurableTestObject } from "./configurable-test-object"; import type { TestObjectData, TestFunction, TestFunctionCtx } from "./types"; type TestOpts = TestObjectData & - Pick & { + Pick & { fn: TestFunction; }; @@ -14,8 +14,8 @@ export class Test extends ConfigurableTestObject { return new this(opts); } - constructor({ title, file, id, fn }: TestOpts) { - super({ title, file, id }); + constructor({ title, file, id, location, fn }: TestOpts) { + super({ title, file, id, location }); this.fn = fn; } @@ -25,6 +25,7 @@ export class Test extends ConfigurableTestObject { title: this.title, file: this.file, id: this.id, + location: this.location, fn: this.fn, }).assign(this); } diff --git a/src/test-reader/test-object/types.ts b/src/test-reader/test-object/types.ts index a26a46863..a5d8b2140 100644 --- a/src/test-reader/test-object/types.ts +++ b/src/test-reader/test-object/types.ts @@ -6,6 +6,11 @@ export type TestObjectData = { title: string; }; +export type Location = { + line: number; + column: number; +}; + export type ConfigurableTestObjectData = { id: string; pending: boolean; @@ -16,6 +21,7 @@ export type ConfigurableTestObjectData = { silentSkip: boolean; browserId: string; browserVersion?: string; + location?: Location; }; export interface TestFunctionCtx { diff --git a/src/test-reader/test-parser.ts b/src/test-reader/test-parser.ts index a8a9b35f5..a641d0d43 100644 --- a/src/test-reader/test-parser.ts +++ b/src/test-reader/test-parser.ts @@ -20,16 +20,23 @@ import { getShortMD5 } from "../utils/crypto"; import { Test } from "./test-object"; import { Config } from "../config"; import { BrowserConfig } from "../config/browser-config"; +import type { ReadTestsOpts } from "../testplane"; export type TestParserOpts = { testRunEnv?: "nodejs" | "browser"; }; + export type TestParserParseOpts = { browserId: string; grep?: RegExp; config: BrowserConfig; }; +type LoadFilesOpts = { + config: Config; + runnableOpts?: ReadTestsOpts["runnableOpts"]; +}; + const getFailedTestId = (test: { fullTitle: string; browserId: string; browserVersion?: string }): string => getShortMD5(`${test.fullTitle}${test.browserId}${test.browserVersion}`); @@ -46,7 +53,7 @@ export class TestParser extends EventEmitter { this.#buildInstructions = new InstructionsList(); } - async loadFiles(files: string[], config: Config): Promise { + async loadFiles(files: string[], { config, runnableOpts }: LoadFilesOpts): Promise { const eventBus = new EventEmitter(); const { system: { ctx, mochaOpts }, @@ -80,7 +87,7 @@ export class TestParser extends EventEmitter { const rand = Math.random(); const esmDecorator = (f: string): string => f + `?rand=${rand}`; - await readFiles(files, { esmDecorator, config: mochaOpts, eventBus }); + await readFiles(files, { esmDecorator, config: mochaOpts, eventBus, runnableOpts }); if (config.lastFailed.only) { try { diff --git a/src/testplane.ts b/src/testplane.ts index 8f8a3c12b..aad8a88ff 100644 --- a/src/testplane.ts +++ b/src/testplane.ts @@ -44,10 +44,15 @@ export type FailedListItem = { fullTitle: string; }; +interface RunnableOpts { + saveLocations?: boolean; +} + export interface ReadTestsOpts extends Pick { silent: boolean; ignore: string | string[]; failed: FailedListItem[]; + runnableOpts?: RunnableOpts; } export interface Testplane { @@ -152,7 +157,7 @@ export class Testplane extends BaseTestplane { async readTests( testPaths: string[], - { browsers, sets, grep, silent, ignore, replMode }: Partial = {}, + { browsers, sets, grep, silent, ignore, replMode, runnableOpts }: Partial = {}, ): Promise { const testReader = TestReader.create(this._config); @@ -165,7 +170,7 @@ export class Testplane extends BaseTestplane { ]); } - const specs = await testReader.read({ paths: testPaths, browsers, ignore, sets, grep, replMode }); + const specs = await testReader.read({ paths: testPaths, browsers, ignore, sets, grep, replMode, runnableOpts }); const collection = TestCollection.create(specs); collection.getBrowsers().forEach(bro => { diff --git a/src/types/helpers.ts b/src/types/helpers.ts new file mode 100644 index 000000000..5f2cf2cf0 --- /dev/null +++ b/src/types/helpers.ts @@ -0,0 +1 @@ +export type ValueOf = T[keyof T]; diff --git a/src/utils/cli.ts b/src/utils/cli.ts new file mode 100644 index 000000000..cfe87f51e --- /dev/null +++ b/src/utils/cli.ts @@ -0,0 +1,45 @@ +import _ from "lodash"; +import type { Command } from "@gemini-testing/commander"; +import logger from "./logger"; +import { requireModule } from "./module"; + +export const collectCliValues = (newValue: unknown, array = [] as unknown[]): unknown[] => { + return array.concat(newValue); +}; + +export const compileGrep = (grep: string): RegExp => { + try { + return new RegExp(`(${grep})|(${_.escapeRegExp(grep)})`); + } catch (error) { + logger.warn(`Invalid regexp provided to grep, searching by its string representation. ${error}`); + return new RegExp(_.escapeRegExp(grep)); + } +}; + +export const handleRequires = async (requires: string[] = []): Promise => { + for (const modulePath of requires) { + await requireModule(modulePath); + } +}; + +export type CommonCmdOpts = { + config?: string; + browser?: Array; + set?: Array; + require?: Array; + grep?: RegExp; +}; + +export const withCommonCliOptions = ({ cmd, actionName = "run" }: { cmd: Command; actionName: string }): Command => { + const isMainCmd = ["testplane", "hermione"].includes(cmd.name()); + + if (!isMainCmd) { + cmd.option("-c, --config ", "path to configuration file"); + } + + return cmd + .option("-b, --browser ", `${actionName} tests only in specified browser`, collectCliValues) + .option("-s, --set ", `${actionName} tests only in the specified set`, collectCliValues) + .option("-r, --require ", "require module", collectCliValues) + .option("--grep ", `${actionName} only tests matching the pattern`, compileGrep); +}; diff --git a/src/worker/runner/simple-test-parser.ts b/src/worker/runner/simple-test-parser.ts index 70b8fd2fe..58df657c9 100644 --- a/src/worker/runner/simple-test-parser.ts +++ b/src/worker/runner/simple-test-parser.ts @@ -38,7 +38,7 @@ export class SimpleTestParser extends EventEmitter { passthroughEvent(parser, this, [WorkerEvents.BEFORE_FILE_READ, WorkerEvents.AFTER_FILE_READ]); - await parser.loadFiles([file], this._config); + await parser.loadFiles([file], { config: this._config }); return parser.parse([file], { browserId, config: this._config.forBrowser(browserId) }); } diff --git a/test/src/cli/commands/list-tests/index.ts b/test/src/cli/commands/list-tests/index.ts new file mode 100644 index 000000000..8bed5bf1a --- /dev/null +++ b/test/src/cli/commands/list-tests/index.ts @@ -0,0 +1,171 @@ +import path from "node:path"; +import { Command } from "@gemini-testing/commander"; +import fs from "fs-extra"; +import sinon, { SinonStub } from "sinon"; +import proxyquire from "proxyquire"; + +import { Formatters } from "../../../../../src/test-collection"; +import logger from "../../../../../src/utils/logger"; +import { Testplane } from "../../../../../src/testplane"; +import * as testplaneCli from "../../../../../src/cli"; +import { TestCollection } from "../../../../../src/test-collection"; + +describe("cli/commands/list-tests", () => { + const sandbox = sinon.createSandbox(); + + const listTests_ = async (argv: string = "", cli: { run: VoidFunction } = testplaneCli): Promise => { + process.argv = ["foo/bar/node", "foo/bar/script", "list-tests", ...argv.split(" ")].filter(Boolean); + cli.run(); + + await (Command.prototype.action as SinonStub).lastCall.returnValue; + }; + + beforeEach(() => { + sandbox.stub(Testplane, "create").returns(Object.create(Testplane.prototype)); + sandbox.stub(Testplane.prototype, "readTests").resolves(TestCollection.create({})); + + sandbox.stub(fs, "ensureDir").resolves(); + sandbox.stub(fs, "writeJson").resolves(); + + sandbox.stub(logger, "error"); + sandbox.stub(console, "info"); + sandbox.stub(process, "exit"); + + sandbox.spy(Command.prototype, "action"); + }); + + afterEach(() => sandbox.restore()); + + it("should validate passed formatter", async () => { + const validateFormatterStub = sandbox.stub(); + const cli = proxyquire("../../../../../src/cli", { + [path.resolve(process.cwd(), "src/cli/commands/list-tests")]: proxyquire( + "../../../../../src/cli/commands/list-tests", + { + "../../../test-collection": { + validateFormatter: validateFormatterStub, + }, + }, + ), + }); + + await listTests_("--formatter foo", cli); + + assert.calledOnceWith(validateFormatterStub, "foo"); + }); + + it("should exit with code 0", async () => { + await listTests_(); + + assert.calledWith(process.exit as unknown as SinonStub, 0); + }); + + it("should exit with code 1 if read tests failed", async () => { + (Testplane.prototype.readTests as SinonStub).rejects(new Error("o.O")); + + await listTests_(); + + assert.calledWith(process.exit as unknown as SinonStub, 1); + }); + + describe("read tests", () => { + it("should use paths from cli", async () => { + await listTests_("first.testplane.js second.testplane.js"); + + assert.calledWith(Testplane.prototype.readTests as SinonStub, [ + "first.testplane.js", + "second.testplane.js", + ]); + }); + + it("should use browsers from cli", async () => { + await listTests_("--browser first --browser second"); + + assert.calledWithMatch(Testplane.prototype.readTests as SinonStub, sinon.match.any, { + browsers: ["first", "second"], + }); + }); + + it("should use sets from cli", async () => { + await listTests_("--set first --set second"); + + assert.calledWithMatch(Testplane.prototype.readTests as SinonStub, sinon.match.any, { + sets: ["first", "second"], + }); + }); + + it("should use grep from cli", async () => { + await listTests_("--grep some"); + + assert.calledWithMatch(Testplane.prototype.readTests as SinonStub, sinon.match.any, { + grep: sinon.match.instanceOf(RegExp), + }); + }); + + it("should use ignore paths from cli", async () => { + await listTests_("--ignore first --ignore second"); + + assert.calledWithMatch(Testplane.prototype.readTests as SinonStub, sinon.match.any, { + ignore: ["first", "second"], + }); + }); + + describe("silent", () => { + it("should be disabled by default", async () => { + await listTests_(); + + assert.calledWithMatch(Testplane.prototype.readTests as SinonStub, sinon.match.any, { silent: false }); + }); + + it("should use from cli", async () => { + await listTests_("--silent"); + + assert.calledWithMatch(Testplane.prototype.readTests as SinonStub, sinon.match.any, { silent: true }); + }); + }); + + describe("runnableOpts", () => { + it("should not save runnale locations by default", async () => { + await listTests_(); + + assert.calledWithMatch(Testplane.prototype.readTests as SinonStub, sinon.match.any, { + runnableOpts: { + saveLocations: false, + }, + }); + }); + + it(`should save runnale locations if "${Formatters.TREE}" formatter is used`, async () => { + await listTests_(`--formatter ${Formatters.TREE}`); + + assert.calledWithMatch(Testplane.prototype.readTests as SinonStub, sinon.match.any, { + runnableOpts: { + saveLocations: true, + }, + }); + }); + }); + }); + + [Formatters.LIST, Formatters.TREE].forEach(formatterName => { + describe(`${formatterName} formatter`, () => { + beforeEach(() => { + sandbox.stub(TestCollection, "create").returns(Object.create(TestCollection.prototype)); + sandbox.stub(TestCollection.prototype, "format").returns([]); + }); + + it("should send result to stdout", async () => { + await listTests_(`--formatter ${formatterName}`); + + assert.calledOnceWith(console.info, JSON.stringify([])); + }); + + it("should save result to output file", async () => { + await listTests_(`--formatter ${formatterName} --output-file ./folder/file.json`); + + (fs.ensureDir as SinonStub).calledOnceWith("./folder"); + (fs.writeJson as SinonStub).calledOnceWith("./folder/file.json", []); + }); + }); + }); +}); diff --git a/test/src/cli/index.js b/test/src/cli/index.js index 2942881dc..d5ec54184 100644 --- a/test/src/cli/index.js +++ b/test/src/cli/index.js @@ -7,6 +7,7 @@ const { configOverriding } = require("src/cli/info"); const defaults = require("src/config/defaults"); const { Testplane } = require("src/testplane"); const logger = require("src/utils/logger"); +const { collectCliValues, withCommonCliOptions } = require("src/utils/cli"); const any = sinon.match.any; @@ -40,10 +41,12 @@ describe("cli", () => { describe("config overriding", () => { it('should show information about config overriding on "--help"', async () => { + sandbox.stub(console, "log"); + await run_("--help"); - assert.calledOnce(logger.log); - assert.calledWith(logger.log, configOverriding()); + assert.calledOnce(console.log); + assert.calledWith(console.log, configOverriding()); }); it("should show information about testplane by default", async () => { @@ -68,14 +71,14 @@ describe("cli", () => { }); it('should require modules specified in "require" option', async () => { - const requireModule = sandbox.stub(); + const handleRequires = sandbox.stub(); const stubTestplaneCli = proxyquire("src/cli", { - "../utils/module": { requireModule }, + "../utils/cli": { handleRequires, withCommonCliOptions }, }); await run_("--require foo", stubTestplaneCli); - assert.calledOnceWith(requireModule, "foo"); + assert.calledOnceWith(handleRequires, ["foo"]); }); it("should create Testplane without config by default", async () => { @@ -172,7 +175,7 @@ describe("cli", () => { it("should use require modules from cli", async () => { const stubTestplaneCli = proxyquire("src/cli", { - "../utils/module": { requireModule: sandbox.stub() }, + "../utils/cli": { handleRequires: sandbox.stub(), collectCliValues, withCommonCliOptions }, }); await run_("--require foo", stubTestplaneCli); diff --git a/test/src/test-collection/formatters/list.ts b/test/src/test-collection/formatters/list.ts new file mode 100644 index 000000000..aa0afb8c3 --- /dev/null +++ b/test/src/test-collection/formatters/list.ts @@ -0,0 +1,97 @@ +import path from "node:path"; +import _ from "lodash"; + +import { TestCollection } from "../../../../src/test-collection"; +import { format } from "../../../../src/test-collection/formatters/list"; +import { Test, Suite } from "../../../../src/test-reader/test-object"; + +type TestOpts = { + id: string; + title: string; + file: string; + parent: Suite; + disabled: boolean; + pending: boolean; + skipReason: string; +}; + +describe("test-collection/formatters/list", () => { + const mkTest_ = (opts: Partial = { title: "default-title" }): Test => { + const paramNames = ["id", "title", "file"]; + + const test = new Test(_.pick(opts, paramNames) as any); + for (const [key, value] of _.entries(_.omit(opts, paramNames))) { + _.set(test, key, value); + } + + return test; + }; + + it("should return empty array if all tests are disabled", () => { + const collection = TestCollection.create({ + bro1: [mkTest_({ disabled: true })], + bro2: [mkTest_({ disabled: true })], + }); + + const result = format(collection); + + assert.deepEqual(result, []); + }); + + it("should return skipped test", () => { + const root = new Suite({ title: "root" } as any); + const file = path.resolve(process.cwd(), "./folder/file.ts"); + const test = mkTest_({ id: "0", title: "test", file: file, parent: root, pending: true, skipReason: "flaky" }); + const collection = TestCollection.create({ bro: [test] }); + + const result = format(collection); + + assert.deepEqual(result, [ + { + id: "0", + titlePath: ["root", "test"], + browserIds: ["bro"], + file: "folder/file.ts", + pending: true, + skipReason: "flaky", + }, + ]); + }); + + it("should return tests with correct fields", () => { + const root1 = new Suite({ title: "root1" } as any); + const root2 = new Suite({ title: "root2" } as any); + + const file1 = path.resolve(process.cwd(), "./folder/file1.ts"); + const file2 = path.resolve(process.cwd(), "./folder/file2.ts"); + + const test1 = mkTest_({ id: "0", title: "test1", file: file1, parent: root1 }); + const test2 = mkTest_({ id: "1", title: "test2", file: file2, parent: root2 }); + + const collection = TestCollection.create({ + bro1: [test1], + bro2: [test1, test2], + }); + + const result = format(collection); + + assert.deepEqual(result, [ + { + id: "0", + titlePath: ["root1", "test1"], + browserIds: ["bro1", "bro2"], + file: "folder/file1.ts", + pending: false, + skipReason: "", + }, + { + id: "1", + titlePath: ["root2", "test2"], + browserIds: ["bro2"], + file: "folder/file2.ts", + pending: false, + skipReason: "", + }, + ]); + }); +}); diff --git a/test/src/test-collection/formatters/tree.ts b/test/src/test-collection/formatters/tree.ts new file mode 100644 index 000000000..f5b329b83 --- /dev/null +++ b/test/src/test-collection/formatters/tree.ts @@ -0,0 +1,351 @@ +import path from "node:path"; +import _ from "lodash"; + +import { TestCollection } from "../../../../src/test-collection"; +import { format } from "../../../../src/test-collection/formatters/tree"; +import { Test, Suite } from "../../../../src/test-reader/test-object"; + +type SuiteOpts = { + id: string; + title: string; + file: string; + parent: Suite; + root: boolean; + location: { + line: number; + column: number; + }; + pending: boolean; + skipReason: string; +}; + +type TestOpts = Omit & { + disabled: boolean; +}; + +describe("test-collection/formatters/tree", () => { + const mkSuite_ = (opts: Partial = { title: "default-suite-title" }): Suite => { + const paramNames = ["id", "title", "file", "location"]; + + const suite = new Suite(_.pick(opts, paramNames) as any); + for (const [key, value] of _.entries(_.omit(opts, paramNames))) { + _.set(suite, key, value); + } + + return suite; + }; + + const mkTest_ = (opts: Partial = { title: "default-test-title" }): Test => { + const paramNames = ["id", "title", "file", "location"]; + + const test = new Test(_.pick(opts, paramNames) as any); + for (const [key, value] of _.entries(_.omit(opts, paramNames))) { + _.set(test, key, value); + } + + return test; + }; + + it("should return empty array if all tests are disabled", () => { + const collection = TestCollection.create({ + bro1: [mkTest_({ disabled: true })], + bro2: [mkTest_({ disabled: true })], + }); + + const result = format(collection); + + assert.deepEqual(result, []); + }); + + describe("should return main test", () => { + it("in one browser", () => { + const rootSuite = mkSuite_({ id: "0", root: true }); + const testOpts: Partial = { + id: "1", + title: "test", + file: path.resolve(process.cwd(), "./folder/file.ts"), + location: { + line: 1, + column: 1, + }, + parent: rootSuite, + }; + + const collection = TestCollection.create({ + bro1: [mkTest_(testOpts)], + }); + + const result = format(collection); + + assert.deepEqual(result, [ + { + id: "1", + title: "test", + file: "folder/file.ts", + line: 1, + column: 1, + pending: false, + skipReason: "", + browserIds: ["bro1"], + }, + ]); + }); + + it("in few browsers", () => { + const rootSuite = mkSuite_({ id: "0", root: true }); + const testOpts: Partial = { + id: "1", + title: "test", + file: path.resolve(process.cwd(), "./folder/file.ts"), + location: { + line: 1, + column: 1, + }, + parent: rootSuite, + }; + + const collection = TestCollection.create({ + bro1: [mkTest_(testOpts)], + bro2: [mkTest_(testOpts)], + }); + + const result = format(collection); + + assert.deepEqual(result, [ + { + id: "1", + title: "test", + file: "folder/file.ts", + line: 1, + column: 1, + pending: false, + skipReason: "", + browserIds: ["bro1", "bro2"], + }, + ]); + }); + + it("in skipped state", () => { + const rootSuite = mkSuite_({ id: "0", root: true }); + const testOpts: Partial = { + id: "1", + title: "test", + file: path.resolve(process.cwd(), "./folder/file.ts"), + location: { + line: 1, + column: 1, + }, + parent: rootSuite, + pending: true, + skipReason: "flaky", + }; + + const collection = TestCollection.create({ + bro1: [mkTest_(testOpts)], + }); + + const result = format(collection); + + assert.deepEqual(result, [ + { + id: "1", + title: "test", + file: "folder/file.ts", + line: 1, + column: 1, + pending: true, + skipReason: "flaky", + browserIds: ["bro1"], + }, + ]); + }); + }); + + it("should return main tests with equal titles", () => { + const rootSuite = mkSuite_({ id: "0", root: true }); + const commonTestOpts: Partial = { + id: "1", + title: "test", + location: { + line: 1, + column: 1, + }, + parent: rootSuite, + }; + const testOpts1: Partial = { + ...commonTestOpts, + id: "1", + file: path.resolve(process.cwd(), "./folder/file1.ts"), + }; + const testOpts2: Partial = { + ...commonTestOpts, + id: "2", + file: path.resolve(process.cwd(), "./folder/file2.ts"), + }; + + const collection = TestCollection.create({ + bro1: [mkTest_(testOpts1)], + bro2: [mkTest_(testOpts2)], + }); + + const result = format(collection); + + assert.deepEqual(result, [ + { + id: "1", + title: "test", + file: "folder/file1.ts", + line: 1, + column: 1, + pending: false, + skipReason: "", + browserIds: ["bro1"], + }, + { + id: "2", + title: "test", + file: "folder/file2.ts", + line: 1, + column: 1, + pending: false, + skipReason: "", + browserIds: ["bro2"], + }, + ]); + }); + + it("should return main suite with one test", () => { + const rootSuite = mkSuite_({ id: "0", root: true }); + const suiteOpts: Partial = { + id: "1", + title: "suite", + file: path.resolve(process.cwd(), "./folder/file.ts"), + location: { + line: 1, + column: 1, + }, + parent: rootSuite, + }; + const suite = mkSuite_(suiteOpts); + + const testOpts: Partial = { + id: "2", + title: "test", + file: path.resolve(process.cwd(), "./folder/file.ts"), + location: { + line: 2, + column: 5, + }, + parent: suite, + }; + + const collection = TestCollection.create({ + bro1: [mkTest_(testOpts)], + }); + + const result = format(collection); + + assert.deepEqual(result, [ + { + id: "1", + title: "suite", + file: "folder/file.ts", + line: 1, + column: 1, + pending: false, + skipReason: "", + suites: [], + tests: [ + { + id: "2", + title: "test", + line: 2, + column: 5, + pending: false, + skipReason: "", + browserIds: ["bro1"], + }, + ], + }, + ]); + }); + + it("should return main suite with child suite", () => { + const rootSuite = mkSuite_({ id: "0", root: true }); + const mainSuiteOpts: Partial = { + id: "1", + title: "suite1", + file: path.resolve(process.cwd(), "./folder/file.ts"), + location: { + line: 1, + column: 1, + }, + parent: rootSuite, + }; + const mainSuite = mkSuite_(mainSuiteOpts); + + const childSuiteOpts: Partial = { + id: "2", + title: "suite2", + file: path.resolve(process.cwd(), "./folder/file.ts"), + location: { + line: 2, + column: 5, + }, + parent: mainSuite, + }; + const childSuite = mkSuite_(childSuiteOpts); + + const testOpts: Partial = { + id: "3", + title: "test", + file: path.resolve(process.cwd(), "./folder/file.ts"), + location: { + line: 3, + column: 9, + }, + parent: childSuite, + }; + + const collection = TestCollection.create({ + bro1: [mkTest_(testOpts)], + }); + + const result = format(collection); + + assert.deepEqual(result, [ + { + id: "1", + title: "suite1", + file: "folder/file.ts", + line: 1, + column: 1, + pending: false, + skipReason: "", + suites: [ + { + id: "2", + title: "suite2", + line: 2, + column: 5, + pending: false, + skipReason: "", + suites: [], + tests: [ + { + id: "3", + title: "test", + line: 3, + column: 9, + pending: false, + skipReason: "", + browserIds: ["bro1"], + }, + ], + }, + ], + tests: [], + }, + ]); + }); +}); diff --git a/test/src/test-collection.ts b/test/src/test-collection/index.ts similarity index 89% rename from test/src/test-collection.ts rename to test/src/test-collection/index.ts index 28bbcf803..6f9320da9 100644 --- a/test/src/test-collection.ts +++ b/test/src/test-collection/index.ts @@ -1,12 +1,17 @@ +import path from "node:path"; import _ from "lodash"; -import { TestCollection } from "src/test-collection"; -import { Test } from "src/test-reader/test-object"; +import proxyquire from "proxyquire"; +import sinon, { SinonStub } from "sinon"; -import type { Suite } from "src/test-reader/test-object/suite"; +import { TestCollection, Formatters, AVAILABLE_FORMATTERS } from "../../../src/test-collection"; +import { Test } from "../../../src/test-reader/test-object"; +import type { Suite } from "../../../src/test-reader/test-object/suite"; type TestAndBrowser = { test: Test; browser: string }; describe("test-collection", () => { + const sandbox = sinon.createSandbox(); + interface TestOpts { title: string; browserVersion: string; @@ -23,6 +28,8 @@ describe("test-collection", () => { return test; }; + afterEach(() => sandbox.restore()); + describe("getBrowsers", () => { it("should return browsers from passed specs", () => { const collection = TestCollection.create({ @@ -498,4 +505,52 @@ describe("test-collection", () => { assert.deepEqual(rootSuites, { bro1: root }); }); }); + + describe("format", () => { + it("should throw error if passed formatter is not supported", () => { + const collection = TestCollection.create({}); + + try { + collection.format("foo" as any); + } catch (e) { + assert.match( + (e as Error).message, + `"formatter" option must be one of: ${AVAILABLE_FORMATTERS.join(", ")}`, + ); + } + }); + + [Formatters.LIST, Formatters.TREE].forEach(formatterName => { + let formatterStub: SinonStub; + let TestCollectionStub: typeof TestCollection; + + beforeEach(() => { + formatterStub = sandbox.stub().returns([]); + + TestCollectionStub = proxyquire("src/test-collection", { + [path.resolve(process.cwd(), `src/test-collection/formatters/${formatterName}`)]: { + format: formatterStub, + }, + }).TestCollection; + }); + + describe(`${formatterName} formatter`, () => { + it("should call 'format' method", () => { + const testCollection = TestCollectionStub.create({}); + + testCollection.format(formatterName); + + assert.calledOnceWith(formatterStub, testCollection); + }); + + it("should return result", () => { + const testCollection = TestCollectionStub.create({}); + + const result = testCollection.format(formatterName); + + assert.deepEqual(result, []); + }); + }); + }); + }); }); diff --git a/test/src/test-reader/index.js b/test/src/test-reader/index.js index 05f03eedb..782f82a61 100644 --- a/test/src/test-reader/index.js +++ b/test/src/test-reader/index.js @@ -155,10 +155,11 @@ describe("test-reader", () => { const config = makeConfigStub(); const files = ["file1.js", "file2.js"]; SetCollection.prototype.getAllFiles.returns(files); + const runnableOpts = { saveLocations: true }; - await readTests_({ config }); + await readTests_({ config, opts: { runnableOpts } }); - assert.calledOnceWith(TestParser.prototype.loadFiles, files, config); + assert.calledOnceWith(TestParser.prototype.loadFiles, files, { config, runnableOpts }); }); it("should load files before parsing", async () => { diff --git a/test/src/test-reader/mocha-reader/index.js b/test/src/test-reader/mocha-reader/index.js index b3ab8a83a..e1cf28ba7 100644 --- a/test/src/test-reader/mocha-reader/index.js +++ b/test/src/test-reader/mocha-reader/index.js @@ -1,5 +1,6 @@ "use strict"; +const _ = require("lodash"); const { MochaEventBus } = require("src/test-reader/mocha-reader/mocha-event-bus"); const { TreeBuilderDecorator } = require("src/test-reader/mocha-reader/tree-builder-decorator"); const { TreeBuilder } = require("src/test-reader/tree-builder"); @@ -14,6 +15,8 @@ describe("test-reader/mocha-reader", () => { const sandbox = sinon.createSandbox(); let MochaConstructorStub; + let SourceMapSupportStub; + let getMethodsByInterfaceStub; let readFiles; const mkMochaSuiteStub_ = () => { @@ -36,8 +39,19 @@ describe("test-reader/mocha-reader", () => { MochaConstructorStub = sinon.stub().returns(mkMochaStub_()); MochaConstructorStub.Suite = Mocha.Suite; + SourceMapSupportStub = { + wrapCallSite: sinon.stub().returns({ + getLineNumber: () => 1, + getColumnNumber: () => 1, + }), + install: sinon.stub(), + }; + getMethodsByInterfaceStub = sinon.stub().returns({ suiteMethods: [], testMethods: [] }); + readFiles = proxyquire("src/test-reader/mocha-reader", { mocha: MochaConstructorStub, + "@cspotcode/source-map-support": SourceMapSupportStub, + "./utils": { getMethodsByInterface: getMethodsByInterfaceStub }, }).readFiles; sandbox.stub(MochaEventBus, "create").returns(Object.create(MochaEventBus.prototype)); @@ -206,6 +220,141 @@ describe("test-reader/mocha-reader", () => { }); }); + describe("add locations to runnables", () => { + const emitAddRunnable_ = (runnable, event) => { + MochaEventBus.create.lastCall.returnValue.emit(MochaEventBus.events[event], runnable); + }; + + it("should do nothing if 'saveLocations' is not enabled", async () => { + const globalCtx = { + describe: () => {}, + }; + + Mocha.prototype.loadFilesAsync.callsFake(() => { + MochaEventBus.create.lastCall.returnValue.emit( + MochaEventBus.events.EVENT_FILE_PRE_REQUIRE, + globalCtx, + ); + }); + + await readFiles_({ runnableOpts: { saveLocations: false } }); + globalCtx.describe(); + + assert.notCalled(SourceMapSupportStub.wrapCallSite); + }); + + it("should not throw if source-map-support is not installed", async () => { + readFiles = proxyquire("src/test-reader/mocha-reader", { + "@cspotcode/source-map-support": null, + }).readFiles; + + const globalCtx = { describe: _.noop }; + + Mocha.prototype.loadFilesAsync.callsFake(() => { + MochaEventBus.create.lastCall.returnValue.emit( + MochaEventBus.events.EVENT_FILE_PRE_REQUIRE, + globalCtx, + ); + }); + + await readFiles_({ runnableOpts: { saveLocations: true } }); + + assert.doesNotThrow(() => globalCtx.describe()); + }); + + it("should set 'hookRequire' option on install source-map-support", async () => { + await readFiles_({ config: { ui: "bdd" }, runnableOpts: { saveLocations: true } }); + + assert.calledOnceWith(SourceMapSupportStub.install, { hookRequire: true }); + }); + + ["describe", "describe.only", "describe.skip", "xdescribe"].forEach(methodName => { + it(`should add location to suite using "${methodName}"`, async () => { + getMethodsByInterfaceStub.withArgs("bdd").returns({ suiteMethods: ["describe"], testMethods: [] }); + const suite = {}; + const globalCtx = _.set({}, methodName, () => emitAddRunnable_(suite, "EVENT_SUITE_ADD_SUITE")); + + Mocha.prototype.loadFilesAsync.callsFake(() => { + MochaEventBus.create.lastCall.returnValue.emit( + MochaEventBus.events.EVENT_FILE_PRE_REQUIRE, + globalCtx, + ); + }); + + SourceMapSupportStub.wrapCallSite.returns({ + getLineNumber: () => 100, + getColumnNumber: () => 500, + }); + + await readFiles_({ config: { ui: "bdd" }, runnableOpts: { saveLocations: true } }); + _.get(globalCtx, methodName)(); + + assert.deepEqual(suite, { location: { line: 100, column: 500 } }); + }); + }); + + ["it", "it.only", "it.skip", "xit"].forEach(methodName => { + it(`should add location to test using "${methodName}"`, async () => { + getMethodsByInterfaceStub.withArgs("bdd").returns({ suiteMethods: [], testMethods: ["it"] }); + const test = {}; + const globalCtx = _.set({}, methodName, () => emitAddRunnable_(test, "EVENT_SUITE_ADD_TEST")); + + Mocha.prototype.loadFilesAsync.callsFake(() => { + MochaEventBus.create.lastCall.returnValue.emit( + MochaEventBus.events.EVENT_FILE_PRE_REQUIRE, + globalCtx, + ); + }); + + SourceMapSupportStub.wrapCallSite.returns({ + getLineNumber: () => 500, + getColumnNumber: () => 100, + }); + + await readFiles_({ config: { ui: "bdd" }, runnableOpts: { saveLocations: true } }); + _.get(globalCtx, methodName)(); + + assert.deepEqual(test, { location: { line: 500, column: 100 } }); + }); + }); + + it(`should add location to each runnable`, async () => { + getMethodsByInterfaceStub.withArgs("bdd").returns({ suiteMethods: ["describe"], testMethods: ["it"] }); + const suite = {}; + const test = {}; + const globalCtx = { + describe: () => emitAddRunnable_(suite, "EVENT_SUITE_ADD_SUITE"), + it: () => emitAddRunnable_(test, "EVENT_SUITE_ADD_TEST"), + }; + + Mocha.prototype.loadFilesAsync.callsFake(() => { + MochaEventBus.create.lastCall.returnValue.emit( + MochaEventBus.events.EVENT_FILE_PRE_REQUIRE, + globalCtx, + ); + }); + + SourceMapSupportStub.wrapCallSite + .onFirstCall() + .returns({ + getLineNumber: () => 111, + getColumnNumber: () => 222, + }) + .onSecondCall() + .returns({ + getLineNumber: () => 333, + getColumnNumber: () => 444, + }); + + await readFiles_({ config: { ui: "bdd" }, runnableOpts: { saveLocations: true } }); + globalCtx.describe(); + globalCtx.it(); + + assert.deepEqual(suite, { location: { line: 111, column: 222 } }); + assert.deepEqual(test, { location: { line: 333, column: 444 } }); + }); + }); + describe("test objects", () => { [ ["EVENT_SUITE_ADD_SUITE", "addSuite"], diff --git a/test/src/test-reader/test-parser.js b/test/src/test-reader/test-parser.js index f901190e2..72be40843 100644 --- a/test/src/test-reader/test-parser.js +++ b/test/src/test-reader/test-parser.js @@ -53,11 +53,11 @@ describe("test-reader/test-parser", () => { }); describe("loadFiles", () => { - const loadFiles_ = async ({ parser, files, config } = {}) => { + const loadFiles_ = async ({ parser, files, config, runnableOpts } = {}) => { parser = parser || new TestParser(); config = config || makeConfigStub(); - return parser.loadFiles(files || [], config); + return parser.loadFiles(files || [], { config, runnableOpts }); }; describe("globals", () => { @@ -413,6 +413,14 @@ describe("test-reader/test-parser", () => { assert.calledWithMatch(readFiles, sinon.match.any, { eventBus: sinon.match.instanceOf(EventEmitter) }); }); + it("should pass runnable options to reader", async () => { + const runnableOpts = { saveLocations: true }; + + await loadFiles_({ runnableOpts }); + + assert.calledWithMatch(readFiles, sinon.match.any, { runnableOpts }); + }); + describe("esm decorator", () => { it("should be passed to mocha reader", async () => { await loadFiles_(); @@ -546,7 +554,7 @@ describe("test-reader/test-parser", () => { }); const parser = new TestParser(); - await parser.loadFiles([], loadFilesConfig); + await parser.loadFiles([], { config: loadFilesConfig }); return parser.parse(files || [], { browserId, config, grep }); }; diff --git a/test/src/test-reader/test-transformer.ts b/test/src/test-reader/test-transformer.ts index ddc278eec..ed39e0bd1 100644 --- a/test/src/test-reader/test-transformer.ts +++ b/test/src/test-reader/test-transformer.ts @@ -77,7 +77,9 @@ describe("test-transformer", () => { expectedCode.push("", `require("some${extName}");`); } - assert.equal(transformedCode, expectedCode.join("\n")); + expectedCode.push("//# sourceMappingURL="); + + assert.match(transformedCode, expectedCode.join("\n")); }); }); }); diff --git a/test/src/testplane.js b/test/src/testplane.js index 1cd157ed3..6c7f2d5fd 100644 --- a/test/src/testplane.js +++ b/test/src/testplane.js @@ -639,6 +639,9 @@ describe("testplane", () => { sets: ["s1", "s2"], grep: "grep", replMode: { enabled: false }, + runnableOpts: { + saveLocations: true, + }, }); assert.calledOnceWith(TestReader.prototype.read, { @@ -648,6 +651,9 @@ describe("testplane", () => { sets: ["s1", "s2"], grep: "grep", replMode: { enabled: false }, + runnableOpts: { + saveLocations: true, + }, }); }); diff --git a/test/src/worker/browser-env/runner/test-runner/index.ts b/test/src/worker/browser-env/runner/test-runner/index.ts index 022d0d259..b136f4b17 100644 --- a/test/src/worker/browser-env/runner/test-runner/index.ts +++ b/test/src/worker/browser-env/runner/test-runner/index.ts @@ -67,6 +67,7 @@ describe("worker/browser-env/runner/test-runner", () => { file: "/default/file/path", id: "12345", fn: sinon.stub(), + location: undefined, }) as TestType; test.parent = Suite.create({ id: "67890", title: "", file: test.file }); diff --git a/test/src/worker/runner/simple-test-parser.js b/test/src/worker/runner/simple-test-parser.js index 6c7350a70..0cc07ea47 100644 --- a/test/src/worker/runner/simple-test-parser.js +++ b/test/src/worker/runner/simple-test-parser.js @@ -43,7 +43,7 @@ describe("worker/runner/simple-test-parser", () => { await simpleParser.parse({ file: "some/file.js" }); - assert.calledOnceWith(TestParser.prototype.loadFiles, ["some/file.js"], config); + assert.calledOnceWith(TestParser.prototype.loadFiles, ["some/file.js"], { config }); }); it("should load file before parse", async () => {