From 3c6c2212dda1a46e9405d74adac0f6ad94364537 Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Thu, 26 Oct 2023 16:26:01 +0200 Subject: [PATCH 01/31] fix(workers): improve events --- docs/internal-api.md | 27 +++++++++++++++++++++++++++ lib/event.js | 2 ++ lib/workers.js | 17 ++++++++++++++++- test/unit/worker_test.js | 4 ---- 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/docs/internal-api.md b/docs/internal-api.md index 5e63ddd96..d5c202eaa 100644 --- a/docs/internal-api.md +++ b/docs/internal-api.md @@ -96,6 +96,32 @@ module.exports = function() { }); } ``` +You could get test stats when running with workers + +```js +const { event } = require('codeceptjs'); + +module.exports = function() { + + event.dispatcher.on(event.workers.result, function (result) { + + console.log(result); + + }); +} + +// in console log +FAIL | 7 passed, 1 failed, 1 skipped // 2s +{ + passes: 7, + failures: 1, + tests: 9, + pending: 1, + start: 2023-10-26T13:15:57.685Z, + end: 2023-10-26T13:15:59.880Z, + duration: 2195 +} +``` Available events: @@ -119,6 +145,7 @@ Available events: * `event.all.result` - when results are printed * `event.workers.before` - before spawning workers in parallel run * `event.workers.after` - after workers finished in parallel run +* `event.workers.result` - test results after workers finished in parallel run > *sync* - means that event is fired in the moment of the action happening. diff --git a/lib/event.js b/lib/event.js index 0ba8393dc..676354be7 100644 --- a/lib/event.js +++ b/lib/event.js @@ -127,10 +127,12 @@ module.exports = { * @inner * @property {'workers.before'} before * @property {'workers.after'} after + * @property {'workers.result'} result */ workers: { before: 'workers.before', after: 'workers.after', + result: 'workers.result', }, /** diff --git a/lib/workers.js b/lib/workers.js index e36a1509c..25086978f 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -399,44 +399,57 @@ class Workers extends EventEmitter { this._appendStats(message.data.stats); break; case event.suite.before: + event.dispatcher.emit(event.suite.before, repackTest(message.data)); this.emit(event.suite.before, repackTest(message.data)); break; case event.hook.failed: + event.dispatcher.emit(event.hook.failed, repackTest(message.data)); this.emit(event.hook.failed, repackTest(message.data)); this.errors.push(message.data.err); break; case event.test.before: + event.dispatcher.emit(event.test.before, repackTest(message.data)); this.emit(event.test.before, repackTest(message.data)); break; case event.test.started: + event.dispatcher.emit(event.test.started, repackTest(message.data)); this.emit(event.test.started, repackTest(message.data)); break; case event.test.failed: + event.dispatcher.emit(event.test.failed, repackTest(message.data)); this.emit(event.test.failed, repackTest(message.data)); break; case event.test.passed: + event.dispatcher.emit(event.test.passed, repackTest(message.data)); this.emit(event.test.passed, repackTest(message.data)); break; case event.test.skipped: + event.dispatcher.emit(event.test.skipped, repackTest(message.data)); this.emit(event.test.skipped, repackTest(message.data)); break; case event.test.finished: + event.dispatcher.emit(event.test.finished, repackTest(message.data)); this.emit(event.test.finished, repackTest(message.data)); break; case event.test.after: + event.dispatcher.emit(event.test.after, repackTest(message.data)); this.emit(event.test.after, repackTest(message.data)); break; case event.step.finished: + event.dispatcher.emit(event.step.finished, message.data); this.emit(event.step.finished, message.data); break; case event.step.started: + event.dispatcher.emit(event.step.started, message.data); this.emit(event.step.started, message.data); break; case event.step.passed: + event.dispatcher.emit(event.step.passed, message.data); this.emit(event.step.passed, message.data); break; case event.step.failed: - this.emit(event.step.failed, message.data); + event.dispatcher.emit(event.step.failed, message.data); + this.emit(event.step.passed, message.data); break; } }); @@ -492,6 +505,8 @@ class Workers extends EventEmitter { } output.result(this.stats.passes, this.stats.failures, this.stats.pending, ms(this.stats.duration)); + // expose test stats after all workers finished their execution + event.dispatcher.emit(event.workers.result, this.stats); } } diff --git a/test/unit/worker_test.js b/test/unit/worker_test.js index a5f614075..c0ff252c0 100644 --- a/test/unit/worker_test.js +++ b/test/unit/worker_test.js @@ -9,10 +9,6 @@ describe('Workers', function () { global.codecept_dir = path.join(__dirname, '/../data/sandbox'); }); - beforeEach(function () { - this.timeout(40000); - }); - it('should run simple worker', (done) => { const workerConfig = { by: 'test', From 081e11ce5346b38fac445003ab0bcbf929b00987 Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Thu, 26 Oct 2023 16:27:42 +0200 Subject: [PATCH 02/31] fix(workers): improve events --- lib/workers.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/workers.js b/lib/workers.js index 25086978f..0bca9d735 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -440,15 +440,12 @@ class Workers extends EventEmitter { this.emit(event.step.finished, message.data); break; case event.step.started: - event.dispatcher.emit(event.step.started, message.data); this.emit(event.step.started, message.data); break; case event.step.passed: - event.dispatcher.emit(event.step.passed, message.data); this.emit(event.step.passed, message.data); break; case event.step.failed: - event.dispatcher.emit(event.step.failed, message.data); this.emit(event.step.passed, message.data); break; } From d072254fc32039881f3708fd7226934cc3353086 Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Fri, 27 Oct 2023 06:39:04 +0200 Subject: [PATCH 03/31] expose env var --- lib/workers.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/workers.js b/lib/workers.js index 0bca9d735..5473107c1 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -359,6 +359,8 @@ class Workers extends EventEmitter { this.stats.start = new Date(); recorder.startUnlessRunning(); event.dispatcher.emit(event.workers.before); + // expose this to manage events better when using run-workers command + process.env.RUN_WITH_WORKERS = 'true'; recorder.add('starting workers', () => { for (const worker of this.workers) { const workerThread = createWorker(worker); @@ -504,6 +506,8 @@ class Workers extends EventEmitter { output.result(this.stats.passes, this.stats.failures, this.stats.pending, ms(this.stats.duration)); // expose test stats after all workers finished their execution event.dispatcher.emit(event.workers.result, this.stats); + // reset this after the run-workers finished + process.env.RUN_WITH_WORKERS = 'false'; } } From b7430b75cc71e4c0e40388b6dd723dcc1b369c03 Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Fri, 27 Oct 2023 06:48:34 +0200 Subject: [PATCH 04/31] some improvements --- docs/internal-api.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/internal-api.md b/docs/internal-api.md index d5c202eaa..f136f69fe 100644 --- a/docs/internal-api.md +++ b/docs/internal-api.md @@ -123,6 +123,25 @@ FAIL | 7 passed, 1 failed, 1 skipped // 2s } ``` +CodeceptJS also expose the env var `process.env.RUN_WITH_WORKERS` when running tests with `run-workers` command so that you could handle the events better in your plugins/helpers + +```js +const { event } = require('codeceptjs'); + +module.exports = function() { + // this event would trigger the `_publishResultsToTestrail` when running `run-workers` command + event.dispatcher.on(event.workers.result, async () => { + await _publishResultsToTestrail(); + }); + + // this event would not trigger the `_publishResultsToTestrail` multiple times when running `run-workers` command + event.dispatcher.on(event.all.result, async () => { + // when running `run` command, this env var is undefined + if (!process.env.RUN_WITH_WORKERS) await _publishResultsToTestrail(); + }); +} +``` + Available events: * `event.test.before(test)` - *async* when `Before` hooks from helpers and from test is executed From 58514ae5dd3a169d142704dc8c4754d4394bf9e1 Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Sat, 28 Oct 2023 13:25:15 +0200 Subject: [PATCH 05/31] emit step events in workers --- lib/workers.js | 5 ++++- package.json | 3 ++- test/unit/worker_test.js | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/workers.js b/lib/workers.js index 5473107c1..afae6f6fd 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -442,13 +442,16 @@ class Workers extends EventEmitter { this.emit(event.step.finished, message.data); break; case event.step.started: + event.dispatcher.emit(event.step.started, message.data); this.emit(event.step.started, message.data); break; case event.step.passed: + event.dispatcher.emit(event.step.passed, message.data); this.emit(event.step.passed, message.data); break; case event.step.failed: - this.emit(event.step.passed, message.data); + event.dispatcher.emit(event.step.failed, message.data); + this.emit(event.step.failed, message.data); break; } }); diff --git a/package.json b/package.json index d92f860c0..637f1b538 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,8 @@ "lint": "eslint bin/ examples/ lib/ test/ translations/ runok.js", "lint-fix": "eslint bin/ examples/ lib/ test/ translations/ runok.js --fix", "docs": "./runok.js docs", - "test:unit": "mocha test/unit --recursive --timeout 10000", + "test:workers": "mocha test/unit/worker_test.js --timeout 10000", + "test:unit": "npm run test:workers && NON_WORKERS_TESTS=true mocha test/unit --recursive --timeout 10000", "test:runner": "mocha test/runner --recursive --timeout 10000", "test": "npm run test:unit && npm run test:runner", "test:appium-quick": "mocha test/helper/AppiumV2_test.js --grep 'quick'", diff --git a/test/unit/worker_test.js b/test/unit/worker_test.js index c0ff252c0..e8b609777 100644 --- a/test/unit/worker_test.js +++ b/test/unit/worker_test.js @@ -2,7 +2,7 @@ const { expect } = require('chai'); const path = require('path'); const { Workers, event, recorder } = require('../../lib/index'); -describe('Workers', function () { +(process.env.NON_WORKERS_TESTS ? describe.skip : describe)('Workers', function () { this.timeout(40000); before(() => { From 1245816e1db6be1094da20acda60a698ff12a63f Mon Sep 17 00:00:00 2001 From: KobeNguyenT <7845001+kobenguyent@users.noreply.github.com> Date: Sun, 29 Oct 2023 05:35:56 +0100 Subject: [PATCH 06/31] Update internal-api.md Co-authored-by: Michael Bodnarchuk --- docs/internal-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/internal-api.md b/docs/internal-api.md index f136f69fe..beebb8020 100644 --- a/docs/internal-api.md +++ b/docs/internal-api.md @@ -123,7 +123,7 @@ FAIL | 7 passed, 1 failed, 1 skipped // 2s } ``` -CodeceptJS also expose the env var `process.env.RUN_WITH_WORKERS` when running tests with `run-workers` command so that you could handle the events better in your plugins/helpers +CodeceptJS also exposes the env var `process.env.RUN_WITH_WORKERS` when running tests with `run-workers` command so that you could handle the events better in your plugins/helpers ```js const { event } = require('codeceptjs'); From bf70dfc7ffa4cc511aca6ac6d534fea3cc5930b2 Mon Sep 17 00:00:00 2001 From: KobeNguyenT <7845001+kobenguyent@users.noreply.github.com> Date: Sun, 29 Oct 2023 05:36:07 +0100 Subject: [PATCH 07/31] Update workers.js Co-authored-by: Michael Bodnarchuk --- lib/workers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/workers.js b/lib/workers.js index afae6f6fd..d1847a357 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -360,7 +360,7 @@ class Workers extends EventEmitter { recorder.startUnlessRunning(); event.dispatcher.emit(event.workers.before); // expose this to manage events better when using run-workers command - process.env.RUN_WITH_WORKERS = 'true'; + process.env.RUNS_WITH_WORKERS = 'true'; recorder.add('starting workers', () => { for (const worker of this.workers) { const workerThread = createWorker(worker); From f346f3da1a2191b2cd2de7ed6814fba4963b8e8e Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Sun, 29 Oct 2023 07:49:14 +0100 Subject: [PATCH 08/31] remove duplicate events --- lib/workers.js | 48 ++++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/lib/workers.js b/lib/workers.js index d1847a357..e5b61d4dc 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -76,7 +76,8 @@ const simplifyObject = (object) => { }; const repackTest = (test) => { - test = Object.assign(new Test(test.title || '', () => { }), test); + test = Object.assign(new Test(test.title || '', () => { + }), test); test.parent = Object.assign(new Suite(test.parent.title), test.parent); return test; }; @@ -245,6 +246,12 @@ class Workers extends EventEmitter { pending: 0, }; this.testGroups = []; + this.suiteArr = []; + this.passedTestArr = []; + this.failedTestArr = []; + this.skippedTestArr = []; + this.passedStepArr = []; + this.failedStepArr = []; createOutputDir(config.testConfig); if (numberOfWorkers) this._initWorkers(numberOfWorkers, config); @@ -359,7 +366,6 @@ class Workers extends EventEmitter { this.stats.start = new Date(); recorder.startUnlessRunning(); event.dispatcher.emit(event.workers.before); - // expose this to manage events better when using run-workers command process.env.RUNS_WITH_WORKERS = 'true'; recorder.add('starting workers', () => { for (const worker of this.workers) { @@ -401,56 +407,52 @@ class Workers extends EventEmitter { this._appendStats(message.data.stats); break; case event.suite.before: - event.dispatcher.emit(event.suite.before, repackTest(message.data)); + this.suiteArr.push(repackTest(message.data)); this.emit(event.suite.before, repackTest(message.data)); break; + case event.suite.after: + this.emit(event.suite.after, repackTest(message.data)); + break; case event.hook.failed: - event.dispatcher.emit(event.hook.failed, repackTest(message.data)); this.emit(event.hook.failed, repackTest(message.data)); this.errors.push(message.data.err); break; case event.test.before: - event.dispatcher.emit(event.test.before, repackTest(message.data)); this.emit(event.test.before, repackTest(message.data)); break; case event.test.started: - event.dispatcher.emit(event.test.started, repackTest(message.data)); this.emit(event.test.started, repackTest(message.data)); break; case event.test.failed: - event.dispatcher.emit(event.test.failed, repackTest(message.data)); + this.failedTestArr.push(repackTest(message.data)); this.emit(event.test.failed, repackTest(message.data)); break; case event.test.passed: - event.dispatcher.emit(event.test.passed, repackTest(message.data)); + this.passedTestArr.push(repackTest(message.data)); this.emit(event.test.passed, repackTest(message.data)); break; case event.test.skipped: - event.dispatcher.emit(event.test.skipped, repackTest(message.data)); + this.skippedTestArr.push(repackTest(message.data)); this.emit(event.test.skipped, repackTest(message.data)); break; case event.test.finished: - event.dispatcher.emit(event.test.finished, repackTest(message.data)); this.emit(event.test.finished, repackTest(message.data)); break; case event.test.after: - event.dispatcher.emit(event.test.after, repackTest(message.data)); this.emit(event.test.after, repackTest(message.data)); break; case event.step.finished: - event.dispatcher.emit(event.step.finished, message.data); this.emit(event.step.finished, message.data); break; case event.step.started: - event.dispatcher.emit(event.step.started, message.data); this.emit(event.step.started, message.data); break; case event.step.passed: - event.dispatcher.emit(event.step.passed, message.data); + this.passedStepArr.push(message.data); this.emit(event.step.passed, message.data); break; case event.step.failed: - event.dispatcher.emit(event.step.failed, message.data); + this.failedStepArr.push(message.data); this.emit(event.step.failed, message.data); break; } @@ -508,9 +510,19 @@ class Workers extends EventEmitter { output.result(this.stats.passes, this.stats.failures, this.stats.pending, ms(this.stats.duration)); // expose test stats after all workers finished their execution - event.dispatcher.emit(event.workers.result, this.stats); - // reset this after the run-workers finished - process.env.RUN_WITH_WORKERS = 'false'; + event.dispatcher.emit(event.workers.result, { + suites: this.suiteArr, + tests: { + passed: this.passedTestArr, + failed: this.failedTestArr, + skipped: this.skippedTestArr, + }, + steps: { + passed: this.passedStepArr, + failed: this.failedStepArr, + }, + }); + process.env.RUNS_WITH_WORKERS = 'false'; } } From 4436a9174611ca855bcbd13302e0822e0dd7c3ee Mon Sep 17 00:00:00 2001 From: KobeNguyenT <7845001+kobenguyent@users.noreply.github.com> Date: Sun, 29 Oct 2023 08:50:19 +0100 Subject: [PATCH 09/31] Update internal-api.md --- docs/internal-api.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/internal-api.md b/docs/internal-api.md index beebb8020..1d0eca5bf 100644 --- a/docs/internal-api.md +++ b/docs/internal-api.md @@ -112,14 +112,18 @@ module.exports = function() { // in console log FAIL | 7 passed, 1 failed, 1 skipped // 2s +2s { - passes: 7, - failures: 1, - tests: 9, - pending: 1, - start: 2023-10-26T13:15:57.685Z, - end: 2023-10-26T13:15:59.880Z, - duration: 2195 + suites: [...], + tests: { + passed: [...], + failed: [...], + skipped: [...], + }, + steps: { + passed: [...], + failed: [...], + }, } ``` From 24baf510e6f1367cd6e37384c885ec860f97975e Mon Sep 17 00:00:00 2001 From: KobeNguyenT <7845001+kobenguyent@users.noreply.github.com> Date: Sun, 29 Oct 2023 08:53:16 +0100 Subject: [PATCH 10/31] Update package.json --- package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package.json b/package.json index 637f1b538..d92f860c0 100644 --- a/package.json +++ b/package.json @@ -40,8 +40,7 @@ "lint": "eslint bin/ examples/ lib/ test/ translations/ runok.js", "lint-fix": "eslint bin/ examples/ lib/ test/ translations/ runok.js --fix", "docs": "./runok.js docs", - "test:workers": "mocha test/unit/worker_test.js --timeout 10000", - "test:unit": "npm run test:workers && NON_WORKERS_TESTS=true mocha test/unit --recursive --timeout 10000", + "test:unit": "mocha test/unit --recursive --timeout 10000", "test:runner": "mocha test/runner --recursive --timeout 10000", "test": "npm run test:unit && npm run test:runner", "test:appium-quick": "mocha test/helper/AppiumV2_test.js --grep 'quick'", From 62ea8e8397ff862adb9c49b77cd533916e66490f Mon Sep 17 00:00:00 2001 From: KobeNguyenT <7845001+kobenguyent@users.noreply.github.com> Date: Sun, 29 Oct 2023 08:54:01 +0100 Subject: [PATCH 11/31] Update worker_test.js --- test/unit/worker_test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/worker_test.js b/test/unit/worker_test.js index e8b609777..c0ff252c0 100644 --- a/test/unit/worker_test.js +++ b/test/unit/worker_test.js @@ -2,7 +2,7 @@ const { expect } = require('chai'); const path = require('path'); const { Workers, event, recorder } = require('../../lib/index'); -(process.env.NON_WORKERS_TESTS ? describe.skip : describe)('Workers', function () { +describe('Workers', function () { this.timeout(40000); before(() => { From 9ed686d47e476d1ac3abd976f3f96fa7ac01c720 Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Mon, 30 Oct 2023 07:15:05 +0100 Subject: [PATCH 12/31] fix: expose test stats when run-workers finished --- lib/command/run-workers.js | 54 +++++++++++++++++++++++++++++++++++--- lib/workers.js | 31 +--------------------- 2 files changed, 51 insertions(+), 34 deletions(-) diff --git a/lib/command/run-workers.js b/lib/command/run-workers.js index 20fa44f51..8d9d4cc14 100644 --- a/lib/command/run-workers.js +++ b/lib/command/run-workers.js @@ -7,6 +7,14 @@ const Workers = require('../workers'); module.exports = async function (workerCount, selectedRuns, options) { process.env.profile = options.profile; + const suiteArr = []; + const passedTestArr = []; + const failedTestArr = []; + const skippedTestArr = []; + const passedStepArr = []; + const failedStepArr = []; + const skippedStepArr = []; + const { config: testConfig, override = '' } = options; const overrideConfigs = tryOrDefault(() => JSON.parse(override), {}); const by = options.suites ? 'suite' : 'test'; @@ -26,15 +34,53 @@ module.exports = async function (workerCount, selectedRuns, options) { const workers = new Workers(numberOfWorkers, config); workers.overrideConfig(overrideConfigs); - workers.on(event.test.failed, (failedTest) => { - output.test.failed(failedTest); + + workers.on(event.suite.before, (suite) => { + suiteArr.push(suite); + }); + + workers.on(event.test.failed, (test) => { + failedTestArr.push(test); + output.test.failed(test); + }); + + workers.on(event.test.passed, (test) => { + passedTestArr.push(test); + output.test.passed(test); + }); + + workers.on(event.test.skipped, (test) => { + skippedTestArr.push(test); + output.test.passed(test); + }); + + workers.on(event.step.failed, (step) => { + failedStepArr.push(step); + }); + + workers.on(event.step.passed, (step) => { + passedStepArr.push(step); }); - workers.on(event.test.passed, (successTest) => { - output.test.passed(successTest); + workers.on(event.step.passed, (step) => { + skippedStepArr.push(step); }); workers.on(event.all.result, () => { + // expose test stats after all workers finished their execution + event.dispatcher.emit(event.workers.result, { + suites: suiteArr, + tests: { + passed: passedTestArr, + failed: failedTestArr, + skipped: skippedTestArr, + }, + steps: { + passed: passedStepArr, + failed: failedStepArr, + skipped: skippedStepArr, + }, + }); workers.printResults(); }); diff --git a/lib/workers.js b/lib/workers.js index e5b61d4dc..78c38a3bf 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -76,8 +76,7 @@ const simplifyObject = (object) => { }; const repackTest = (test) => { - test = Object.assign(new Test(test.title || '', () => { - }), test); + test = Object.assign(new Test(test.title || '', () => { }), test); test.parent = Object.assign(new Suite(test.parent.title), test.parent); return test; }; @@ -246,12 +245,6 @@ class Workers extends EventEmitter { pending: 0, }; this.testGroups = []; - this.suiteArr = []; - this.passedTestArr = []; - this.failedTestArr = []; - this.skippedTestArr = []; - this.passedStepArr = []; - this.failedStepArr = []; createOutputDir(config.testConfig); if (numberOfWorkers) this._initWorkers(numberOfWorkers, config); @@ -407,12 +400,8 @@ class Workers extends EventEmitter { this._appendStats(message.data.stats); break; case event.suite.before: - this.suiteArr.push(repackTest(message.data)); this.emit(event.suite.before, repackTest(message.data)); break; - case event.suite.after: - this.emit(event.suite.after, repackTest(message.data)); - break; case event.hook.failed: this.emit(event.hook.failed, repackTest(message.data)); this.errors.push(message.data.err); @@ -424,15 +413,12 @@ class Workers extends EventEmitter { this.emit(event.test.started, repackTest(message.data)); break; case event.test.failed: - this.failedTestArr.push(repackTest(message.data)); this.emit(event.test.failed, repackTest(message.data)); break; case event.test.passed: - this.passedTestArr.push(repackTest(message.data)); this.emit(event.test.passed, repackTest(message.data)); break; case event.test.skipped: - this.skippedTestArr.push(repackTest(message.data)); this.emit(event.test.skipped, repackTest(message.data)); break; case event.test.finished: @@ -448,11 +434,9 @@ class Workers extends EventEmitter { this.emit(event.step.started, message.data); break; case event.step.passed: - this.passedStepArr.push(message.data); this.emit(event.step.passed, message.data); break; case event.step.failed: - this.failedStepArr.push(message.data); this.emit(event.step.failed, message.data); break; } @@ -509,19 +493,6 @@ class Workers extends EventEmitter { } output.result(this.stats.passes, this.stats.failures, this.stats.pending, ms(this.stats.duration)); - // expose test stats after all workers finished their execution - event.dispatcher.emit(event.workers.result, { - suites: this.suiteArr, - tests: { - passed: this.passedTestArr, - failed: this.failedTestArr, - skipped: this.skippedTestArr, - }, - steps: { - passed: this.passedStepArr, - failed: this.failedStepArr, - }, - }); process.env.RUNS_WITH_WORKERS = 'false'; } } From 00d385bcd8c127a6e04b2013d63512025dbe05d8 Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Tue, 31 Oct 2023 14:44:11 +0100 Subject: [PATCH 13/31] fix: attach step object to test object --- lib/command/run-workers.js | 20 -------------------- lib/command/workers/runTests.js | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/lib/command/run-workers.js b/lib/command/run-workers.js index 8d9d4cc14..24dbb2269 100644 --- a/lib/command/run-workers.js +++ b/lib/command/run-workers.js @@ -11,9 +11,6 @@ module.exports = async function (workerCount, selectedRuns, options) { const passedTestArr = []; const failedTestArr = []; const skippedTestArr = []; - const passedStepArr = []; - const failedStepArr = []; - const skippedStepArr = []; const { config: testConfig, override = '' } = options; const overrideConfigs = tryOrDefault(() => JSON.parse(override), {}); @@ -54,18 +51,6 @@ module.exports = async function (workerCount, selectedRuns, options) { output.test.passed(test); }); - workers.on(event.step.failed, (step) => { - failedStepArr.push(step); - }); - - workers.on(event.step.passed, (step) => { - passedStepArr.push(step); - }); - - workers.on(event.step.passed, (step) => { - skippedStepArr.push(step); - }); - workers.on(event.all.result, () => { // expose test stats after all workers finished their execution event.dispatcher.emit(event.workers.result, { @@ -75,11 +60,6 @@ module.exports = async function (workerCount, selectedRuns, options) { failed: failedTestArr, skipped: skippedTestArr, }, - steps: { - passed: passedStepArr, - failed: failedStepArr, - skipped: skippedStepArr, - }, }); workers.printResults(); }); diff --git a/lib/command/workers/runTests.js b/lib/command/workers/runTests.js index e6e220058..6456dd1f7 100644 --- a/lib/command/workers/runTests.js +++ b/lib/command/workers/runTests.js @@ -132,9 +132,31 @@ function initializeListeners() { duration: test.duration || 0, err, parent, + steps: test.steps ? simplifyStepsInTestObject(test.steps) : [], }; } + function simplifyStepsInTestObject(steps) { + steps = [...steps]; + const _steps = []; + + for (step of steps) { + _steps.push({ + actor: step.actor, + name: step.name, + status: step.status, + agrs: step.args, + startedAt: step.startedAt, + startTime: step.startTime, + endTime: step.endTime, + finishedAt: step.finishedAt, + duration: step.duration, + }); + } + + return _steps; + } + function simplifyStep(step, err = null) { step = { ...step }; From c86f5ef00a5fe4e6a67d3c1bcd5f8969cf38e953 Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Tue, 7 Nov 2023 09:57:05 +0100 Subject: [PATCH 14/31] resolve conflict --- docs/internal-api.md | 2 -- lib/workers.js | 15 ++++++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/internal-api.md b/docs/internal-api.md index 1d0eca5bf..453499eb9 100644 --- a/docs/internal-api.md +++ b/docs/internal-api.md @@ -112,9 +112,7 @@ module.exports = function() { // in console log FAIL | 7 passed, 1 failed, 1 skipped // 2s -2s { - suites: [...], tests: { passed: [...], failed: [...], diff --git a/lib/workers.js b/lib/workers.js index 78c38a3bf..f64646ce6 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -400,44 +400,57 @@ class Workers extends EventEmitter { this._appendStats(message.data.stats); break; case event.suite.before: + event.dispatcher.emit(event.suite.before, repackTest(message.data)); this.emit(event.suite.before, repackTest(message.data)); break; case event.hook.failed: + event.dispatcher.emit(event.hook.failed, repackTest(message.data)); this.emit(event.hook.failed, repackTest(message.data)); this.errors.push(message.data.err); break; case event.test.before: + event.dispatcher.emit(event.test.before, repackTest(message.data)); this.emit(event.test.before, repackTest(message.data)); break; case event.test.started: + event.dispatcher.emit(event.test.started, repackTest(message.data)); this.emit(event.test.started, repackTest(message.data)); break; case event.test.failed: + event.dispatcher.emit(event.test.failed, repackTest(message.data)); this.emit(event.test.failed, repackTest(message.data)); break; case event.test.passed: + event.dispatcher.emit(event.test.passed, repackTest(message.data)); this.emit(event.test.passed, repackTest(message.data)); break; case event.test.skipped: + event.dispatcher.emit(event.test.skipped, repackTest(message.data)); this.emit(event.test.skipped, repackTest(message.data)); break; case event.test.finished: + event.dispatcher.emit(event.test.finished, repackTest(message.data)); this.emit(event.test.finished, repackTest(message.data)); break; case event.test.after: + event.dispatcher.emit(event.test.after, repackTest(message.data)); this.emit(event.test.after, repackTest(message.data)); break; case event.step.finished: + event.dispatcher.emit(event.step.finished, message.data); this.emit(event.step.finished, message.data); break; case event.step.started: + event.dispatcher.emit(event.step.started, message.data); this.emit(event.step.started, message.data); break; case event.step.passed: + event.dispatcher.emit(event.step.passed, message.data); this.emit(event.step.passed, message.data); break; case event.step.failed: - this.emit(event.step.failed, message.data); + event.dispatcher.emit(event.step.failed, message.data); + this.emit(event.step.passed, message.data); break; } }); From 7df2d9fcb201e3dc2827ed51ec931023e15d9a00 Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Thu, 26 Oct 2023 16:27:42 +0200 Subject: [PATCH 15/31] fix(workers): improve events --- lib/workers.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/workers.js b/lib/workers.js index f64646ce6..e017580e4 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -441,15 +441,12 @@ class Workers extends EventEmitter { this.emit(event.step.finished, message.data); break; case event.step.started: - event.dispatcher.emit(event.step.started, message.data); this.emit(event.step.started, message.data); break; case event.step.passed: - event.dispatcher.emit(event.step.passed, message.data); this.emit(event.step.passed, message.data); break; case event.step.failed: - event.dispatcher.emit(event.step.failed, message.data); this.emit(event.step.passed, message.data); break; } From a5796057eaae3fcee33505a365148cddd5b86ff1 Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Tue, 7 Nov 2023 09:58:31 +0100 Subject: [PATCH 16/31] resolve conflict --- lib/workers.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/workers.js b/lib/workers.js index e017580e4..889d910bc 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -360,6 +360,7 @@ class Workers extends EventEmitter { recorder.startUnlessRunning(); event.dispatcher.emit(event.workers.before); process.env.RUNS_WITH_WORKERS = 'true'; + recorder.add('starting workers', () => { for (const worker of this.workers) { const workerThread = createWorker(worker); From 61457c885d9baee1f6d6c6b63d2c9276f2138bf5 Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Fri, 27 Oct 2023 06:48:34 +0200 Subject: [PATCH 17/31] some improvements --- docs/internal-api.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/internal-api.md b/docs/internal-api.md index 453499eb9..5c87bde86 100644 --- a/docs/internal-api.md +++ b/docs/internal-api.md @@ -144,6 +144,25 @@ module.exports = function() { } ``` +CodeceptJS also expose the env var `process.env.RUN_WITH_WORKERS` when running tests with `run-workers` command so that you could handle the events better in your plugins/helpers + +```js +const { event } = require('codeceptjs'); + +module.exports = function() { + // this event would trigger the `_publishResultsToTestrail` when running `run-workers` command + event.dispatcher.on(event.workers.result, async () => { + await _publishResultsToTestrail(); + }); + + // this event would not trigger the `_publishResultsToTestrail` multiple times when running `run-workers` command + event.dispatcher.on(event.all.result, async () => { + // when running `run` command, this env var is undefined + if (!process.env.RUN_WITH_WORKERS) await _publishResultsToTestrail(); + }); +} +``` + Available events: * `event.test.before(test)` - *async* when `Before` hooks from helpers and from test is executed From c5713b6b71ca76d7ec631f470fb7441ad1c2d9cf Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Sat, 28 Oct 2023 13:25:15 +0200 Subject: [PATCH 18/31] emit step events in workers --- lib/workers.js | 5 ++++- package.json | 3 ++- test/unit/worker_test.js | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/workers.js b/lib/workers.js index 889d910bc..9376bbacb 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -442,13 +442,16 @@ class Workers extends EventEmitter { this.emit(event.step.finished, message.data); break; case event.step.started: + event.dispatcher.emit(event.step.started, message.data); this.emit(event.step.started, message.data); break; case event.step.passed: + event.dispatcher.emit(event.step.passed, message.data); this.emit(event.step.passed, message.data); break; case event.step.failed: - this.emit(event.step.passed, message.data); + event.dispatcher.emit(event.step.failed, message.data); + this.emit(event.step.failed, message.data); break; } }); diff --git a/package.json b/package.json index d92f860c0..637f1b538 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,8 @@ "lint": "eslint bin/ examples/ lib/ test/ translations/ runok.js", "lint-fix": "eslint bin/ examples/ lib/ test/ translations/ runok.js --fix", "docs": "./runok.js docs", - "test:unit": "mocha test/unit --recursive --timeout 10000", + "test:workers": "mocha test/unit/worker_test.js --timeout 10000", + "test:unit": "npm run test:workers && NON_WORKERS_TESTS=true mocha test/unit --recursive --timeout 10000", "test:runner": "mocha test/runner --recursive --timeout 10000", "test": "npm run test:unit && npm run test:runner", "test:appium-quick": "mocha test/helper/AppiumV2_test.js --grep 'quick'", diff --git a/test/unit/worker_test.js b/test/unit/worker_test.js index c0ff252c0..e8b609777 100644 --- a/test/unit/worker_test.js +++ b/test/unit/worker_test.js @@ -2,7 +2,7 @@ const { expect } = require('chai'); const path = require('path'); const { Workers, event, recorder } = require('../../lib/index'); -describe('Workers', function () { +(process.env.NON_WORKERS_TESTS ? describe.skip : describe)('Workers', function () { this.timeout(40000); before(() => { From 45c37cc379dad7b62dbeb2f5d083d42822517c31 Mon Sep 17 00:00:00 2001 From: KobeNguyenT <7845001+kobenguyent@users.noreply.github.com> Date: Sun, 29 Oct 2023 05:35:56 +0100 Subject: [PATCH 19/31] Update internal-api.md Co-authored-by: Michael Bodnarchuk --- docs/internal-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/internal-api.md b/docs/internal-api.md index 5c87bde86..de1f14550 100644 --- a/docs/internal-api.md +++ b/docs/internal-api.md @@ -144,7 +144,7 @@ module.exports = function() { } ``` -CodeceptJS also expose the env var `process.env.RUN_WITH_WORKERS` when running tests with `run-workers` command so that you could handle the events better in your plugins/helpers +CodeceptJS also exposes the env var `process.env.RUN_WITH_WORKERS` when running tests with `run-workers` command so that you could handle the events better in your plugins/helpers ```js const { event } = require('codeceptjs'); From c9391a22bbc2fb195af2e831ac715a916beeda3c Mon Sep 17 00:00:00 2001 From: KobeNguyenT <7845001+kobenguyent@users.noreply.github.com> Date: Sun, 29 Oct 2023 05:36:07 +0100 Subject: [PATCH 20/31] Update workers.js Co-authored-by: Michael Bodnarchuk --- lib/workers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/workers.js b/lib/workers.js index 9376bbacb..65945f3d3 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -359,8 +359,8 @@ class Workers extends EventEmitter { this.stats.start = new Date(); recorder.startUnlessRunning(); event.dispatcher.emit(event.workers.before); + // expose this to manage events better when using run-workers command process.env.RUNS_WITH_WORKERS = 'true'; - recorder.add('starting workers', () => { for (const worker of this.workers) { const workerThread = createWorker(worker); From 26f8b8d89339d95eee89c0330c4fce5e01d0782e Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Sun, 29 Oct 2023 07:49:14 +0100 Subject: [PATCH 21/31] remove duplicate events --- lib/workers.js | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/lib/workers.js b/lib/workers.js index 65945f3d3..2aeafbe10 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -76,7 +76,8 @@ const simplifyObject = (object) => { }; const repackTest = (test) => { - test = Object.assign(new Test(test.title || '', () => { }), test); + test = Object.assign(new Test(test.title || '', () => { + }), test); test.parent = Object.assign(new Suite(test.parent.title), test.parent); return test; }; @@ -245,6 +246,12 @@ class Workers extends EventEmitter { pending: 0, }; this.testGroups = []; + this.suiteArr = []; + this.passedTestArr = []; + this.failedTestArr = []; + this.skippedTestArr = []; + this.passedStepArr = []; + this.failedStepArr = []; createOutputDir(config.testConfig); if (numberOfWorkers) this._initWorkers(numberOfWorkers, config); @@ -359,7 +366,6 @@ class Workers extends EventEmitter { this.stats.start = new Date(); recorder.startUnlessRunning(); event.dispatcher.emit(event.workers.before); - // expose this to manage events better when using run-workers command process.env.RUNS_WITH_WORKERS = 'true'; recorder.add('starting workers', () => { for (const worker of this.workers) { @@ -401,56 +407,52 @@ class Workers extends EventEmitter { this._appendStats(message.data.stats); break; case event.suite.before: - event.dispatcher.emit(event.suite.before, repackTest(message.data)); + this.suiteArr.push(repackTest(message.data)); this.emit(event.suite.before, repackTest(message.data)); break; + case event.suite.after: + this.emit(event.suite.after, repackTest(message.data)); + break; case event.hook.failed: - event.dispatcher.emit(event.hook.failed, repackTest(message.data)); this.emit(event.hook.failed, repackTest(message.data)); this.errors.push(message.data.err); break; case event.test.before: - event.dispatcher.emit(event.test.before, repackTest(message.data)); this.emit(event.test.before, repackTest(message.data)); break; case event.test.started: - event.dispatcher.emit(event.test.started, repackTest(message.data)); this.emit(event.test.started, repackTest(message.data)); break; case event.test.failed: - event.dispatcher.emit(event.test.failed, repackTest(message.data)); + this.failedTestArr.push(repackTest(message.data)); this.emit(event.test.failed, repackTest(message.data)); break; case event.test.passed: - event.dispatcher.emit(event.test.passed, repackTest(message.data)); + this.passedTestArr.push(repackTest(message.data)); this.emit(event.test.passed, repackTest(message.data)); break; case event.test.skipped: - event.dispatcher.emit(event.test.skipped, repackTest(message.data)); + this.skippedTestArr.push(repackTest(message.data)); this.emit(event.test.skipped, repackTest(message.data)); break; case event.test.finished: - event.dispatcher.emit(event.test.finished, repackTest(message.data)); this.emit(event.test.finished, repackTest(message.data)); break; case event.test.after: - event.dispatcher.emit(event.test.after, repackTest(message.data)); this.emit(event.test.after, repackTest(message.data)); break; case event.step.finished: - event.dispatcher.emit(event.step.finished, message.data); this.emit(event.step.finished, message.data); break; case event.step.started: - event.dispatcher.emit(event.step.started, message.data); this.emit(event.step.started, message.data); break; case event.step.passed: - event.dispatcher.emit(event.step.passed, message.data); + this.passedStepArr.push(message.data); this.emit(event.step.passed, message.data); break; case event.step.failed: - event.dispatcher.emit(event.step.failed, message.data); + this.failedStepArr.push(message.data); this.emit(event.step.failed, message.data); break; } From 18a93e9ee987d88525e6075e024b5b6237c3e546 Mon Sep 17 00:00:00 2001 From: KobeNguyenT <7845001+kobenguyent@users.noreply.github.com> Date: Sun, 29 Oct 2023 08:53:16 +0100 Subject: [PATCH 22/31] Update package.json --- package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package.json b/package.json index 637f1b538..d92f860c0 100644 --- a/package.json +++ b/package.json @@ -40,8 +40,7 @@ "lint": "eslint bin/ examples/ lib/ test/ translations/ runok.js", "lint-fix": "eslint bin/ examples/ lib/ test/ translations/ runok.js --fix", "docs": "./runok.js docs", - "test:workers": "mocha test/unit/worker_test.js --timeout 10000", - "test:unit": "npm run test:workers && NON_WORKERS_TESTS=true mocha test/unit --recursive --timeout 10000", + "test:unit": "mocha test/unit --recursive --timeout 10000", "test:runner": "mocha test/runner --recursive --timeout 10000", "test": "npm run test:unit && npm run test:runner", "test:appium-quick": "mocha test/helper/AppiumV2_test.js --grep 'quick'", From a727708802cf47e1ffb8b017f7e6f840502e211e Mon Sep 17 00:00:00 2001 From: KobeNguyenT <7845001+kobenguyent@users.noreply.github.com> Date: Sun, 29 Oct 2023 08:54:01 +0100 Subject: [PATCH 23/31] Update worker_test.js --- test/unit/worker_test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/worker_test.js b/test/unit/worker_test.js index e8b609777..c0ff252c0 100644 --- a/test/unit/worker_test.js +++ b/test/unit/worker_test.js @@ -2,7 +2,7 @@ const { expect } = require('chai'); const path = require('path'); const { Workers, event, recorder } = require('../../lib/index'); -(process.env.NON_WORKERS_TESTS ? describe.skip : describe)('Workers', function () { +describe('Workers', function () { this.timeout(40000); before(() => { From ee47cd2b413611ff27a0220fb2d40db740a3f8ff Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Mon, 30 Oct 2023 07:15:05 +0100 Subject: [PATCH 24/31] fix: expose test stats when run-workers finished --- lib/command/run-workers.js | 29 +++++++++++++++++++++++++++++ lib/workers.js | 18 +----------------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/lib/command/run-workers.js b/lib/command/run-workers.js index 24dbb2269..949fd0806 100644 --- a/lib/command/run-workers.js +++ b/lib/command/run-workers.js @@ -11,6 +11,12 @@ module.exports = async function (workerCount, selectedRuns, options) { const passedTestArr = []; const failedTestArr = []; const skippedTestArr = []; +<<<<<<< HEAD +======= + const passedStepArr = []; + const failedStepArr = []; + const skippedStepArr = []; +>>>>>>> ecb95487 (fix: expose test stats when run-workers finished) const { config: testConfig, override = '' } = options; const overrideConfigs = tryOrDefault(() => JSON.parse(override), {}); @@ -49,6 +55,21 @@ module.exports = async function (workerCount, selectedRuns, options) { workers.on(event.test.skipped, (test) => { skippedTestArr.push(test); output.test.passed(test); +<<<<<<< HEAD +======= + }); + + workers.on(event.step.failed, (step) => { + failedStepArr.push(step); + }); + + workers.on(event.step.passed, (step) => { + passedStepArr.push(step); + }); + + workers.on(event.step.passed, (step) => { + skippedStepArr.push(step); +>>>>>>> ecb95487 (fix: expose test stats when run-workers finished) }); workers.on(event.all.result, () => { @@ -60,6 +81,14 @@ module.exports = async function (workerCount, selectedRuns, options) { failed: failedTestArr, skipped: skippedTestArr, }, +<<<<<<< HEAD +======= + steps: { + passed: passedStepArr, + failed: failedStepArr, + skipped: skippedStepArr, + }, +>>>>>>> ecb95487 (fix: expose test stats when run-workers finished) }); workers.printResults(); }); diff --git a/lib/workers.js b/lib/workers.js index 2aeafbe10..78c38a3bf 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -76,8 +76,7 @@ const simplifyObject = (object) => { }; const repackTest = (test) => { - test = Object.assign(new Test(test.title || '', () => { - }), test); + test = Object.assign(new Test(test.title || '', () => { }), test); test.parent = Object.assign(new Suite(test.parent.title), test.parent); return test; }; @@ -246,12 +245,6 @@ class Workers extends EventEmitter { pending: 0, }; this.testGroups = []; - this.suiteArr = []; - this.passedTestArr = []; - this.failedTestArr = []; - this.skippedTestArr = []; - this.passedStepArr = []; - this.failedStepArr = []; createOutputDir(config.testConfig); if (numberOfWorkers) this._initWorkers(numberOfWorkers, config); @@ -407,12 +400,8 @@ class Workers extends EventEmitter { this._appendStats(message.data.stats); break; case event.suite.before: - this.suiteArr.push(repackTest(message.data)); this.emit(event.suite.before, repackTest(message.data)); break; - case event.suite.after: - this.emit(event.suite.after, repackTest(message.data)); - break; case event.hook.failed: this.emit(event.hook.failed, repackTest(message.data)); this.errors.push(message.data.err); @@ -424,15 +413,12 @@ class Workers extends EventEmitter { this.emit(event.test.started, repackTest(message.data)); break; case event.test.failed: - this.failedTestArr.push(repackTest(message.data)); this.emit(event.test.failed, repackTest(message.data)); break; case event.test.passed: - this.passedTestArr.push(repackTest(message.data)); this.emit(event.test.passed, repackTest(message.data)); break; case event.test.skipped: - this.skippedTestArr.push(repackTest(message.data)); this.emit(event.test.skipped, repackTest(message.data)); break; case event.test.finished: @@ -448,11 +434,9 @@ class Workers extends EventEmitter { this.emit(event.step.started, message.data); break; case event.step.passed: - this.passedStepArr.push(message.data); this.emit(event.step.passed, message.data); break; case event.step.failed: - this.failedStepArr.push(message.data); this.emit(event.step.failed, message.data); break; } From a8d153be823819d471fd1f2bf15c1ec382453855 Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Tue, 31 Oct 2023 14:44:11 +0100 Subject: [PATCH 25/31] fix: attach step object to test object --- lib/command/run-workers.js | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/lib/command/run-workers.js b/lib/command/run-workers.js index 949fd0806..c5321980d 100644 --- a/lib/command/run-workers.js +++ b/lib/command/run-workers.js @@ -11,12 +11,7 @@ module.exports = async function (workerCount, selectedRuns, options) { const passedTestArr = []; const failedTestArr = []; const skippedTestArr = []; -<<<<<<< HEAD -======= const passedStepArr = []; - const failedStepArr = []; - const skippedStepArr = []; ->>>>>>> ecb95487 (fix: expose test stats when run-workers finished) const { config: testConfig, override = '' } = options; const overrideConfigs = tryOrDefault(() => JSON.parse(override), {}); @@ -55,21 +50,6 @@ module.exports = async function (workerCount, selectedRuns, options) { workers.on(event.test.skipped, (test) => { skippedTestArr.push(test); output.test.passed(test); -<<<<<<< HEAD -======= - }); - - workers.on(event.step.failed, (step) => { - failedStepArr.push(step); - }); - - workers.on(event.step.passed, (step) => { - passedStepArr.push(step); - }); - - workers.on(event.step.passed, (step) => { - skippedStepArr.push(step); ->>>>>>> ecb95487 (fix: expose test stats when run-workers finished) }); workers.on(event.all.result, () => { @@ -81,14 +61,6 @@ module.exports = async function (workerCount, selectedRuns, options) { failed: failedTestArr, skipped: skippedTestArr, }, -<<<<<<< HEAD -======= - steps: { - passed: passedStepArr, - failed: failedStepArr, - skipped: skippedStepArr, - }, ->>>>>>> ecb95487 (fix: expose test stats when run-workers finished) }); workers.printResults(); }); From 082ba9361572b69f0f64f0abfd14a322ab747e82 Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Tue, 31 Oct 2023 14:47:57 +0100 Subject: [PATCH 26/31] fix: conflict --- lib/command/run-workers.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/command/run-workers.js b/lib/command/run-workers.js index c5321980d..24dbb2269 100644 --- a/lib/command/run-workers.js +++ b/lib/command/run-workers.js @@ -11,7 +11,6 @@ module.exports = async function (workerCount, selectedRuns, options) { const passedTestArr = []; const failedTestArr = []; const skippedTestArr = []; - const passedStepArr = []; const { config: testConfig, override = '' } = options; const overrideConfigs = tryOrDefault(() => JSON.parse(override), {}); From c78345d987e0f88b98f566271783c3173dce2c7b Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Tue, 31 Oct 2023 15:04:42 +0100 Subject: [PATCH 27/31] fix: update docs --- docs/internal-api.md | 80 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 72 insertions(+), 8 deletions(-) diff --git a/docs/internal-api.md b/docs/internal-api.md index de1f14550..d304cc338 100644 --- a/docs/internal-api.md +++ b/docs/internal-api.md @@ -113,15 +113,79 @@ module.exports = function() { // in console log FAIL | 7 passed, 1 failed, 1 skipped // 2s { - tests: { - passed: [...], - failed: [...], - skipped: [...], - }, - steps: { - passed: [...], - failed: [...], + "tests": { + "passed": [ + { + "type": "test", + "title": "Assert @C3", + "body": "() => { }", + "async": 0, + "sync": true, + "_timeout": 2000, + "_slow": 75, + "_retries": -1, + "timedOut": false, + "_currentRetry": 0, + "pending": false, + "opts": {}, + "tags": [ + "@C3" + ], + "uid": "xe4q1HdqpRrZG5dPe0JG+A", + "workerIndex": 3, + "retries": -1, + "duration": 493, + "err": null, + "parent": { + "title": "My", + "ctx": {}, + "suites": [], + "tests": [], + "root": false, + "pending": false, + "_retries": -1, + "_beforeEach": [], + "_beforeAll": [], + "_afterEach": [], + "_afterAll": [], + "_timeout": 2000, + "_slow": 75, + "_bail": false, + "_onlyTests": [], + "_onlySuites": [], + "delayed": false }, + "steps": [ + { + "actor": "I", + "name": "amOnPage", + "status": "success", + "agrs": [ + "https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/POST" + ], + "startedAt": 1698760652610, + "startTime": 1698760652611, + "endTime": 1698760653098, + "finishedAt": 1698760653098, + "duration": 488 + }, + { + "actor": "I", + "name": "grabCurrentUrl", + "status": "success", + "agrs": [], + "startedAt": 1698760653098, + "startTime": 1698760653098, + "endTime": 1698760653099, + "finishedAt": 1698760653099, + "duration": 1 + } + ] + } + ], + "failed": [], + "skipped": [] +} } ``` From 8c2212b485575bcd65c1213be1ca6aae2ec985df Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Tue, 31 Oct 2023 17:25:11 +0100 Subject: [PATCH 28/31] fix: add err object to step --- lib/command/workers/runTests.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/command/workers/runTests.js b/lib/command/workers/runTests.js index 6456dd1f7..430a7b450 100644 --- a/lib/command/workers/runTests.js +++ b/lib/command/workers/runTests.js @@ -132,11 +132,11 @@ function initializeListeners() { duration: test.duration || 0, err, parent, - steps: test.steps ? simplifyStepsInTestObject(test.steps) : [], + steps: test.steps ? simplifyStepsInTestObject(test.steps, err) : [], }; } - function simplifyStepsInTestObject(steps) { + function simplifyStepsInTestObject(steps, err) { steps = [...steps]; const _steps = []; @@ -151,6 +151,7 @@ function initializeListeners() { endTime: step.endTime, finishedAt: step.finishedAt, duration: step.duration, + err, }); } From 06fcdeda6ad53b0686bb244657c76a16f80d459b Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Tue, 7 Nov 2023 10:05:59 +0100 Subject: [PATCH 29/31] remove redundant --- docs/internal-api.md | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/docs/internal-api.md b/docs/internal-api.md index d304cc338..8e5c8d635 100644 --- a/docs/internal-api.md +++ b/docs/internal-api.md @@ -207,26 +207,6 @@ module.exports = function() { }); } ``` - -CodeceptJS also exposes the env var `process.env.RUN_WITH_WORKERS` when running tests with `run-workers` command so that you could handle the events better in your plugins/helpers - -```js -const { event } = require('codeceptjs'); - -module.exports = function() { - // this event would trigger the `_publishResultsToTestrail` when running `run-workers` command - event.dispatcher.on(event.workers.result, async () => { - await _publishResultsToTestrail(); - }); - - // this event would not trigger the `_publishResultsToTestrail` multiple times when running `run-workers` command - event.dispatcher.on(event.all.result, async () => { - // when running `run` command, this env var is undefined - if (!process.env.RUN_WITH_WORKERS) await _publishResultsToTestrail(); - }); -} -``` - Available events: * `event.test.before(test)` - *async* when `Before` hooks from helpers and from test is executed From b71b01f80749d40fafc8872027dd98a2f495d44b Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Tue, 7 Nov 2023 10:07:15 +0100 Subject: [PATCH 30/31] remove redundant --- docs/internal-api.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/internal-api.md b/docs/internal-api.md index 8e5c8d635..e3f607c5c 100644 --- a/docs/internal-api.md +++ b/docs/internal-api.md @@ -189,7 +189,7 @@ FAIL | 7 passed, 1 failed, 1 skipped // 2s } ``` -CodeceptJS also exposes the env var `process.env.RUN_WITH_WORKERS` when running tests with `run-workers` command so that you could handle the events better in your plugins/helpers +CodeceptJS also exposes the env var `process.env.RUNS_WITH_WORKERS` when running tests with `run-workers` command so that you could handle the events better in your plugins/helpers ```js const { event } = require('codeceptjs'); @@ -203,7 +203,7 @@ module.exports = function() { // this event would not trigger the `_publishResultsToTestrail` multiple times when running `run-workers` command event.dispatcher.on(event.all.result, async () => { // when running `run` command, this env var is undefined - if (!process.env.RUN_WITH_WORKERS) await _publishResultsToTestrail(); + if (!process.env.RUNS_WITH_WORKERS) await _publishResultsToTestrail(); }); } ``` From 0f284da3ce1b898af616848589b222b3afd68c6d Mon Sep 17 00:00:00 2001 From: kobenguyent Date: Tue, 5 Dec 2023 06:02:35 +0100 Subject: [PATCH 31/31] address CR --- docs/internal-api.md | 110 ---------------------------------------- docs/parallel.md | 116 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 112 deletions(-) diff --git a/docs/internal-api.md b/docs/internal-api.md index e3f607c5c..aa0846036 100644 --- a/docs/internal-api.md +++ b/docs/internal-api.md @@ -96,117 +96,7 @@ module.exports = function() { }); } ``` -You could get test stats when running with workers -```js -const { event } = require('codeceptjs'); - -module.exports = function() { - - event.dispatcher.on(event.workers.result, function (result) { - - console.log(result); - - }); -} - -// in console log -FAIL | 7 passed, 1 failed, 1 skipped // 2s -{ - "tests": { - "passed": [ - { - "type": "test", - "title": "Assert @C3", - "body": "() => { }", - "async": 0, - "sync": true, - "_timeout": 2000, - "_slow": 75, - "_retries": -1, - "timedOut": false, - "_currentRetry": 0, - "pending": false, - "opts": {}, - "tags": [ - "@C3" - ], - "uid": "xe4q1HdqpRrZG5dPe0JG+A", - "workerIndex": 3, - "retries": -1, - "duration": 493, - "err": null, - "parent": { - "title": "My", - "ctx": {}, - "suites": [], - "tests": [], - "root": false, - "pending": false, - "_retries": -1, - "_beforeEach": [], - "_beforeAll": [], - "_afterEach": [], - "_afterAll": [], - "_timeout": 2000, - "_slow": 75, - "_bail": false, - "_onlyTests": [], - "_onlySuites": [], - "delayed": false - }, - "steps": [ - { - "actor": "I", - "name": "amOnPage", - "status": "success", - "agrs": [ - "https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/POST" - ], - "startedAt": 1698760652610, - "startTime": 1698760652611, - "endTime": 1698760653098, - "finishedAt": 1698760653098, - "duration": 488 - }, - { - "actor": "I", - "name": "grabCurrentUrl", - "status": "success", - "agrs": [], - "startedAt": 1698760653098, - "startTime": 1698760653098, - "endTime": 1698760653099, - "finishedAt": 1698760653099, - "duration": 1 - } - ] - } - ], - "failed": [], - "skipped": [] -} -} -``` - -CodeceptJS also exposes the env var `process.env.RUNS_WITH_WORKERS` when running tests with `run-workers` command so that you could handle the events better in your plugins/helpers - -```js -const { event } = require('codeceptjs'); - -module.exports = function() { - // this event would trigger the `_publishResultsToTestrail` when running `run-workers` command - event.dispatcher.on(event.workers.result, async () => { - await _publishResultsToTestrail(); - }); - - // this event would not trigger the `_publishResultsToTestrail` multiple times when running `run-workers` command - event.dispatcher.on(event.all.result, async () => { - // when running `run` command, this env var is undefined - if (!process.env.RUNS_WITH_WORKERS) await _publishResultsToTestrail(); - }); -} -``` Available events: * `event.test.before(test)` - *async* when `Before` hooks from helpers and from test is executed diff --git a/docs/parallel.md b/docs/parallel.md index 148de0557..df0e8ac00 100644 --- a/docs/parallel.md +++ b/docs/parallel.md @@ -26,12 +26,124 @@ This command is similar to `run`, however, steps output can't be shown in worker Each worker spins an instance of CodeceptJS, executes a group of tests, and sends back report to the main process. -By default the tests are assigned one by one to the available workers this may lead to multiple execution of `BeforeSuite()`. Use the option `--suites` to assigne the suites one by one to the workers. +By default, the tests are assigned one by one to the available workers this may lead to multiple execution of `BeforeSuite()`. Use the option `--suites` to assign the suites one by one to the workers. ```sh npx codeceptjs run-workers --suites 2 ``` +## Test stats with Parallel Execution by Workers + +```js +const { event } = require('codeceptjs'); + +module.exports = function() { + + event.dispatcher.on(event.workers.result, function (result) { + + console.log(result); + + }); +} + +// in console log +FAIL | 7 passed, 1 failed, 1 skipped // 2s +{ + "tests": { + "passed": [ + { + "type": "test", + "title": "Assert @C3", + "body": "() => { }", + "async": 0, + "sync": true, + "_timeout": 2000, + "_slow": 75, + "_retries": -1, + "timedOut": false, + "_currentRetry": 0, + "pending": false, + "opts": {}, + "tags": [ + "@C3" + ], + "uid": "xe4q1HdqpRrZG5dPe0JG+A", + "workerIndex": 3, + "retries": -1, + "duration": 493, + "err": null, + "parent": { + "title": "My", + "ctx": {}, + "suites": [], + "tests": [], + "root": false, + "pending": false, + "_retries": -1, + "_beforeEach": [], + "_beforeAll": [], + "_afterEach": [], + "_afterAll": [], + "_timeout": 2000, + "_slow": 75, + "_bail": false, + "_onlyTests": [], + "_onlySuites": [], + "delayed": false + }, + "steps": [ + { + "actor": "I", + "name": "amOnPage", + "status": "success", + "agrs": [ + "https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/POST" + ], + "startedAt": 1698760652610, + "startTime": 1698760652611, + "endTime": 1698760653098, + "finishedAt": 1698760653098, + "duration": 488 + }, + { + "actor": "I", + "name": "grabCurrentUrl", + "status": "success", + "agrs": [], + "startedAt": 1698760653098, + "startTime": 1698760653098, + "endTime": 1698760653099, + "finishedAt": 1698760653099, + "duration": 1 + } + ] + } + ], + "failed": [], + "skipped": [] +} +} +``` + +CodeceptJS also exposes the env var `process.env.RUNS_WITH_WORKERS` when running tests with `run-workers` command so that you could handle the events better in your plugins/helpers + +```js +const { event } = require('codeceptjs'); + +module.exports = function() { + // this event would trigger the `_publishResultsToTestrail` when running `run-workers` command + event.dispatcher.on(event.workers.result, async () => { + await _publishResultsToTestrail(); + }); + + // this event would not trigger the `_publishResultsToTestrail` multiple times when running `run-workers` command + event.dispatcher.on(event.all.result, async () => { + // when running `run` command, this env var is undefined + if (!process.env.RUNS_WITH_WORKERS) await _publishResultsToTestrail(); + }); +} +``` + ## Parallel Execution by Workers on Multiple Browsers To run tests in parallel across multiple browsers, modify your `codecept.conf.js` file to configure multiple browsers on which you want to run your tests and your tests will run across multiple browsers. @@ -236,7 +348,7 @@ customWorkers.on(event.all.result, () => { ### Emitting messages to the parent worker -Child workers can send non test events to the main process. This is useful if you want to pass along information not related to the tests event cycles itself such as `event.test.success`. +Child workers can send non-test events to the main process. This is useful if you want to pass along information not related to the tests event cycles itself such as `event.test.success`. ```js // inside main process