diff --git a/README.md b/README.md index 3a7224b8d44..f8a761ca117 100644 --- a/README.md +++ b/README.md @@ -23,13 +23,15 @@ Most of the documentation for `dd-trace` is available on these webpages: ## Version Release Lines and Maintenance -| Release Line | Latest Version | Node.js | Status |Initial Release | End of Life | -| :---: | :---: | :---: | :---: | :---: | :---: | -| [`v1`](https://github.com/DataDog/dd-trace-js/tree/v1.x) | ![npm v1](https://img.shields.io/npm/v/dd-trace/legacy-v1?color=white&label=%20&style=flat-square) | `>= v12` | **End of Life** | 2021-07-13 | 2022-02-25 | -| [`v2`](https://github.com/DataDog/dd-trace-js/tree/v2.x) | ![npm v2](https://img.shields.io/npm/v/dd-trace/latest-node12?color=white&label=%20&style=flat-square) | `>= v12` | **End of Life** | 2022-01-28 | 2023-08-15 | -| [`v3`](https://github.com/DataDog/dd-trace-js/tree/v3.x) | ![npm v3](https://img.shields.io/npm/v/dd-trace/latest-node14?color=white&label=%20&style=flat-square) | `>= v14` | **End of Life** | 2022-08-15 | 2024-05-15 | -| [`v4`](https://github.com/DataDog/dd-trace-js/tree/v4.x) | ![npm v4](https://img.shields.io/npm/v/dd-trace/latest-node16?color=white&label=%20&style=flat-square) | `>= v16` | **Maintenance** | 2023-05-12 | 2025-01-11 | -| [`v5`](https://github.com/DataDog/dd-trace-js/tree/v5.x) | ![npm v5](https://img.shields.io/npm/v/dd-trace/latest?color=white&label=%20&style=flat-square) | `>= v18` | **Current** | 2024-01-11 | Unknown | +| Release Line | Latest Version | Node.js | [SSI](https://docs.datadoghq.com/tracing/trace_collection/automatic_instrumentation/single-step-apm/?tab=linuxhostorvm) | [K8s Injection](https://docs.datadoghq.com/tracing/trace_collection/library_injection_local/?tab=kubernetes) |Status |Initial Release | End of Life | +| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | +| [`v1`](https://github.com/DataDog/dd-trace-js/tree/v1.x) | ![npm v1](https://img.shields.io/npm/v/dd-trace/legacy-v1?color=white&label=%20&style=flat-square) | `>= v12` | NO | NO | **End of Life** | 2021-07-13 | 2022-02-25 | +| [`v2`](https://github.com/DataDog/dd-trace-js/tree/v2.x) | ![npm v2](https://img.shields.io/npm/v/dd-trace/latest-node12?color=white&label=%20&style=flat-square) | `>= v12` | NO | NO | **End of Life** | 2022-01-28 | 2023-08-15 | +| [`v3`](https://github.com/DataDog/dd-trace-js/tree/v3.x) | ![npm v3](https://img.shields.io/npm/v/dd-trace/latest-node14?color=white&label=%20&style=flat-square) | `>= v14` | NO | YES | **End of Life** | 2022-08-15 | 2024-05-15 | +| [`v4`](https://github.com/DataDog/dd-trace-js/tree/v4.x) | ![npm v4](https://img.shields.io/npm/v/dd-trace/latest-node16?color=white&label=%20&style=flat-square) | `>= v16` | YES | YES | **Maintenance** | 2023-05-12 | 2025-01-11 | +| [`v5`](https://github.com/DataDog/dd-trace-js/tree/v5.x) | ![npm v5](https://img.shields.io/npm/v/dd-trace/latest?color=white&label=%20&style=flat-square) | `>= v18` | YES | YES | **Current** | 2024-01-11 | Unknown | + +* SSI = Single-Step Install We currently maintain two release lines, namely `v5`, and `v4`. Features and bug fixes that are merged are released to the `v5` line and, if appropriate, also `v4`. diff --git a/integration-tests/cucumber/cucumber.spec.js b/integration-tests/cucumber/cucumber.spec.js index b46205fcb05..00102734b28 100644 --- a/integration-tests/cucumber/cucumber.spec.js +++ b/integration-tests/cucumber/cucumber.spec.js @@ -42,7 +42,8 @@ const { DI_DEBUG_ERROR_PREFIX, DI_DEBUG_ERROR_FILE_SUFFIX, DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX, - DI_DEBUG_ERROR_LINE_SUFFIX + DI_DEBUG_ERROR_LINE_SUFFIX, + TEST_RETRY_REASON } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') @@ -844,15 +845,13 @@ versions.forEach(version => { it('retries new tests', (done) => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new receiver.setKnownTests( @@ -884,6 +883,9 @@ versions.forEach(version => { retriedTests.length ) assert.equal(retriedTests.length, NUM_RETRIES_EFD) + retriedTests.forEach(test => { + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'efd') + }) // Test name does not change newTests.forEach(test => { assert.equal(test.meta[TEST_NAME], 'Say whatever') @@ -907,15 +909,13 @@ versions.forEach(version => { it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -928,8 +928,12 @@ versions.forEach(version => { const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true' ) - // new tests are not detected - assert.equal(newTests.length, 0) + // new tests are detected but not retried + assert.equal(newTests.length, 1) + const retriedTests = tests.filter(test => + test.meta[TEST_IS_RETRY] === 'true' + ) + assert.equal(retriedTests.length, 0) }) // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new receiver.setKnownTests({ @@ -957,15 +961,13 @@ versions.forEach(version => { it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) // Tests in "cucumber.ci-visibility/features-flaky/flaky.feature" will be considered new receiver.setKnownTests({}) @@ -1014,15 +1016,13 @@ versions.forEach(version => { it('does not retry tests that are skipped', (done) => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) // "cucumber.ci-visibility/features/farewell.feature.Say whatever" will be considered new // "cucumber.ci-visibility/features/greetings.feature.Say skip" will be considered new @@ -1066,15 +1066,13 @@ versions.forEach(version => { it('does not run EFD if the known tests request fails', (done) => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTestsResponseCode(500) receiver.setKnownTests({}) @@ -1108,16 +1106,14 @@ versions.forEach(version => { it('bails out of EFD if the percentage of new tests is too high', (done) => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 0 - } + }, + known_tests_enabled: true }) // tests in cucumber.ci-visibility/features/farewell.feature will be considered new receiver.setKnownTests( @@ -1160,20 +1156,70 @@ versions.forEach(version => { }) }) + it('disables early flake detection if known tests should not be requested', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + }, + known_tests_enabled: false + }) + // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new + receiver.setKnownTests( + { + cucumber: { + 'ci-visibility/features/farewell.feature': ['Say farewell'], + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] + } + } + ) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // no new tests detected + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) + // no retries + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + runTestsCommand, + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + if (version !== '7.0.0') { // EFD in parallel mode only supported from cucumber>=11 context('parallel mode', () => { it('retries new tests', (done) => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new receiver.setKnownTests( @@ -1231,15 +1277,13 @@ versions.forEach(version => { it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) // Tests in "cucumber.ci-visibility/features-flaky/flaky.feature" will be considered new receiver.setKnownTests({}) @@ -1293,16 +1337,14 @@ versions.forEach(version => { it('bails out of EFD if the percentage of new tests is too high', (done) => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 0 - } + }, + known_tests_enabled: true }) // tests in cucumber.ci-visibility/features/farewell.feature will be considered new receiver.setKnownTests( @@ -1350,15 +1392,13 @@ versions.forEach(version => { it('does not retry tests that are skipped', (done) => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) // "cucumber.ci-visibility/features/farewell.feature.Say whatever" will be considered new // "cucumber.ci-visibility/features/greetings.feature.Say skip" will be considered new @@ -1909,5 +1949,54 @@ versions.forEach(version => { }) }) }) + + context('known tests without early flake detection', () => { + it('detects new tests without retrying them', (done) => { + receiver.setSettings({ + early_flake_detection: { + enabled: false + }, + known_tests_enabled: true + }) + // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new + receiver.setKnownTests( + { + cucumber: { + 'ci-visibility/features/farewell.feature': ['Say farewell'], + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] + } + } + ) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // new tests detected but not retried + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 1) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + runTestsCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) }) }) diff --git a/integration-tests/cypress/cypress.spec.js b/integration-tests/cypress/cypress.spec.js index 0a6f5f065f9..d1fda8baa23 100644 --- a/integration-tests/cypress/cypress.spec.js +++ b/integration-tests/cypress/cypress.spec.js @@ -35,7 +35,8 @@ const { TEST_SUITE, TEST_CODE_OWNERS, TEST_SESSION_NAME, - TEST_LEVEL_EVENT_TYPES + TEST_LEVEL_EVENT_TYPES, + TEST_RETRY_REASON } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -1019,15 +1020,13 @@ moduleTypes.forEach(({ context('early flake detection', () => { it('retries new tests', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTests({ @@ -1051,6 +1050,10 @@ moduleTypes.forEach(({ const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') assert.equal(retriedTests.length, NUM_RETRIES_EFD) + retriedTests.forEach((retriedTest) => { + assert.equal(retriedTest.meta[TEST_RETRY_REASON], 'efd') + }) + newTests.forEach(newTest => { assert.equal(newTest.resource, 'cypress/e2e/spec.cy.js.context passes') }) @@ -1092,15 +1095,13 @@ moduleTypes.forEach(({ it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTests({ @@ -1123,8 +1124,12 @@ moduleTypes.forEach(({ const tests = events.filter(event => event.type === 'test').map(event => event.content) assert.equal(tests.length, 2) + // new tests are detected but not retried const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') - assert.equal(newTests.length, 0) + assert.equal(newTests.length, 1) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) const testSession = events.find(event => event.type === 'test_session_end').content assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) @@ -1154,15 +1159,13 @@ moduleTypes.forEach(({ it('does not retry tests that are skipped', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTests({}) @@ -1211,15 +1214,13 @@ moduleTypes.forEach(({ it('does not run EFD if the known tests request fails', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTestsResponseCode(500) @@ -1264,6 +1265,70 @@ moduleTypes.forEach(({ }).catch(done) }) }) + + it('disables early flake detection if known tests should not be requested', (done) => { + receiver.setSettings({ + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + }, + known_tests_enabled: false + }) + + receiver.setKnownTests({ + cypress: { + 'cypress/e2e/spec.cy.js': [ + // 'context passes', // This test will be considered new + 'other context fails' + ] + } + }) + + const { + NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress + ...restEnvVars + } = getCiVisEvpProxyConfig(receiver.port) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 2) + + // new tests are not detected + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + }) + + const specToRun = 'cypress/e2e/spec.cy.js' + childProcess = exec( + version === 'latest' ? testCommand : `${testCommand} --spec ${specToRun}`, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: specToRun, + DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => { + done() + }).catch(done) + }) + }) }) context('flaky test retries', () => { @@ -1511,5 +1576,65 @@ moduleTypes.forEach(({ }).catch(done) }) }) + + context('known tests without early flake detection', () => { + it('detects new tests without retrying them', (done) => { + receiver.setSettings({ + known_tests_enabled: true + }) + + receiver.setKnownTests({ + cypress: { + 'cypress/e2e/spec.cy.js': [ + // 'context passes', // This test will be considered new + 'other context fails' + ] + } + }) + + const { + NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress + ...restEnvVars + } = getCiVisEvpProxyConfig(receiver.port) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 2) + + // new tests are detected but not retried + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 1) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + }) + + const specToRun = 'cypress/e2e/spec.cy.js' + childProcess = exec( + version === 'latest' ? testCommand : `${testCommand} --spec ${specToRun}`, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: specToRun, + DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => { + done() + }).catch(done) + }) + }) + }) }) }) diff --git a/integration-tests/jest/jest.spec.js b/integration-tests/jest/jest.spec.js index ac604d96b5e..784ea393e5a 100644 --- a/integration-tests/jest/jest.spec.js +++ b/integration-tests/jest/jest.spec.js @@ -30,6 +30,7 @@ const { TEST_NAME, JEST_DISPLAY_NAME, TEST_EARLY_FLAKE_ABORT_REASON, + TEST_RETRY_REASON, TEST_SOURCE_START, TEST_CODE_OWNERS, TEST_SESSION_NAME, @@ -1609,16 +1610,14 @@ describe('jest CommonJS', () => { }) const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { @@ -1652,6 +1651,9 @@ describe('jest CommonJS', () => { retriedTests.length ) assert.equal(retriedTests.length, NUM_RETRIES_EFD) + retriedTests.forEach(test => { + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'efd') + }) // Test name does not change newTests.forEach(test => { assert.equal(test.meta[TEST_NAME], 'ci visibility 2 can report tests 2') @@ -1682,16 +1684,14 @@ describe('jest CommonJS', () => { } }) receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': 3 }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const parameterizedTestFile = 'test-parameterized.js' @@ -1757,16 +1757,14 @@ describe('jest CommonJS', () => { } }) receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': 3 }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1779,8 +1777,12 @@ describe('jest CommonJS', () => { const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true' ) - // new tests are not detected - assert.equal(newTests.length, 0) + // new tests are detected but not retried + assert.equal(newTests.length, 1) + const retriedTests = tests.filter(test => + test.meta[TEST_IS_RETRY] === 'true' + ) + assert.equal(retriedTests.length, 0) }) childProcess = exec( @@ -1809,16 +1811,14 @@ describe('jest CommonJS', () => { const NUM_RETRIES_EFD = 5 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1875,16 +1875,14 @@ describe('jest CommonJS', () => { const NUM_RETRIES_EFD = 5 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1931,16 +1929,14 @@ describe('jest CommonJS', () => { receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': 3 }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) // Tests from ci-visibility/test/skipped-and-todo-test will be considered new receiver.setKnownTests({ @@ -1999,16 +1995,14 @@ describe('jest CommonJS', () => { const NUM_RETRIES_EFD = 5 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -2051,16 +2045,14 @@ describe('jest CommonJS', () => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -2127,16 +2119,14 @@ describe('jest CommonJS', () => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -2183,16 +2173,14 @@ describe('jest CommonJS', () => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 1 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -2235,16 +2223,14 @@ describe('jest CommonJS', () => { }) const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -2301,6 +2287,66 @@ describe('jest CommonJS', () => { }).catch(done) }) }) + + it('disables early flake detection if known tests should not be requested', (done) => { + receiver.setSettings({ + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + } + }, + known_tests_enabled: false + }) + + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + jest: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const oldTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' + ) + oldTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + assert.equal(oldTests.length, 1) + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' + ) + newTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { ...getCiVisEvpProxyConfig(receiver.port), TESTS_TO_RUN: 'test/ci-visibility-test' }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) }) context('flaky test retries', () => { @@ -2797,4 +2843,66 @@ describe('jest CommonJS', () => { }) }) }) + + context('known tests without early flake detection', () => { + it('detects new tests without retrying them', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + jest: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + receiver.setSettings({ + early_flake_detection: { + enabled: false + }, + known_tests_enabled: true + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // no other tests are considered new + const oldTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' + ) + oldTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + assert.equal(oldTests.length, 1) + + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // no test has been retried + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { ...getCiVisEvpProxyConfig(receiver.port), TESTS_TO_RUN: 'test/ci-visibility-test' }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) }) diff --git a/integration-tests/mocha/mocha.spec.js b/integration-tests/mocha/mocha.spec.js index a7c23b067df..21e7670d077 100644 --- a/integration-tests/mocha/mocha.spec.js +++ b/integration-tests/mocha/mocha.spec.js @@ -40,7 +40,8 @@ const { DI_DEBUG_ERROR_PREFIX, DI_DEBUG_ERROR_FILE_SUFFIX, DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX, - DI_DEBUG_ERROR_LINE_SUFFIX + DI_DEBUG_ERROR_LINE_SUFFIX, + TEST_RETRY_REASON } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -1141,16 +1142,14 @@ describe('mocha CommonJS', function () { }) const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { @@ -1184,6 +1183,9 @@ describe('mocha CommonJS', function () { retriedTests.length ) assert.equal(retriedTests.length, NUM_RETRIES_EFD) + retriedTests.forEach(test => { + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'efd') + }) // Test name does not change newTests.forEach(test => { assert.equal(test.meta[TEST_NAME], 'ci visibility 2 can report tests 2') @@ -1220,16 +1222,14 @@ describe('mocha CommonJS', function () { } }) receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': 3 }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1298,16 +1298,14 @@ describe('mocha CommonJS', function () { } }) receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': 3 }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1320,8 +1318,12 @@ describe('mocha CommonJS', function () { const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true' ) - // new tests are not detected - assert.equal(newTests.length, 0) + // new tests are detected but not retried + assert.equal(newTests.length, 1) + const retriedTests = tests.filter(test => + test.meta[TEST_IS_RETRY] === 'true' + ) + assert.equal(retriedTests.length, 0) }) childProcess = exec( @@ -1339,6 +1341,7 @@ describe('mocha CommonJS', function () { stdio: 'inherit' } ) + childProcess.on('exit', () => { eventsPromise.then(() => { done() @@ -1352,16 +1355,14 @@ describe('mocha CommonJS', function () { const NUM_RETRIES_EFD = 5 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1421,16 +1422,14 @@ describe('mocha CommonJS', function () { const NUM_RETRIES_EFD = 5 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1472,16 +1471,14 @@ describe('mocha CommonJS', function () { it('handles spaces in test names', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': 3 }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) // Tests from ci-visibility/test/skipped-and-todo-test will be considered new receiver.setKnownTests({ @@ -1541,16 +1538,14 @@ describe('mocha CommonJS', function () { const NUM_RETRIES_EFD = 5 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1595,16 +1590,14 @@ describe('mocha CommonJS', function () { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1668,16 +1661,14 @@ describe('mocha CommonJS', function () { const NUM_RETRIES_EFD = 5 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 0 - } + }, + known_tests_enabled: true }) // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new receiver.setKnownTests({ @@ -1732,16 +1723,14 @@ describe('mocha CommonJS', function () { const NUM_RETRIES_EFD = 5 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1770,6 +1759,7 @@ describe('mocha CommonJS', function () { // Test name does not change retriedTests.forEach(test => { assert.equal(test.meta[TEST_NAME], 'fail occasionally fails') + assert.equal(test.meta[TEST_RETRY_REASON], 'efd') }) }) @@ -1787,22 +1777,21 @@ describe('mocha CommonJS', function () { }).catch(done) }) }) + it('retries new tests when using the programmatic API', (done) => { // Tests from ci-visibility/test/occasionally-failing-test will be considered new receiver.setKnownTests({}) const NUM_RETRIES_EFD = 5 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1855,20 +1844,19 @@ describe('mocha CommonJS', function () { }).catch(done) }) }) + it('bails out of EFD if the percentage of new tests is too high', (done) => { const NUM_RETRIES_EFD = 5 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 0 - } + }, + known_tests_enabled: true }) // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new receiver.setKnownTests({ @@ -1917,6 +1905,71 @@ describe('mocha CommonJS', function () { }) }) }) + + it('disables early flake detection if known tests should not be requested', (done) => { + receiver.setSettings({ + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + } + }, + known_tests_enabled: false + }) + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + mocha: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const oldTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' + ) + oldTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + assert.equal(oldTests.length, 1) + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' + ) + newTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test/ci-visibility-test.js', + './test/ci-visibility-test-2.js' + ]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) }) context('auto test retries', () => { @@ -2399,4 +2452,72 @@ describe('mocha CommonJS', function () { }) }) }) + + context('known tests without early flake detection', () => { + it('detects new tests without retrying them', (done) => { + receiver.setSettings({ + early_flake_detection: { + enabled: false + }, + known_tests_enabled: true + }) + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + mocha: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // no other tests are considered new + const oldTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' + ) + oldTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + assert.equal(oldTests.length, 1) + + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // no test has been retried + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test/ci-visibility-test.js', + './test/ci-visibility-test-2.js' + ]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) }) diff --git a/integration-tests/playwright/playwright.spec.js b/integration-tests/playwright/playwright.spec.js index 3f6a49e01b7..691a09b4d13 100644 --- a/integration-tests/playwright/playwright.spec.js +++ b/integration-tests/playwright/playwright.spec.js @@ -24,7 +24,8 @@ const { TEST_SUITE, TEST_CODE_OWNERS, TEST_SESSION_NAME, - TEST_LEVEL_EVENT_TYPES + TEST_LEVEL_EVENT_TYPES, + TEST_RETRY_REASON } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -252,15 +253,13 @@ versions.forEach((version) => { context('early flake detection', () => { it('retries new tests', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTests( @@ -303,6 +302,10 @@ versions.forEach((version) => { assert.equal(retriedTests.length, NUM_RETRIES_EFD) + retriedTests.forEach(test => { + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'efd') + }) + // all but one has been retried assert.equal(retriedTests.length, newTests.length - 1) }) @@ -326,15 +329,13 @@ versions.forEach((version) => { it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTests( @@ -366,12 +367,12 @@ versions.forEach((version) => { const newTests = tests.filter(test => test.resource.endsWith('should work with passing tests') ) + // new tests are detected but not retried newTests.forEach(test => { - assert.notProperty(test.meta, TEST_IS_NEW) + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') }) const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - assert.equal(retriedTests.length, 0) }) @@ -395,15 +396,13 @@ versions.forEach((version) => { it('does not retry tests that are skipped', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTests( @@ -467,15 +466,13 @@ versions.forEach((version) => { it('does not run EFD if the known tests request fails', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTestsResponseCode(500) @@ -515,6 +512,74 @@ versions.forEach((version) => { .catch(done) }) }) + + it('disables early flake detection if known tests should not be requested', (done) => { + receiver.setSettings({ + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + }, + known_tests_enabled: false + }) + + receiver.setKnownTests( + { + playwright: { + 'landing-page-test.js': [ + // it will be considered new + // 'highest-level-describe leading and trailing spaces should work with passing tests', + 'highest-level-describe leading and trailing spaces should work with skipped tests', + 'highest-level-describe leading and trailing spaces should work with fixme', + 'highest-level-describe leading and trailing spaces should work with annotated tests' + ], + 'skipped-suite-test.js': [ + 'should work with fixme root' + ], + 'todo-list-page-test.js': [ + 'playwright should work with failing tests', + 'should work with fixme root' + ] + } + } + ) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const newTests = tests.filter(test => + test.resource.endsWith('should work with passing tests') + ) + newTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + './node_modules/.bin/playwright test -c playwright.config.js', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PW_BASE_URL: `http://localhost:${webAppPort}` + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => done()).catch(done) + }) + }) }) } @@ -716,5 +781,72 @@ versions.forEach((version) => { }).catch(done) }) }) + + if (version === 'latest') { + context('known tests without early flake detection', () => { + it('detects new tests without retrying them', (done) => { + receiver.setSettings({ + known_tests_enabled: true + }) + + receiver.setKnownTests( + { + playwright: { + 'landing-page-test.js': [ + // it will be considered new + // 'highest-level-describe leading and trailing spaces should work with passing tests', + 'highest-level-describe leading and trailing spaces should work with skipped tests', + 'highest-level-describe leading and trailing spaces should work with fixme', + 'highest-level-describe leading and trailing spaces should work with annotated tests' + ], + 'skipped-suite-test.js': [ + 'should work with fixme root' + ], + 'todo-list-page-test.js': [ + 'playwright should work with failing tests', + 'should work with fixme root' + ] + } + } + ) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const newTests = tests.filter(test => + test.resource.endsWith('should work with passing tests') + ) + // new tests detected but no retries + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + './node_modules/.bin/playwright test -c playwright.config.js', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PW_BASE_URL: `http://localhost:${webAppPort}` + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => done()).catch(done) + }) + }) + }) + } }) }) diff --git a/integration-tests/vitest/vitest.spec.js b/integration-tests/vitest/vitest.spec.js index eb2fe21ba78..eb53b395202 100644 --- a/integration-tests/vitest/vitest.spec.js +++ b/integration-tests/vitest/vitest.spec.js @@ -29,7 +29,8 @@ const { DI_DEBUG_ERROR_PREFIX, DI_DEBUG_ERROR_FILE_SUFFIX, DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX, - DI_DEBUG_ERROR_LINE_SUFFIX + DI_DEBUG_ERROR_LINE_SUFFIX, + TEST_RETRY_REASON } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') @@ -421,15 +422,13 @@ versions.forEach((version) => { context('early flake detection', () => { it('retries new tests', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTests({ @@ -469,10 +468,15 @@ versions.forEach((version) => { 'early flake detection does not retry if the test is skipped' ]) const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') - assert.equal(newTests.length, 12) // 4 executions of the three new tests + // 4 executions of the 3 new tests + 1 new skipped test (not retried) + assert.equal(newTests.length, 13) const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - assert.equal(retriedTests.length, 9) // 3 retries of the three new tests + assert.equal(retriedTests.length, 9) // 3 retries of the 3 new tests + + retriedTests.forEach(test => { + assert.equal(test.meta[TEST_RETRY_REASON], 'efd') + }) // exit code should be 0 and test session should be reported as passed, // even though there are some failing executions @@ -507,15 +511,13 @@ versions.forEach((version) => { it('fails if all the attempts fail', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTests({ @@ -550,10 +552,11 @@ versions.forEach((version) => { 'early flake detection does not retry if the test is skipped' ]) const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') - assert.equal(newTests.length, 8) // 4 executions of the two new tests + // 4 executions of the 2 new tests + 1 new skipped test (not retried) + assert.equal(newTests.length, 9) const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - assert.equal(retriedTests.length, 6) // 3 retries of the two new tests + assert.equal(retriedTests.length, 6) // 3 retries of the 2 new tests // the multiple attempts did not result in a single pass, // so the test session should be reported as failed @@ -588,16 +591,14 @@ versions.forEach((version) => { it('bails out of EFD if the percentage of new tests is too high', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 0 - } + }, + known_tests_enabled: true }) receiver.setKnownTests({ @@ -628,9 +629,7 @@ versions.forEach((version) => { env: { ...getCiVisAgentlessConfig(receiver.port), TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*', - NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', - DD_TRACE_DEBUG: '1', - DD_TRACE_LOG_LEVEL: 'error' + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init' }, stdio: 'pipe' } @@ -646,15 +645,13 @@ versions.forEach((version) => { it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTests({ @@ -662,7 +659,7 @@ versions.forEach((version) => { 'ci-visibility/vitest-tests/early-flake-detection.mjs': [ // 'early flake detection can retry tests that eventually pass', // will be considered new // 'early flake detection can retry tests that always pass', // will be considered new - // 'early flake detection does not retry if the test is skipped', // skipped so not retried + // 'early flake detection does not retry if the test is skipped', // will be considered new 'early flake detection does not retry if it is not new' ] } @@ -682,8 +679,10 @@ versions.forEach((version) => { 'early flake detection does not retry if it is not new', 'early flake detection does not retry if the test is skipped' ]) + + // new tests are detected but not retried const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') - assert.equal(newTests.length, 0) + assert.equal(newTests.length, 3) const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') assert.equal(retriedTests.length, 0) @@ -718,15 +717,13 @@ versions.forEach((version) => { it('does not run EFD if the known tests request fails', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTestsResponseCode(500) @@ -781,15 +778,13 @@ versions.forEach((version) => { it('works when the cwd is not the repository root', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTests({ @@ -837,11 +832,21 @@ versions.forEach((version) => { it('works with repeats config when EFD is disabled', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: false + }, + known_tests_enabled: true + }) + + receiver.setKnownTests({ + vitest: { + 'ci-visibility/vitest-tests/early-flake-detection.mjs': [ + // 'early flake detection can retry tests that eventually pass', // will be considered new + // 'early flake detection can retry tests that always pass', // will be considered new + // 'early flake detection can retry tests that eventually fail', // will be considered new + // 'early flake detection does not retry if the test is skipped', // will be considered new + 'early flake detection does not retry if it is not new' + ] } }) @@ -864,13 +869,14 @@ versions.forEach((version) => { 'early flake detection does not retry if the test is skipped' ]) const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') - assert.equal(newTests.length, 0) // no new test detected + // all but one are considered new + assert.equal(newTests.length, 7) const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') assert.equal(retriedTests.length, 4) // 2 repetitions on 2 tests // vitest reports the test as failed if any of the repetitions fail, so we'll follow that - // TODO: we might want to improve htis + // TODO: we might want to improve this const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') assert.equal(failedTests.length, 3) @@ -900,6 +906,77 @@ versions.forEach((version) => { }).catch(done) }) }) + + it('disables early flake detection if known tests should not be requested', (done) => { + receiver.setSettings({ + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + }, + known_tests_enabled: false + }) + + receiver.setKnownTests({ + vitest: { + 'ci-visibility/vitest-tests/early-flake-detection.mjs': [ + // 'early flake detection can retry tests that eventually pass', // will be considered new + // 'early flake detection can retry tests that always pass', // will be considered new + // 'early flake detection does not retry if the test is skipped', // will be considered new + 'early flake detection does not retry if it is not new' + ] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(test => test.content) + + assert.equal(tests.length, 4) + + assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [ + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that always pass', + 'early flake detection does not retry if it is not new', + 'early flake detection does not retry if the test is skipped' + ]) + + // new tests are not detected and not retried + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 1) + const testSessionEvent = events.find(event => event.type === 'test_session_end').content + assert.equal(testSessionEvent.meta[TEST_STATUS], 'fail') + }) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 1) + done() + }).catch(done) + }) + }) }) // dynamic instrumentation only supported from >=2.0.0 @@ -1150,5 +1227,76 @@ versions.forEach((version) => { }) }) } + + context('known tests without early flake detection', () => { + it('detects new tests without retrying them', (done) => { + receiver.setSettings({ + early_flake_detection: { + enabled: false + }, + known_tests_enabled: true + }) + + receiver.setKnownTests({ + vitest: { + 'ci-visibility/vitest-tests/early-flake-detection.mjs': [ + // 'early flake detection can retry tests that eventually pass', // will be considered new + // 'early flake detection can retry tests that always pass', // will be considered new + // 'early flake detection does not retry if the test is skipped', // will be considered new + 'early flake detection does not retry if it is not new' + ] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(test => test.content) + + assert.equal(tests.length, 4) + + assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [ + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that always pass', + 'early flake detection does not retry if it is not new', + 'early flake detection does not retry if the test is skipped' + ]) + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + // all but one are considered new + assert.equal(newTests.length, 3) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 1) + + const testSessionEvent = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSessionEvent.meta, TEST_STATUS, 'fail') + assert.notProperty(testSessionEvent.meta, TEST_EARLY_FLAKE_ENABLED) + }) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 1) + done() + }).catch(done) + }) + }) + }) }) }) diff --git a/packages/datadog-instrumentations/src/cucumber.js b/packages/datadog-instrumentations/src/cucumber.js index a3a5ae105fd..639f955cc56 100644 --- a/packages/datadog-instrumentations/src/cucumber.js +++ b/packages/datadog-instrumentations/src/cucumber.js @@ -70,6 +70,7 @@ let earlyFlakeDetectionNumRetries = 0 let earlyFlakeDetectionFaultyThreshold = 0 let isEarlyFlakeDetectionFaulty = false let isFlakyTestRetriesEnabled = false +let isKnownTestsEnabled = false let numTestRetries = 0 let knownTests = [] let skippedSuites = [] @@ -292,7 +293,7 @@ function wrapRun (pl, isLatestVersion) { } let isNew = false let isEfdRetry = false - if (isEarlyFlakeDetectionEnabled && status !== 'skip') { + if (isKnownTestsEnabled && status !== 'skip') { const numRetries = numRetriesByPickleId.get(this.pickle.id) isNew = numRetries !== undefined @@ -394,13 +395,15 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin isSuitesSkippingEnabled = configurationResponse.libraryConfig?.isSuitesSkippingEnabled isFlakyTestRetriesEnabled = configurationResponse.libraryConfig?.isFlakyTestRetriesEnabled numTestRetries = configurationResponse.libraryConfig?.flakyTestRetriesCount + isKnownTestsEnabled = configurationResponse.libraryConfig?.isKnownTestsEnabled - if (isEarlyFlakeDetectionEnabled) { + if (isKnownTestsEnabled) { const knownTestsResponse = await getChannelPromise(knownTestsCh) if (!knownTestsResponse.err) { knownTests = knownTestsResponse.knownTests } else { isEarlyFlakeDetectionEnabled = false + isKnownTestsEnabled = false } } @@ -437,7 +440,7 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin pickleByFile = isCoordinator ? getPickleByFileNew(this) : getPickleByFile(this) - if (isEarlyFlakeDetectionEnabled) { + if (isKnownTestsEnabled) { const isFaulty = getIsFaultyEarlyFlakeDetection( Object.keys(pickleByFile), knownTests.cucumber || {}, @@ -445,6 +448,7 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin ) if (isFaulty) { isEarlyFlakeDetectionEnabled = false + isKnownTestsEnabled = false isEarlyFlakeDetectionFaulty = true } } @@ -533,7 +537,7 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa let isNew = false - if (isEarlyFlakeDetectionEnabled) { + if (isKnownTestsEnabled) { isNew = isNewTest(testSuitePath, pickle.name) if (isNew) { numRetriesByPickleId.set(pickle.id, 0) @@ -678,14 +682,14 @@ function getWrappedParseWorkerMessage (parseWorkerMessageFunction, isNewVersion) const { status } = getStatusFromResultLatest(worstTestStepResult) let isNew = false - if (isEarlyFlakeDetectionEnabled) { + if (isKnownTestsEnabled) { isNew = isNewTest(pickle.uri, pickle.name) } const testFileAbsolutePath = pickle.uri const finished = pickleResultByFile[testFileAbsolutePath] - if (isNew) { + if (isEarlyFlakeDetectionEnabled && isNew) { const testFullname = `${pickle.uri}:${pickle.name}` let testStatuses = newTestsByTestFullname.get(testFullname) if (!testStatuses) { @@ -839,7 +843,8 @@ addHook({ ) // EFD in parallel mode only supported in >=11.0.0 shimmer.wrap(adapterPackage.ChildProcessAdapter.prototype, 'startWorker', startWorker => function () { - if (isEarlyFlakeDetectionEnabled) { + if (isKnownTestsEnabled) { + this.options.worldParameters._ddIsEarlyFlakeDetectionEnabled = isEarlyFlakeDetectionEnabled this.options.worldParameters._ddKnownTests = knownTests this.options.worldParameters._ddEarlyFlakeDetectionNumRetries = earlyFlakeDetectionNumRetries } @@ -862,9 +867,12 @@ addHook({ 'initialize', initialize => async function () { await initialize.apply(this, arguments) - isEarlyFlakeDetectionEnabled = !!this.options.worldParameters._ddKnownTests - if (isEarlyFlakeDetectionEnabled) { + isKnownTestsEnabled = !!this.options.worldParameters._ddKnownTests + if (isKnownTestsEnabled) { knownTests = this.options.worldParameters._ddKnownTests + } + isEarlyFlakeDetectionEnabled = !!this.options.worldParameters._ddIsEarlyFlakeDetectionEnabled + if (isEarlyFlakeDetectionEnabled) { earlyFlakeDetectionNumRetries = this.options.worldParameters._ddEarlyFlakeDetectionNumRetries } } diff --git a/packages/datadog-instrumentations/src/jest.js b/packages/datadog-instrumentations/src/jest.js index 7a1001d11f3..bc01fecc150 100644 --- a/packages/datadog-instrumentations/src/jest.js +++ b/packages/datadog-instrumentations/src/jest.js @@ -69,6 +69,7 @@ let earlyFlakeDetectionNumRetries = 0 let earlyFlakeDetectionFaultyThreshold = 30 let isEarlyFlakeDetectionFaulty = false let hasFilteredSkippableSuites = false +let isKnownTestsEnabled = false const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') @@ -138,17 +139,19 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { this.isFlakyTestRetriesEnabled = this.testEnvironmentOptions._ddIsFlakyTestRetriesEnabled this.flakyTestRetriesCount = this.testEnvironmentOptions._ddFlakyTestRetriesCount this.isDiEnabled = this.testEnvironmentOptions._ddIsDiEnabled + this.isKnownTestsEnabled = this.testEnvironmentOptions._ddIsKnownTestsEnabled - if (this.isEarlyFlakeDetectionEnabled) { - const hasKnownTests = !!knownTests.jest - earlyFlakeDetectionNumRetries = this.testEnvironmentOptions._ddEarlyFlakeDetectionNumRetries + if (this.isKnownTestsEnabled) { try { + const hasKnownTests = !!knownTests.jest + earlyFlakeDetectionNumRetries = this.testEnvironmentOptions._ddEarlyFlakeDetectionNumRetries this.knownTestsForThisSuite = hasKnownTests ? (knownTests.jest[this.testSuite] || []) : this.getKnownTestsForSuite(this.testEnvironmentOptions._ddKnownTests) } catch (e) { // If there has been an error parsing the tests, we'll disable Early Flake Deteciton this.isEarlyFlakeDetectionEnabled = false + this.isKnownTestsEnabled = false } } @@ -228,7 +231,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { asyncResources.set(event.test, asyncResource) const testName = getJestTestName(event.test) - if (this.isEarlyFlakeDetectionEnabled) { + if (this.isKnownTestsEnabled) { const originalTestName = removeEfdStringFromTestName(testName) isNewTest = retriedTestsToNumAttempts.has(originalTestName) if (isNewTest) { @@ -254,24 +257,26 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { }) } if (event.name === 'add_test') { - if (this.isEarlyFlakeDetectionEnabled) { + if (this.isKnownTestsEnabled) { const testName = this.getTestNameFromAddTestEvent(event, state) const isNew = !this.knownTestsForThisSuite?.includes(testName) const isSkipped = event.mode === 'todo' || event.mode === 'skip' if (isNew && !isSkipped && !retriedTestsToNumAttempts.has(testName)) { retriedTestsToNumAttempts.set(testName, 0) - // Retrying snapshots has proven to be problematic, so we'll skip them for now - // We'll still detect new tests, but we won't retry them. - // TODO: do not bail out of EFD with the whole test suite - if (this.getHasSnapshotTests()) { - log.warn('Early flake detection is disabled for suites with snapshots') - return - } - for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { - if (this.global.test) { - this.global.test(addEfdStringToTestName(event.testName, retryIndex), event.fn, event.timeout) - } else { - log.error('Early flake detection could not retry test because global.test is undefined') + if (this.isEarlyFlakeDetectionEnabled) { + // Retrying snapshots has proven to be problematic, so we'll skip them for now + // We'll still detect new tests, but we won't retry them. + // TODO: do not bail out of EFD with the whole test suite + if (this.getHasSnapshotTests()) { + log.warn('Early flake detection is disabled for suites with snapshots') + return + } + for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { + if (this.global.test) { + this.global.test(addEfdStringToTestName(event.testName, retryIndex), event.fn, event.timeout) + } else { + log.error('Early flake detection could not retry test because global.test is undefined') + } } } } @@ -286,7 +291,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { event.test.fn = originalTestFns.get(event.test) // We'll store the test statuses of the retries - if (this.isEarlyFlakeDetectionEnabled) { + if (this.isKnownTestsEnabled) { const testName = getJestTestName(event.test) const originalTestName = removeEfdStringFromTestName(testName) const isNewTest = retriedTestsToNumAttempts.has(originalTestName) @@ -483,12 +488,13 @@ function cliWrapper (cli, jestVersion) { isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries earlyFlakeDetectionFaultyThreshold = libraryConfig.earlyFlakeDetectionFaultyThreshold + isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled } } catch (err) { log.error('Jest library configuration error', err) } - if (isEarlyFlakeDetectionEnabled) { + if (isKnownTestsEnabled) { const knownTestsPromise = new Promise((resolve) => { onDone = resolve }) @@ -504,6 +510,7 @@ function cliWrapper (cli, jestVersion) { } else { // We disable EFD if there has been an error in the known tests request isEarlyFlakeDetectionEnabled = false + isKnownTestsEnabled = false } } catch (err) { log.error('Jest known tests error', err) @@ -821,6 +828,7 @@ addHook({ _ddIsFlakyTestRetriesEnabled, _ddFlakyTestRetriesCount, _ddIsDiEnabled, + _ddIsKnownTestsEnabled, ...restOfTestEnvironmentOptions } = testEnvironmentOptions @@ -848,17 +856,19 @@ addHook({ const testPaths = await getTestPaths.apply(this, arguments) const [{ rootDir, shard }] = arguments - if (isEarlyFlakeDetectionEnabled) { + if (isKnownTestsEnabled) { const projectSuites = testPaths.tests.map(test => getTestSuitePath(test.path, test.context.config.rootDir)) const isFaulty = getIsFaultyEarlyFlakeDetection(projectSuites, knownTests.jest || {}, earlyFlakeDetectionFaultyThreshold) if (isFaulty) { log.error('Early flake detection is disabled because the number of new suites is too high.') isEarlyFlakeDetectionEnabled = false + isKnownTestsEnabled = false const testEnvironmentOptions = testPaths.tests[0]?.context?.config?.testEnvironmentOptions // Project config is shared among all tests, so we can modify it here if (testEnvironmentOptions) { testEnvironmentOptions._ddIsEarlyFlakeDetectionEnabled = false + testEnvironmentOptions._ddIsKnownTestsEnabled = false } isEarlyFlakeDetectionFaulty = true } @@ -929,6 +939,11 @@ addHook({ return runtimePackage }) +/* +* This hook does two things: +* - Pass known tests to the workers. +* - Receive trace, coverage and logs payloads from the workers. +*/ addHook({ name: 'jest-worker', versions: ['>=24.9.0'], @@ -936,7 +951,7 @@ addHook({ }, (childProcessWorker) => { const ChildProcessWorker = childProcessWorker.default shimmer.wrap(ChildProcessWorker.prototype, 'send', send => function (request) { - if (!isEarlyFlakeDetectionEnabled) { + if (!isKnownTestsEnabled) { return send.apply(this, arguments) } const [type] = request diff --git a/packages/datadog-instrumentations/src/mocha/main.js b/packages/datadog-instrumentations/src/mocha/main.js index 2e796a71371..afa7bfe0fc4 100644 --- a/packages/datadog-instrumentations/src/mocha/main.js +++ b/packages/datadog-instrumentations/src/mocha/main.js @@ -201,6 +201,7 @@ function getExecutionConfiguration (runner, isParallel, onFinishRequest) { if (err) { config.knownTests = [] config.isEarlyFlakeDetectionEnabled = false + config.isKnownTestsEnabled = false } else { config.knownTests = knownTests } @@ -222,12 +223,13 @@ function getExecutionConfiguration (runner, isParallel, onFinishRequest) { config.isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled config.earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries config.earlyFlakeDetectionFaultyThreshold = libraryConfig.earlyFlakeDetectionFaultyThreshold + config.isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled // ITR and auto test retries are not supported in parallel mode yet config.isSuitesSkippingEnabled = !isParallel && libraryConfig.isSuitesSkippingEnabled config.isFlakyTestRetriesEnabled = !isParallel && libraryConfig.isFlakyTestRetriesEnabled config.flakyTestRetriesCount = !isParallel && libraryConfig.flakyTestRetriesCount - if (config.isEarlyFlakeDetectionEnabled) { + if (config.isKnownTestsEnabled) { knownTestsCh.publish({ onDone: mochaRunAsyncResource.bind(onReceivedKnownTests) }) @@ -273,7 +275,7 @@ addHook({ }) getExecutionConfiguration(runner, false, () => { - if (config.isEarlyFlakeDetectionEnabled) { + if (config.isKnownTestsEnabled) { const testSuites = this.files.map(file => getTestSuitePath(file, process.cwd())) const isFaulty = getIsFaultyEarlyFlakeDetection( testSuites, @@ -283,6 +285,7 @@ addHook({ if (isFaulty) { config.isEarlyFlakeDetectionEnabled = false config.isEarlyFlakeDetectionFaulty = true + config.isKnownTestsEnabled = false } } if (getCodeCoverageCh.hasSubscribers) { @@ -537,7 +540,7 @@ addHook({ this.once('end', getOnEndHandler(true)) getExecutionConfiguration(this, true, () => { - if (config.isEarlyFlakeDetectionEnabled) { + if (config.isKnownTestsEnabled) { const testSuites = files.map(file => getTestSuitePath(file, process.cwd())) const isFaulty = getIsFaultyEarlyFlakeDetection( testSuites, @@ -545,6 +548,7 @@ addHook({ config.earlyFlakeDetectionFaultyThreshold ) if (isFaulty) { + config.isKnownTestsEnabled = false config.isEarlyFlakeDetectionEnabled = false config.isEarlyFlakeDetectionFaulty = true } @@ -569,7 +573,7 @@ addHook({ const { BufferedWorkerPool } = BufferedWorkerPoolPackage shimmer.wrap(BufferedWorkerPool.prototype, 'run', run => async function (testSuiteAbsolutePath, workerArgs) { - if (!testStartCh.hasSubscribers || !config.isEarlyFlakeDetectionEnabled) { + if (!testStartCh.hasSubscribers || !config.isKnownTestsEnabled) { return run.apply(this, arguments) } @@ -584,6 +588,7 @@ addHook({ { ...workerArgs, _ddEfdNumRetries: config.earlyFlakeDetectionNumRetries, + _ddIsEfdEnabled: config.isEarlyFlakeDetectionEnabled, _ddKnownTests: { mocha: { [testPath]: testSuiteKnownTests diff --git a/packages/datadog-instrumentations/src/mocha/utils.js b/packages/datadog-instrumentations/src/mocha/utils.js index 97b5f2d1209..30710ab645b 100644 --- a/packages/datadog-instrumentations/src/mocha/utils.js +++ b/packages/datadog-instrumentations/src/mocha/utils.js @@ -349,12 +349,14 @@ function getOnPendingHandler () { // Hook to add retries to tests if EFD is enabled function getRunTestsWrapper (runTests, config) { return function (suite, fn) { - if (config.isEarlyFlakeDetectionEnabled) { + if (config.isKnownTestsEnabled) { // by the time we reach `this.on('test')`, it is too late. We need to add retries here suite.tests.forEach(test => { if (!test.isPending() && isNewTest(test, config.knownTests)) { test._ddIsNew = true - retryTest(test, config.earlyFlakeDetectionNumRetries) + if (config.isEarlyFlakeDetectionEnabled) { + retryTest(test, config.earlyFlakeDetectionNumRetries) + } } }) } diff --git a/packages/datadog-instrumentations/src/mocha/worker.js b/packages/datadog-instrumentations/src/mocha/worker.js index 63670ba5db2..56a9dc75270 100644 --- a/packages/datadog-instrumentations/src/mocha/worker.js +++ b/packages/datadog-instrumentations/src/mocha/worker.js @@ -25,10 +25,12 @@ addHook({ }, (Mocha) => { shimmer.wrap(Mocha.prototype, 'run', run => function () { if (this.options._ddKnownTests) { - // EFD is enabled if there's a list of known tests - config.isEarlyFlakeDetectionEnabled = true + // If there are known tests, it means isKnownTestsEnabled should be true + config.isKnownTestsEnabled = true + config.isEarlyFlakeDetectionEnabled = this.options._ddIsEfdEnabled config.knownTests = this.options._ddKnownTests config.earlyFlakeDetectionNumRetries = this.options._ddEfdNumRetries + delete this.options._ddIsEfdEnabled delete this.options._ddKnownTests delete this.options._ddEfdNumRetries } diff --git a/packages/datadog-instrumentations/src/playwright.js b/packages/datadog-instrumentations/src/playwright.js index 4eab55b1797..9cc7d64cd1c 100644 --- a/packages/datadog-instrumentations/src/playwright.js +++ b/packages/datadog-instrumentations/src/playwright.js @@ -35,6 +35,7 @@ const STATUS_TO_TEST_STATUS = { } let remainingTestsByFile = {} +let isKnownTestsEnabled = false let isEarlyFlakeDetectionEnabled = false let earlyFlakeDetectionNumRetries = 0 let isFlakyTestRetriesEnabled = false @@ -418,6 +419,7 @@ function runnerHook (runnerExport, playwrightVersion) { try { const { err, libraryConfig } = await getChannelPromise(libraryConfigurationCh) if (!err) { + isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries isFlakyTestRetriesEnabled = libraryConfig.isFlakyTestRetriesEnabled @@ -425,19 +427,22 @@ function runnerHook (runnerExport, playwrightVersion) { } } catch (e) { isEarlyFlakeDetectionEnabled = false + isKnownTestsEnabled = false log.error('Playwright session start error', e) } - if (isEarlyFlakeDetectionEnabled && semver.gte(playwrightVersion, MINIMUM_SUPPORTED_VERSION_EFD)) { + if (isKnownTestsEnabled && semver.gte(playwrightVersion, MINIMUM_SUPPORTED_VERSION_EFD)) { try { const { err, knownTests: receivedKnownTests } = await getChannelPromise(knownTestsCh) if (!err) { knownTests = receivedKnownTests } else { isEarlyFlakeDetectionEnabled = false + isKnownTestsEnabled = false } } catch (err) { isEarlyFlakeDetectionEnabled = false + isKnownTestsEnabled = false log.error('Playwright known tests error', err) } } @@ -553,7 +558,7 @@ addHook({ async function newCreateRootSuite () { const rootSuite = await oldCreateRootSuite.apply(this, arguments) - if (!isEarlyFlakeDetectionEnabled) { + if (!isKnownTestsEnabled) { return rootSuite } const newTests = rootSuite @@ -562,7 +567,7 @@ addHook({ newTests.forEach(newTest => { newTest._ddIsNew = true - if (newTest.expectedStatus !== 'skipped') { + if (isEarlyFlakeDetectionEnabled && newTest.expectedStatus !== 'skipped') { const fileSuite = getSuiteType(newTest, 'file') const projectSuite = getSuiteType(newTest, 'project') for (let repeatEachIndex = 0; repeatEachIndex < earlyFlakeDetectionNumRetries; repeatEachIndex++) { diff --git a/packages/datadog-instrumentations/src/vitest.js b/packages/datadog-instrumentations/src/vitest.js index b3f2a9af8b8..ebde98b4789 100644 --- a/packages/datadog-instrumentations/src/vitest.js +++ b/packages/datadog-instrumentations/src/vitest.js @@ -25,6 +25,7 @@ const isEarlyFlakeDetectionFaultyCh = channel('ci:vitest:is-early-flake-detectio const taskToAsync = new WeakMap() const taskToStatuses = new WeakMap() const newTasks = new WeakSet() +let isRetryReasonEfd = false const switchedStatuses = new WeakSet() const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') @@ -44,14 +45,16 @@ function getProvidedContext () { _ddIsEarlyFlakeDetectionEnabled, _ddIsDiEnabled, _ddKnownTests: knownTests, - _ddEarlyFlakeDetectionNumRetries: numRepeats + _ddEarlyFlakeDetectionNumRetries: numRepeats, + _ddIsKnownTestsEnabled: isKnownTestsEnabled } = globalThis.__vitest_worker__.providedContext return { isDiEnabled: _ddIsDiEnabled, isEarlyFlakeDetectionEnabled: _ddIsEarlyFlakeDetectionEnabled, knownTests, - numRepeats + numRepeats, + isKnownTestsEnabled } } catch (e) { log.error('Vitest workers could not parse provided context, so some features will not work.') @@ -59,7 +62,8 @@ function getProvidedContext () { isDiEnabled: false, isEarlyFlakeDetectionEnabled: false, knownTests: {}, - numRepeats: 0 + numRepeats: 0, + isKnownTestsEnabled: false } } } @@ -153,6 +157,7 @@ function getSortWrapper (sort) { let isEarlyFlakeDetectionEnabled = false let earlyFlakeDetectionNumRetries = 0 let isEarlyFlakeDetectionFaulty = false + let isKnownTestsEnabled = false let isDiEnabled = false let knownTests = {} @@ -164,18 +169,20 @@ function getSortWrapper (sort) { isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries isDiEnabled = libraryConfig.isDiEnabled + isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled } } catch (e) { isFlakyTestRetriesEnabled = false isEarlyFlakeDetectionEnabled = false isDiEnabled = false + isKnownTestsEnabled = false } if (isFlakyTestRetriesEnabled && !this.ctx.config.retry && flakyTestRetriesCount > 0) { this.ctx.config.retry = flakyTestRetriesCount } - if (isEarlyFlakeDetectionEnabled) { + if (isKnownTestsEnabled) { const knownTestsResponse = await getChannelPromise(knownTestsCh) if (!knownTestsResponse.err) { knownTests = knownTestsResponse.knownTests @@ -192,13 +199,15 @@ function getSortWrapper (sort) { }) if (isEarlyFlakeDetectionFaulty) { isEarlyFlakeDetectionEnabled = false - log.warn('Early flake detection is disabled because the number of new tests is too high.') + isKnownTestsEnabled = false + log.warn('New test detection is disabled because the number of new tests is too high.') } else { // TODO: use this to pass session and module IDs to the worker, instead of polluting process.env // Note: setting this.ctx.config.provide directly does not work because it's cached try { const workspaceProject = this.ctx.getCoreWorkspaceProject() - workspaceProject._provided._ddKnownTests = knownTests.vitest + workspaceProject._provided._ddIsKnownTestsEnabled = isKnownTestsEnabled + workspaceProject._provided._ddKnownTests = knownTests.vitest || {} workspaceProject._provided._ddIsEarlyFlakeDetectionEnabled = isEarlyFlakeDetectionEnabled workspaceProject._provided._ddEarlyFlakeDetectionNumRetries = earlyFlakeDetectionNumRetries } catch (e) { @@ -207,6 +216,7 @@ function getSortWrapper (sort) { } } else { isEarlyFlakeDetectionEnabled = false + isKnownTestsEnabled = false } } @@ -295,17 +305,21 @@ addHook({ const { knownTests, isEarlyFlakeDetectionEnabled, + isKnownTestsEnabled, numRepeats } = getProvidedContext() - if (isEarlyFlakeDetectionEnabled) { + if (isKnownTestsEnabled) { isNewTestCh.publish({ knownTests, testSuiteAbsolutePath: task.file.filepath, testName, onDone: (isNew) => { if (isNew) { - task.repeats = numRepeats + if (isEarlyFlakeDetectionEnabled) { + isRetryReasonEfd = task.repeats !== numRepeats + task.repeats = numRepeats + } newTasks.add(task) taskToStatuses.set(task, []) } @@ -344,11 +358,12 @@ addHook({ let isNew = false const { + isKnownTestsEnabled, isEarlyFlakeDetectionEnabled, isDiEnabled } = getProvidedContext() - if (isEarlyFlakeDetectionEnabled) { + if (isKnownTestsEnabled) { isNew = newTasks.has(task) } @@ -431,6 +446,7 @@ addHook({ testName, testSuiteAbsolutePath: task.file.filepath, isRetry: numAttempt > 0 || numRepetition > 0, + isRetryReasonEfd, isNew, mightHitProbe: isDiEnabled && numAttempt > 0 }) @@ -576,7 +592,11 @@ addHook({ if (result) { const { state, duration, errors } = result if (state === 'skip') { // programmatic skip - testSkipCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.file.filepath }) + testSkipCh.publish({ + testName: getTestName(task), + testSuiteAbsolutePath: task.file.filepath, + isNew: newTasks.has(task) + }) } else if (state === 'pass' && !isSwitchedStatus) { if (testAsyncResource) { testAsyncResource.runInAsyncScope(() => { @@ -602,7 +622,11 @@ addHook({ } } } else { // test.skip or test.todo - testSkipCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.file.filepath }) + testSkipCh.publish({ + testName: getTestName(task), + testSuiteAbsolutePath: task.file.filepath, + isNew: newTasks.has(task) + }) } }) diff --git a/packages/datadog-plugin-cucumber/src/index.js b/packages/datadog-plugin-cucumber/src/index.js index 16cca8b6b59..7454c87560b 100644 --- a/packages/datadog-plugin-cucumber/src/index.js +++ b/packages/datadog-plugin-cucumber/src/index.js @@ -26,7 +26,8 @@ const { TEST_MODULE, TEST_MODULE_ID, TEST_SUITE, - CUCUMBER_IS_PARALLEL + CUCUMBER_IS_PARALLEL, + TEST_RETRY_REASON } = require('../../dd-trace/src/plugins/util/test') const { RESOURCE_NAME } = require('../../../ext/tags') const { COMPONENT, ERROR_MESSAGE } = require('../../dd-trace/src/constants') @@ -321,6 +322,7 @@ class CucumberPlugin extends CiPlugin { span.setTag(TEST_IS_NEW, 'true') if (isEfdRetry) { span.setTag(TEST_IS_RETRY, 'true') + span.setTag(TEST_RETRY_REASON, 'efd') } } diff --git a/packages/datadog-plugin-cypress/src/cypress-plugin.js b/packages/datadog-plugin-cypress/src/cypress-plugin.js index 2ed62070fda..31d4d282f64 100644 --- a/packages/datadog-plugin-cypress/src/cypress-plugin.js +++ b/packages/datadog-plugin-cypress/src/cypress-plugin.js @@ -31,7 +31,8 @@ const { TEST_EARLY_FLAKE_ENABLED, getTestSessionName, TEST_SESSION_NAME, - TEST_LEVEL_EVENT_TYPES + TEST_LEVEL_EVENT_TYPES, + TEST_RETRY_REASON } = require('../../dd-trace/src/plugins/util/test') const { isMarkedAsUnskippable } = require('../../datadog-plugin-jest/src/util') const { ORIGIN_KEY, COMPONENT } = require('../../dd-trace/src/constants') @@ -112,7 +113,7 @@ function getCypressCommand (details) { function getLibraryConfiguration (tracer, testConfiguration) { return new Promise(resolve => { if (!tracer._tracer._exporter?.getLibraryConfiguration) { - return resolve({ err: new Error('CI Visibility was not initialized correctly') }) + return resolve({ err: new Error('Test Optimization was not initialized correctly') }) } tracer._tracer._exporter.getLibraryConfiguration(testConfiguration, (err, libraryConfig) => { @@ -124,7 +125,7 @@ function getLibraryConfiguration (tracer, testConfiguration) { function getSkippableTests (tracer, testConfiguration) { return new Promise(resolve => { if (!tracer._tracer._exporter?.getSkippableSuites) { - return resolve({ err: new Error('CI Visibility was not initialized correctly') }) + return resolve({ err: new Error('Test Optimization was not initialized correctly') }) } tracer._tracer._exporter.getSkippableSuites(testConfiguration, (err, skippableTests, correlationId) => { resolve({ @@ -139,7 +140,7 @@ function getSkippableTests (tracer, testConfiguration) { function getKnownTests (tracer, testConfiguration) { return new Promise(resolve => { if (!tracer._tracer._exporter?.getKnownTests) { - return resolve({ err: new Error('CI Visibility was not initialized correctly') }) + return resolve({ err: new Error('Test Optimization was not initialized correctly') }) } tracer._tracer._exporter.getKnownTests(testConfiguration, (err, knownTests) => { resolve({ @@ -203,6 +204,7 @@ class CypressPlugin { this.isSuitesSkippingEnabled = false this.isCodeCoverageEnabled = false this.isEarlyFlakeDetectionEnabled = false + this.isKnownTestsEnabled = false this.earlyFlakeDetectionNumRetries = 0 this.testsToSkip = [] this.skippedTests = [] @@ -232,13 +234,15 @@ class CypressPlugin { isEarlyFlakeDetectionEnabled, earlyFlakeDetectionNumRetries, isFlakyTestRetriesEnabled, - flakyTestRetriesCount + flakyTestRetriesCount, + isKnownTestsEnabled } } = libraryConfigurationResponse this.isSuitesSkippingEnabled = isSuitesSkippingEnabled this.isCodeCoverageEnabled = isCodeCoverageEnabled this.isEarlyFlakeDetectionEnabled = isEarlyFlakeDetectionEnabled this.earlyFlakeDetectionNumRetries = earlyFlakeDetectionNumRetries + this.isKnownTestsEnabled = isKnownTestsEnabled if (isFlakyTestRetriesEnabled) { this.cypressConfig.retries.runMode = flakyTestRetriesCount } @@ -354,7 +358,7 @@ class CypressPlugin { this.frameworkVersion = getCypressVersion(details) this.rootDir = getRootDir(details) - if (this.isEarlyFlakeDetectionEnabled) { + if (this.isKnownTestsEnabled) { const knownTestsResponse = await getKnownTests( this.tracer, this.testConfiguration @@ -362,6 +366,7 @@ class CypressPlugin { if (knownTestsResponse.err) { log.error('Cypress known tests response error', knownTestsResponse.err) this.isEarlyFlakeDetectionEnabled = false + this.isKnownTestsEnabled = false } else { // We use TEST_FRAMEWORK_NAME for the name of the module this.knownTestsByTestSuite = knownTestsResponse.knownTests[TEST_FRAMEWORK_NAME] @@ -567,6 +572,9 @@ class CypressPlugin { cypressTestStatus = CYPRESS_STATUS_TO_TEST_STATUS[cypressTest.attempts[attemptIndex].state] if (attemptIndex > 0) { finishedTest.testSpan.setTag(TEST_IS_RETRY, 'true') + if (finishedTest.isEfdRetry) { + finishedTest.testSpan.setTag(TEST_RETRY_REASON, 'efd') + } } } if (cypressTest.displayError) { @@ -618,7 +626,8 @@ class CypressPlugin { const suitePayload = { isEarlyFlakeDetectionEnabled: this.isEarlyFlakeDetectionEnabled, knownTestsForSuite: this.knownTestsByTestSuite?.[testSuite] || [], - earlyFlakeDetectionNumRetries: this.earlyFlakeDetectionNumRetries + earlyFlakeDetectionNumRetries: this.earlyFlakeDetectionNumRetries, + isKnownTestsEnabled: this.isKnownTestsEnabled } if (this.testSuiteSpan) { @@ -703,13 +712,15 @@ class CypressPlugin { this.activeTestSpan.setTag(TEST_IS_NEW, 'true') if (isEfdRetry) { this.activeTestSpan.setTag(TEST_IS_RETRY, 'true') + this.activeTestSpan.setTag(TEST_RETRY_REASON, 'efd') } } const finishedTest = { testName, testStatus, finishTime: this.activeTestSpan._getTime(), // we store the finish time here - testSpan: this.activeTestSpan + testSpan: this.activeTestSpan, + isEfdRetry } if (this.finishedTestsByFile[testSuite]) { this.finishedTestsByFile[testSuite].push(finishedTest) diff --git a/packages/datadog-plugin-cypress/src/support.js b/packages/datadog-plugin-cypress/src/support.js index 8900f2695fb..6e31e9e45a1 100644 --- a/packages/datadog-plugin-cypress/src/support.js +++ b/packages/datadog-plugin-cypress/src/support.js @@ -1,5 +1,6 @@ /* eslint-disable */ let isEarlyFlakeDetectionEnabled = false +let isKnownTestsEnabled = false let knownTestsForSuite = [] let suiteTests = [] let earlyFlakeDetectionNumRetries = 0 @@ -33,7 +34,7 @@ function retryTest (test, suiteTests) { const oldRunTests = Cypress.mocha.getRunner().runTests Cypress.mocha.getRunner().runTests = function (suite, fn) { - if (!isEarlyFlakeDetectionEnabled) { + if (!isKnownTestsEnabled) { return oldRunTests.apply(this, arguments) } // We copy the new tests at the beginning of the suite run (runTests), so that they're run @@ -41,7 +42,9 @@ Cypress.mocha.getRunner().runTests = function (suite, fn) { suite.tests.forEach(test => { if (!test._ddIsNew && !test.isPending() && isNewTest(test)) { test._ddIsNew = true - retryTest(test, suite.tests) + if (isEarlyFlakeDetectionEnabled) { + retryTest(test, suite.tests) + } } }) @@ -67,6 +70,7 @@ before(function () { }).then((suiteConfig) => { if (suiteConfig) { isEarlyFlakeDetectionEnabled = suiteConfig.isEarlyFlakeDetectionEnabled + isKnownTestsEnabled = suiteConfig.isKnownTestsEnabled knownTestsForSuite = suiteConfig.knownTestsForSuite earlyFlakeDetectionNumRetries = suiteConfig.earlyFlakeDetectionNumRetries } diff --git a/packages/datadog-plugin-fetch/src/index.js b/packages/datadog-plugin-fetch/src/index.js index 44173a561ca..943a1908ddb 100644 --- a/packages/datadog-plugin-fetch/src/index.js +++ b/packages/datadog-plugin-fetch/src/index.js @@ -9,7 +9,7 @@ class FetchPlugin extends HttpClientPlugin { bindStart (ctx) { const req = ctx.req const options = new URL(req.url) - const headers = options.headers = Object.fromEntries(req.headers.entries()) + options.headers = Object.fromEntries(req.headers.entries()) options.method = req.method @@ -17,9 +17,9 @@ class FetchPlugin extends HttpClientPlugin { const store = super.bindStart(ctx) - for (const name in headers) { + for (const name in options.headers) { if (!req.headers.has(name)) { - req.headers.set(name, headers[name]) + req.headers.set(name, options.headers[name]) } } diff --git a/packages/datadog-plugin-fetch/test/index.spec.js b/packages/datadog-plugin-fetch/test/index.spec.js index 1d322de04a4..1d20d375d79 100644 --- a/packages/datadog-plugin-fetch/test/index.spec.js +++ b/packages/datadog-plugin-fetch/test/index.spec.js @@ -14,7 +14,9 @@ const HTTP_RESPONSE_HEADERS = tags.HTTP_RESPONSE_HEADERS const SERVICE_NAME = DD_MAJOR < 3 ? 'test-http-client' : 'test' const describe = globalThis.fetch ? globalThis.describe : globalThis.describe.skip -describe('Plugin', () => { +describe('Plugin', function () { + this.timeout(0) + let express let fetch let appListener @@ -215,102 +217,6 @@ describe('Plugin', () => { }) }) - it('should skip injecting if the Authorization header contains an AWS signature', done => { - const app = express() - - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.undefined - expect(req.get('x-datadog-parent-id')).to.be.undefined - - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - appListener = server(app, port => { - fetch(`http://localhost:${port}/`, { - headers: { - Authorization: 'AWS4-HMAC-SHA256 ...' - } - }) - }) - }) - - it('should skip injecting if one of the Authorization headers contains an AWS signature', done => { - const app = express() - - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.undefined - expect(req.get('x-datadog-parent-id')).to.be.undefined - - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - appListener = server(app, port => { - fetch(`http://localhost:${port}/`, { - headers: { - Authorization: ['AWS4-HMAC-SHA256 ...'] - } - }) - }) - }) - - it('should skip injecting if the X-Amz-Signature header is set', done => { - const app = express() - - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.undefined - expect(req.get('x-datadog-parent-id')).to.be.undefined - - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - appListener = server(app, port => { - fetch(`http://localhost:${port}/`, { - headers: { - 'X-Amz-Signature': 'abc123' - } - }) - }) - }) - - it('should skip injecting if the X-Amz-Signature query param is set', done => { - const app = express() - - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.undefined - expect(req.get('x-datadog-parent-id')).to.be.undefined - - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - appListener = server(app, port => { - fetch(`http://localhost:${port}/?X-Amz-Signature=abc123`) - }) - }) - it('should handle connection errors', done => { let error diff --git a/packages/datadog-plugin-jest/src/index.js b/packages/datadog-plugin-jest/src/index.js index 751cbef790b..f82899f20d1 100644 --- a/packages/datadog-plugin-jest/src/index.js +++ b/packages/datadog-plugin-jest/src/index.js @@ -23,7 +23,8 @@ const { JEST_DISPLAY_NAME, TEST_IS_RUM_ACTIVE, TEST_BROWSER_DRIVER, - getFormattedError + getFormattedError, + TEST_RETRY_REASON } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const id = require('../../dd-trace/src/id') @@ -167,6 +168,7 @@ class JestPlugin extends CiPlugin { config._ddIsFlakyTestRetriesEnabled = this.libraryConfig?.isFlakyTestRetriesEnabled ?? false config._ddFlakyTestRetriesCount = this.libraryConfig?.flakyTestRetriesCount config._ddIsDiEnabled = this.libraryConfig?.isDiEnabled ?? false + config._ddIsKnownTestsEnabled = this.libraryConfig?.isKnownTestsEnabled ?? false }) }) @@ -410,6 +412,7 @@ class JestPlugin extends CiPlugin { extraTags[TEST_IS_NEW] = 'true' if (isEfdRetry) { extraTags[TEST_IS_RETRY] = 'true' + extraTags[TEST_RETRY_REASON] = 'efd' } } diff --git a/packages/datadog-plugin-mocha/src/index.js b/packages/datadog-plugin-mocha/src/index.js index bea9400b083..f4c9b063328 100644 --- a/packages/datadog-plugin-mocha/src/index.js +++ b/packages/datadog-plugin-mocha/src/index.js @@ -30,7 +30,8 @@ const { TEST_SUITE, MOCHA_IS_PARALLEL, TEST_IS_RUM_ACTIVE, - TEST_BROWSER_DRIVER + TEST_BROWSER_DRIVER, + TEST_RETRY_REASON } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const { @@ -421,6 +422,7 @@ class MochaPlugin extends CiPlugin { extraTags[TEST_IS_NEW] = 'true' if (isEfdRetry) { extraTags[TEST_IS_RETRY] = 'true' + extraTags[TEST_RETRY_REASON] = 'efd' } } diff --git a/packages/datadog-plugin-playwright/src/index.js b/packages/datadog-plugin-playwright/src/index.js index 941f779ff54..8fd8ac6fef0 100644 --- a/packages/datadog-plugin-playwright/src/index.js +++ b/packages/datadog-plugin-playwright/src/index.js @@ -15,7 +15,8 @@ const { TEST_IS_NEW, TEST_IS_RETRY, TEST_EARLY_FLAKE_ENABLED, - TELEMETRY_TEST_SESSION + TELEMETRY_TEST_SESSION, + TEST_RETRY_REASON } = require('../../dd-trace/src/plugins/util/test') const { RESOURCE_NAME } = require('../../../ext/tags') const { COMPONENT } = require('../../dd-trace/src/constants') @@ -144,6 +145,7 @@ class PlaywrightPlugin extends CiPlugin { span.setTag(TEST_IS_NEW, 'true') if (isEfdRetry) { span.setTag(TEST_IS_RETRY, 'true') + span.setTag(TEST_RETRY_REASON, 'efd') } } if (isRetry) { diff --git a/packages/datadog-plugin-vitest/src/index.js b/packages/datadog-plugin-vitest/src/index.js index 5b8bc9e865e..c4f94548f10 100644 --- a/packages/datadog-plugin-vitest/src/index.js +++ b/packages/datadog-plugin-vitest/src/index.js @@ -17,7 +17,8 @@ const { TEST_SOURCE_START, TEST_IS_NEW, TEST_EARLY_FLAKE_ENABLED, - TEST_EARLY_FLAKE_ABORT_REASON + TEST_EARLY_FLAKE_ABORT_REASON, + TEST_RETRY_REASON } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const { @@ -60,7 +61,14 @@ class VitestPlugin extends CiPlugin { onDone(isFaulty) }) - this.addSub('ci:vitest:test:start', ({ testName, testSuiteAbsolutePath, isRetry, isNew, mightHitProbe }) => { + this.addSub('ci:vitest:test:start', ({ + testName, + testSuiteAbsolutePath, + isRetry, + isNew, + mightHitProbe, + isRetryReasonEfd + }) => { const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) const store = storage.getStore() @@ -73,6 +81,9 @@ class VitestPlugin extends CiPlugin { if (isNew) { extraTags[TEST_IS_NEW] = 'true' } + if (isRetryReasonEfd) { + extraTags[TEST_RETRY_REASON] = 'efd' + } const span = this.startTestSpan( testName, @@ -147,7 +158,7 @@ class VitestPlugin extends CiPlugin { } }) - this.addSub('ci:vitest:test:skip', ({ testName, testSuiteAbsolutePath }) => { + this.addSub('ci:vitest:test:skip', ({ testName, testSuiteAbsolutePath, isNew }) => { const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) const testSpan = this.startTestSpan( testName, @@ -156,7 +167,8 @@ class VitestPlugin extends CiPlugin { { [TEST_SOURCE_FILE]: testSuite, [TEST_SOURCE_START]: 1, // we can't get the proper start line in vitest - [TEST_STATUS]: 'skip' + [TEST_STATUS]: 'skip', + ...(isNew ? { [TEST_IS_NEW]: 'true' } : {}) } ) this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'test', { diff --git a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js index 3ad1a11e027..3cbd64afbc2 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +++ b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js @@ -87,9 +87,8 @@ class CiVisibilityExporter extends AgentInfoExporter { shouldRequestKnownTests () { return !!( - this._config.isEarlyFlakeDetectionEnabled && this._canUseCiVisProtocol && - this._libraryConfig?.isEarlyFlakeDetectionEnabled + this._libraryConfig?.isKnownTestsEnabled ) } @@ -197,7 +196,8 @@ class CiVisibilityExporter extends AgentInfoExporter { earlyFlakeDetectionNumRetries, earlyFlakeDetectionFaultyThreshold, isFlakyTestRetriesEnabled, - isDiEnabled + isDiEnabled, + isKnownTestsEnabled } = remoteConfiguration return { isCodeCoverageEnabled, @@ -209,7 +209,8 @@ class CiVisibilityExporter extends AgentInfoExporter { earlyFlakeDetectionFaultyThreshold, isFlakyTestRetriesEnabled: isFlakyTestRetriesEnabled && this._config.isFlakyTestRetriesEnabled, flakyTestRetriesCount: this._config.flakyTestRetriesCount, - isDiEnabled: isDiEnabled && this._config.isTestDynamicInstrumentationEnabled + isDiEnabled: isDiEnabled && this._config.isTestDynamicInstrumentationEnabled, + isKnownTestsEnabled } } diff --git a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js index e39770dea82..26d818bcdd2 100644 --- a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js +++ b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js @@ -93,7 +93,8 @@ function getLibraryConfiguration ({ require_git: requireGit, early_flake_detection: earlyFlakeDetectionConfig, flaky_test_retries_enabled: isFlakyTestRetriesEnabled, - di_enabled: isDiEnabled + di_enabled: isDiEnabled, + known_tests_enabled: isKnownTestsEnabled } } } = JSON.parse(res) @@ -103,13 +104,14 @@ function getLibraryConfiguration ({ isSuitesSkippingEnabled, isItrEnabled, requireGit, - isEarlyFlakeDetectionEnabled: earlyFlakeDetectionConfig?.enabled ?? false, + isEarlyFlakeDetectionEnabled: isKnownTestsEnabled && (earlyFlakeDetectionConfig?.enabled ?? false), earlyFlakeDetectionNumRetries: earlyFlakeDetectionConfig?.slow_test_retries?.['5s'] || DEFAULT_EARLY_FLAKE_DETECTION_NUM_RETRIES, earlyFlakeDetectionFaultyThreshold: earlyFlakeDetectionConfig?.faulty_session_threshold ?? DEFAULT_EARLY_FLAKE_DETECTION_ERROR_THRESHOLD, isFlakyTestRetriesEnabled, - isDiEnabled: isDiEnabled && isFlakyTestRetriesEnabled + isDiEnabled: isDiEnabled && isFlakyTestRetriesEnabled, + isKnownTestsEnabled } log.debug(() => `Remote settings: ${JSON.stringify(settings)}`) diff --git a/packages/dd-trace/src/plugins/ci_plugin.js b/packages/dd-trace/src/plugins/ci_plugin.js index 60c1c59a9bc..287d3e6d55d 100644 --- a/packages/dd-trace/src/plugins/ci_plugin.js +++ b/packages/dd-trace/src/plugins/ci_plugin.js @@ -158,6 +158,7 @@ module.exports = class CiPlugin extends Plugin { if (err) { log.error('Known tests could not be fetched. %s', err.message) this.libraryConfig.isEarlyFlakeDetectionEnabled = false + this.libraryConfig.isKnownTestsEnabled = false } onDone({ err, knownTests }) }) diff --git a/packages/dd-trace/src/plugins/util/test.js b/packages/dd-trace/src/plugins/util/test.js index d8aab1a44da..2d8ce1a1d33 100644 --- a/packages/dd-trace/src/plugins/util/test.js +++ b/packages/dd-trace/src/plugins/util/test.js @@ -59,6 +59,7 @@ const TEST_IS_NEW = 'test.is_new' const TEST_IS_RETRY = 'test.is_retry' const TEST_EARLY_FLAKE_ENABLED = 'test.early_flake.enabled' const TEST_EARLY_FLAKE_ABORT_REASON = 'test.early_flake.abort_reason' +const TEST_RETRY_REASON = 'test.retry_reason' const CI_APP_ORIGIN = 'ciapp-test' @@ -145,6 +146,7 @@ module.exports = { TEST_IS_RETRY, TEST_EARLY_FLAKE_ENABLED, TEST_EARLY_FLAKE_ABORT_REASON, + TEST_RETRY_REASON, getTestEnvironmentMetadata, getTestParametersString, finishAllTraceSpans, diff --git a/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js index 7b09f8fba2d..26dd5a7a611 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js @@ -151,6 +151,7 @@ describe('CI Visibility Exporter', () => { }) ciVisibilityExporter._resolveCanUseCiVisProtocol(true) }) + it('should request the API after EVP proxy is resolved', (done) => { const scope = nock(`http://localhost:${port}`) .post('/api/v2/libraries/tests/services/setting') @@ -160,7 +161,8 @@ describe('CI Visibility Exporter', () => { itr_enabled: true, require_git: false, code_coverage: true, - tests_skipping: true + tests_skipping: true, + known_tests_enabled: false } } })) @@ -649,34 +651,39 @@ describe('CI Visibility Exporter', () => { }) describe('getKnownTests', () => { - context('if early flake detection is disabled', () => { - it('should resolve immediately to undefined', (done) => { - const scope = nock(`http://localhost:${port}`) + context('if known tests is disabled', () => { + it('should resolve to undefined', (done) => { + const knownTestsScope = nock(`http://localhost:${port}`) .post('/api/v2/ci/libraries/tests') .reply(200) - const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: false }) + const ciVisibilityExporter = new CiVisibilityExporter({ + port + }) ciVisibilityExporter._resolveCanUseCiVisProtocol(true) + ciVisibilityExporter._libraryConfig = { isKnownTestsEnabled: false } ciVisibilityExporter.getKnownTests({}, (err, knownTests) => { expect(err).to.be.null expect(knownTests).to.eql(undefined) - expect(scope.isDone()).not.to.be.true + expect(knownTestsScope.isDone()).not.to.be.true done() }) }) }) - context('if early flake detection is enabled but can not use CI Visibility protocol', () => { + + context('if known tests is enabled but can not use CI Visibility protocol', () => { it('should not request known tests', (done) => { const scope = nock(`http://localhost:${port}`) .post('/api/v2/ci/libraries/tests') .reply(200) - const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: true }) + const ciVisibilityExporter = new CiVisibilityExporter({ port }) ciVisibilityExporter._resolveCanUseCiVisProtocol(false) - ciVisibilityExporter._libraryConfig = { isEarlyFlakeDetectionEnabled: true } + ciVisibilityExporter._libraryConfig = { isKnownTestsEnabled: true } + ciVisibilityExporter.getKnownTests({}, (err) => { expect(err).to.be.null expect(scope.isDone()).not.to.be.true @@ -684,7 +691,8 @@ describe('CI Visibility Exporter', () => { }) }) }) - context('if early flake detection is enabled and can use CI Vis Protocol', () => { + + context('if known tests is enabled and can use CI Vis Protocol', () => { it('should request known tests', (done) => { const scope = nock(`http://localhost:${port}`) .post('/api/v2/ci/libraries/tests') @@ -701,10 +709,10 @@ describe('CI Visibility Exporter', () => { } })) - const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: true }) + const ciVisibilityExporter = new CiVisibilityExporter({ port }) ciVisibilityExporter._resolveCanUseCiVisProtocol(true) - ciVisibilityExporter._libraryConfig = { isEarlyFlakeDetectionEnabled: true } + ciVisibilityExporter._libraryConfig = { isKnownTestsEnabled: true } ciVisibilityExporter.getKnownTests({}, (err, knownTests) => { expect(err).to.be.null expect(knownTests).to.eql({ @@ -717,20 +725,22 @@ describe('CI Visibility Exporter', () => { done() }) }) + it('should return an error if the request fails', (done) => { const scope = nock(`http://localhost:${port}`) .post('/api/v2/ci/libraries/tests') .reply(500) - const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: true }) + const ciVisibilityExporter = new CiVisibilityExporter({ port }) ciVisibilityExporter._resolveCanUseCiVisProtocol(true) - ciVisibilityExporter._libraryConfig = { isEarlyFlakeDetectionEnabled: true } + ciVisibilityExporter._libraryConfig = { isKnownTestsEnabled: true } ciVisibilityExporter.getKnownTests({}, (err) => { expect(err).not.to.be.null expect(scope.isDone()).to.be.true done() }) }) + it('should accept gzip if the exporter is gzip compatible', (done) => { let requestHeaders = {} const scope = nock(`http://localhost:${port}`) @@ -754,10 +764,10 @@ describe('CI Visibility Exporter', () => { 'content-encoding': 'gzip' }) - const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: true }) + const ciVisibilityExporter = new CiVisibilityExporter({ port }) ciVisibilityExporter._resolveCanUseCiVisProtocol(true) - ciVisibilityExporter._libraryConfig = { isEarlyFlakeDetectionEnabled: true } + ciVisibilityExporter._libraryConfig = { isKnownTestsEnabled: true } ciVisibilityExporter._isGzipCompatible = true ciVisibilityExporter.getKnownTests({}, (err, knownTests) => { expect(err).to.be.null @@ -772,6 +782,7 @@ describe('CI Visibility Exporter', () => { done() }) }) + it('should not accept gzip if the exporter is gzip incompatible', (done) => { let requestHeaders = {} const scope = nock(`http://localhost:${port}`) @@ -793,11 +804,10 @@ describe('CI Visibility Exporter', () => { }) }) - const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: true }) + const ciVisibilityExporter = new CiVisibilityExporter({ port }) ciVisibilityExporter._resolveCanUseCiVisProtocol(true) - ciVisibilityExporter._libraryConfig = { isEarlyFlakeDetectionEnabled: true } - + ciVisibilityExporter._libraryConfig = { isKnownTestsEnabled: true } ciVisibilityExporter._isGzipCompatible = false ciVisibilityExporter.getKnownTests({}, (err, knownTests) => {