From e98b65995a5eff146b0c002731216202e8f2f0d8 Mon Sep 17 00:00:00 2001 From: dkirchan Date: Thu, 17 Oct 2024 16:18:11 +0300 Subject: [PATCH 1/9] dbg commit --- .../mki_security_solution_cypress.sh | 2 +- .../failed_tests_reporter/failed_tests_reporter_cli.ts | 5 +++++ .../cypress/cypress_ci_serverless_qa.config.ts | 3 ++- .../cypress/e2e/explore/overview/overview.cy.ts | 8 ++++++++ 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh b/.buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh index 0032cd93966ad..07caff0f40783 100755 --- a/.buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh +++ b/.buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh @@ -13,7 +13,7 @@ source .buildkite/scripts/common/util.sh export JOB=kibana-security-solution-chrome -buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true" +buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" 'true' source .buildkite/scripts/pipelines/security_solution_quality_gate/prepare_vault_entries.sh diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts index d86f7f5213125..45b325d611f42 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts @@ -33,6 +33,8 @@ const DISABLE_MISSING_TEST_REPORT_ERRORS = run( async ({ log, flags }) => { + + console.log(`log - ${JSON.stringify(log)} : flags - ${JSON.stringify(flags)}`); const indexInEs = flags['index-errors']; let updateGithub = flags['github-update']; @@ -44,6 +46,8 @@ run( let branch: string = ''; let pipeline: string = ''; + console.log(`branch - ${branch} : pipeline - ${pipeline}`); + console.log(`!updateGithub - ${!updateGithub}`); if (updateGithub) { let isPr = false; @@ -155,6 +159,7 @@ run( } const newIssue = await createFailureIssue(buildUrl, failure, githubApi, branch, pipeline); + console.log(`Issue : ${JSON.stringify(newIssue)}`) existingIssues.addNewlyCreated(failure, newIssue); pushMessage('Test has not failed recently on tracked branches'); if (updateGithub) { diff --git a/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts b/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts index c2ad97b6ddb05..2ae73b19e5cfc 100644 --- a/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts +++ b/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts @@ -20,7 +20,8 @@ export default defineCypressConfig({ env: { grepFilterSpecs: true, grepOmitFiltered: true, - grepTags: '@serverless --@skipInServerless --@skipInServerlessMKI', + grepTags: '@testhere', + // grepTags: '@serverless --@skipInServerless --@skipInServerlessMKI', }, execTimeout: 300000, pageLoadTimeout: 300000, diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts index 78135fbd77235..af8ddf4aaff82 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts @@ -32,6 +32,14 @@ describe('Overview Page', { tags: ['@ess', '@serverless'] }, () => { cy.task('esArchiverUnload', { archiveName: 'overview' }); }); + it('Expected failure', { tags: ['@testhere'] }, () => { + expandHostStats(); + + HOST_STATS.forEach((stat) => { + cy.get(stat.domId).should('have.text', 'a random text to fail the test'); + }); + }); + it('Host stats render with correct values', () => { expandHostStats(); From 32a30d30a2f42250f84090620bd09b415d366ed5 Mon Sep 17 00:00:00 2001 From: dkirchan Date: Fri, 18 Oct 2024 12:57:18 +0300 Subject: [PATCH 2/9] Removed console logs except from the ticket created --- .../failed_tests_reporter/failed_tests_reporter_cli.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts index 45b325d611f42..6a239707aad82 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts @@ -33,8 +33,6 @@ const DISABLE_MISSING_TEST_REPORT_ERRORS = run( async ({ log, flags }) => { - - console.log(`log - ${JSON.stringify(log)} : flags - ${JSON.stringify(flags)}`); const indexInEs = flags['index-errors']; let updateGithub = flags['github-update']; @@ -46,8 +44,6 @@ run( let branch: string = ''; let pipeline: string = ''; - console.log(`branch - ${branch} : pipeline - ${pipeline}`); - console.log(`!updateGithub - ${!updateGithub}`); if (updateGithub) { let isPr = false; From 2b795e89ad1ef6934a34e85404a40a7a684e05d8 Mon Sep 17 00:00:00 2001 From: dkirchan Date: Fri, 18 Oct 2024 15:09:10 +0300 Subject: [PATCH 3/9] Added prepended title reference --- .../failed_tests_reporter_cli.ts | 2 + .../report_failure.test.ts | 47 +++++++++++++++++++ .../failed_tests_reporter/report_failure.ts | 10 +++- 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts index 6a239707aad82..de5a865494d9a 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts @@ -44,6 +44,7 @@ run( let branch: string = ''; let pipeline: string = ''; + let prependTitle: string = ''; if (updateGithub) { let isPr = false; @@ -52,6 +53,7 @@ run( pipeline = process.env.BUILDKITE_PIPELINE_SLUG || ''; isPr = process.env.BUILDKITE_PULL_REQUEST === 'true'; updateGithub = process.env.REPORT_FAILED_TESTS_TO_GITHUB === 'true'; + prependTitle = process.env.PREPEND_FAILURE_TITLE || ''; } else { // JOB_NAME is formatted as `elastic+kibana+7.x` in some places and `elastic+kibana+7.x/JOB=kibana-intake,node=immutable` in others const jobNameSplit = (process.env.JOB_NAME || '').split(/\+|\//); diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failure.test.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failure.test.ts index dce47461377c4..c7dbd2db77319 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failure.test.ts +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failure.test.ts @@ -60,6 +60,53 @@ describe('createFailureIssue()', () => { } `); }); + + it('creates new github issue with title prepended', async () => { + const api = new GithubApi(); + + await createFailureIssue( + 'https://build-url', + { + classname: 'some.classname', + failure: 'this is the failure text', + name: 'test name', + time: '2018-01-01T01:00:00Z', + likelyIrrelevant: false, + }, + api, + 'main', + 'kibana-on-merge', + '[MKI][QA]' + ); + + expect(api.createIssue).toMatchInlineSnapshot(` + [MockFunction] { + "calls": Array [ + Array [ + "Failing test: [MKI][QA] some.classname - test name", + "A test failed on a tracked branch + + \`\`\` + this is the failure text + \`\`\` + + First failure: [kibana-on-merge - main](https://build-url) + + ", + Array [ + "failed-test", + ], + ], + ], + "results": Array [ + Object { + "type": "return", + "value": undefined, + }, + ], + } + `); + }); }); describe('updateFailureIssue()', () => { diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failure.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failure.ts index 182bde666c250..d941f94b6b24d 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failure.ts +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failure.ts @@ -17,9 +17,15 @@ export async function createFailureIssue( failure: TestFailure, api: GithubApi, branch: string, - pipeline: string + pipeline: string, + prependTitle: string = '' ) { - const title = `Failing test: ${failure.classname} - ${failure.name}`; + // PrependTitle is introduced to provide some clarity by prepending the failing test title + // in order to give the whole info in the title according to each team's preference. + const title = + prependTitle && prependTitle.trim() !== '' + ? `Failing test: ${prependTitle} ${failure.classname} - ${failure.name}` + : `Failing test: ${failure.classname} - ${failure.name}`; // Github API body length maximum is 65536 characters // Let's keep consistency with Mocha output that is truncated to 8192 characters From 525c0bb60318da3bb7ce6afbceca0a4904f67aaf Mon Sep 17 00:00:00 2001 From: dkirchan Date: Fri, 18 Oct 2024 15:14:19 +0300 Subject: [PATCH 4/9] Renamed the failed test --- .../failed_tests_reporter/failed_tests_reporter_cli.ts | 2 +- .../cypress/e2e/explore/overview/overview.cy.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts index de5a865494d9a..d224aa29d13f3 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts @@ -157,7 +157,7 @@ run( } const newIssue = await createFailureIssue(buildUrl, failure, githubApi, branch, pipeline); - console.log(`Issue : ${JSON.stringify(newIssue)}`) + console.log(`Issue : ${JSON.stringify(newIssue)}`); existingIssues.addNewlyCreated(failure, newIssue); pushMessage('Test has not failed recently on tracked branches'); if (updateGithub) { diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts index af8ddf4aaff82..dd85d7591f190 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts @@ -32,7 +32,7 @@ describe('Overview Page', { tags: ['@ess', '@serverless'] }, () => { cy.task('esArchiverUnload', { archiveName: 'overview' }); }); - it('Expected failure', { tags: ['@testhere'] }, () => { + it('Expected failures', { tags: ['@testhere'] }, () => { expandHostStats(); HOST_STATS.forEach((stat) => { From cee64635a38f577a555b11d6fb84d6f7adc89ce8 Mon Sep 17 00:00:00 2001 From: dkirchan Date: Fri, 18 Oct 2024 16:13:08 +0300 Subject: [PATCH 5/9] Added prependTitle in the method call --- .../failed_tests_reporter/failed_tests_reporter_cli.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts index d224aa29d13f3..d2d9ec45801b8 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts @@ -156,7 +156,14 @@ run( continue; } - const newIssue = await createFailureIssue(buildUrl, failure, githubApi, branch, pipeline); + const newIssue = await createFailureIssue( + buildUrl, + failure, + githubApi, + branch, + pipeline, + prependTitle + ); console.log(`Issue : ${JSON.stringify(newIssue)}`); existingIssues.addNewlyCreated(failure, newIssue); pushMessage('Test has not failed recently on tracked branches'); From e36fbfacdd657f38ffceb307c1f6b570e8307df5 Mon Sep 17 00:00:00 2001 From: dkirchan Date: Fri, 18 Oct 2024 16:14:23 +0300 Subject: [PATCH 6/9] Changed test name --- .../scripts/junit_transformer/lib.ts | 38 +++++++++++++++++++ .../e2e/explore/overview/overview.cy.ts | 2 +- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/scripts/junit_transformer/lib.ts b/x-pack/plugins/security_solution/scripts/junit_transformer/lib.ts index 725baa689cd20..e09a057978bce 100644 --- a/x-pack/plugins/security_solution/scripts/junit_transformer/lib.ts +++ b/x-pack/plugins/security_solution/scripts/junit_transformer/lib.ts @@ -18,6 +18,20 @@ import { PathReporter } from 'io-ts/lib/PathReporter'; import globby from 'globby'; import del from 'del'; +// Function to remove specific fields from an XML object in order to +// compare them as strings. +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function removeFields(obj: any, fieldsToRemove: string[]): any { + for (const key in obj) { + if (fieldsToRemove.includes(key)) { + delete obj[key]; + } else if (typeof obj[key] === 'object') { + obj[key] = removeFields(obj[key], fieldsToRemove); // Recursively remove fields + } + } + return obj; +} + /** * Updates the `name` and `classname` attributes of each testcase. * `name` will have the value of `classname` appended to it. This makes sense because they each contain part of the bdd spec. @@ -174,6 +188,10 @@ export async function command({ flags, log }: CommandArgs) { throw createFlagError('please provide a single --reportName flag'); } + // Going to be used in order to store results as string in order to compare + // and remove duplicated reports. + const xmlResultFiles: string[] = []; + for (const path of await globby(flags.pathPattern)) { // Read the file const source: string = await fs.readFile(path, 'utf8'); @@ -242,6 +260,26 @@ ${boilerplate} rootDirectory: flags.rootDirectory, }); + // We need to check if a XML Junit report is duplicate + // Only if we remove the time and timestamp and the rest of the + // report as a string is completely identical. + const fieldsToRemove = ['time', 'timestamp']; + const tempReport = await parseStringPromise(reportString); + const truncatedReport = removeFields(tempReport, fieldsToRemove); + + // Rebuild the XML to compare (optional, if you want to compare XML strings) + const builder = new Builder(); + const rebuildComparableReport = builder.buildObject(truncatedReport); + + // Compare the cleaned and rebuilt XML objects or strings + if (xmlResultFiles.includes(rebuildComparableReport)) { + // If the report is a duplicate, we need to remove the file + // in order to be excluded from the uploaded results. + await del(path, { force: true }); + continue; + } + xmlResultFiles.push(rebuildComparableReport); + // If the writeInPlace flag was passed, overwrite the original file, otherwise log the output to stdout if (flags.writeInPlace) { log.info(`Wrote transformed junit report to ${path}`); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts index dd85d7591f190..aea094e09a75f 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts @@ -32,7 +32,7 @@ describe('Overview Page', { tags: ['@ess', '@serverless'] }, () => { cy.task('esArchiverUnload', { archiveName: 'overview' }); }); - it('Expected failures', { tags: ['@testhere'] }, () => { + it('Expected failures for ghub ticket test reason', { tags: ['@testhere'] }, () => { expandHostStats(); HOST_STATS.forEach((stat) => { From d4346f16ad3cf5c1bd01aae29d3f153417858a0a Mon Sep 17 00:00:00 2001 From: dkirchan Date: Tue, 22 Oct 2024 17:27:23 +0300 Subject: [PATCH 7/9] removed console statement --- .../failed_tests_reporter/failed_tests_reporter_cli.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts index d2d9ec45801b8..36466c3e3637e 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts @@ -164,7 +164,6 @@ run( pipeline, prependTitle ); - console.log(`Issue : ${JSON.stringify(newIssue)}`); existingIssues.addNewlyCreated(failure, newIssue); pushMessage('Test has not failed recently on tracked branches'); if (updateGithub) { From 0bb0af72b67b67c22dc1c69e6612672aadff54fb Mon Sep 17 00:00:00 2001 From: dkirchan Date: Tue, 22 Oct 2024 17:30:35 +0300 Subject: [PATCH 8/9] Removed testhere tag and test --- .../cypress/cypress_ci_serverless_qa.config.ts | 3 +-- .../cypress/e2e/explore/overview/overview.cy.ts | 8 -------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts b/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts index 2ae73b19e5cfc..c2ad97b6ddb05 100644 --- a/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts +++ b/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts @@ -20,8 +20,7 @@ export default defineCypressConfig({ env: { grepFilterSpecs: true, grepOmitFiltered: true, - grepTags: '@testhere', - // grepTags: '@serverless --@skipInServerless --@skipInServerlessMKI', + grepTags: '@serverless --@skipInServerless --@skipInServerlessMKI', }, execTimeout: 300000, pageLoadTimeout: 300000, diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts index aea094e09a75f..78135fbd77235 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts @@ -32,14 +32,6 @@ describe('Overview Page', { tags: ['@ess', '@serverless'] }, () => { cy.task('esArchiverUnload', { archiveName: 'overview' }); }); - it('Expected failures for ghub ticket test reason', { tags: ['@testhere'] }, () => { - expandHostStats(); - - HOST_STATS.forEach((stat) => { - cy.get(stat.domId).should('have.text', 'a random text to fail the test'); - }); - }); - it('Host stats render with correct values', () => { expandHostStats(); From 84881402207302131fec37fed72c0400c32657ef Mon Sep 17 00:00:00 2001 From: dkirchan Date: Tue, 22 Oct 2024 17:32:27 +0300 Subject: [PATCH 9/9] Reverted quotes change --- .../security_solution_cypress/mki_security_solution_cypress.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh b/.buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh index 07caff0f40783..0032cd93966ad 100755 --- a/.buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh +++ b/.buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh @@ -13,7 +13,7 @@ source .buildkite/scripts/common/util.sh export JOB=kibana-security-solution-chrome -buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" 'true' +buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true" source .buildkite/scripts/pipelines/security_solution_quality_gate/prepare_vault_entries.sh