From f9afb6c068ac83aa7346997ba22cfff0824c7472 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Mon, 1 Nov 2021 11:24:36 -0700 Subject: [PATCH 1/3] [Reporting] Log TM health and drift stats prior to reporting tests Closes https://github.com/elastic/kibana/issues/114946 --- .../reporting_and_security/bwc_generation_urls.ts | 3 +-- .../reporting_and_security/index.ts | 1 + .../reporting_without_security/index.ts | 6 +++++- .../services/scenarios.ts | 15 +++++++++++++++ .../reporting_and_deprecated_security/index.ts | 2 ++ .../reporting_and_security/index.ts | 1 + .../reporting_without_security/index.ts | 6 ++++++ 7 files changed, 31 insertions(+), 3 deletions(-) diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/bwc_generation_urls.ts b/x-pack/test/reporting_api_integration/reporting_and_security/bwc_generation_urls.ts index 03e1592df0818..fd194a1df1f65 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/bwc_generation_urls.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/bwc_generation_urls.ts @@ -36,8 +36,7 @@ export default function ({ getService }: FtrProviderContext) { await reportingAPI.deleteAllReports(); }); - // FLAKY: https://github.com/elastic/kibana/issues/93354 - describe.skip('Pre 6_2', () => { + describe('Pre 6_2', () => { // The URL being tested was captured from release 6.4 and then the layout section was removed to test structure before // preserve_layout was introduced. See https://github.com/elastic/kibana/issues/23414 it('job posted successfully', async () => { diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts index f6654ff5a6b1d..4303cbf076920 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts @@ -14,6 +14,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { before(async () => { const reportingAPI = getService('reportingAPI'); + await reportingAPI.checkTaskManagerHealth(); await reportingAPI.createDataAnalystRole(); await reportingAPI.createTestReportingUserRole(); await reportingAPI.createDataAnalyst(); diff --git a/x-pack/test/reporting_api_integration/reporting_without_security/index.ts b/x-pack/test/reporting_api_integration/reporting_without_security/index.ts index 81ca3e05e4dd0..eaef51efa48f4 100644 --- a/x-pack/test/reporting_api_integration/reporting_without_security/index.ts +++ b/x-pack/test/reporting_api_integration/reporting_without_security/index.ts @@ -8,8 +8,12 @@ import { FtrProviderContext } from '../ftr_provider_context'; // eslint-disable-next-line import/no-default-export -export default function ({ loadTestFile }: FtrProviderContext) { +export default function ({ loadTestFile, getService }: FtrProviderContext) { describe('Reporting API Integration Tests with Security disabled', function () { + before(async () => { + const reportingAPI = getService('reportingAPI'); + await reportingAPI.checkTaskManagerHealth(); + }); this.tags('ciGroup13'); loadTestFile(require.resolve('./job_apis_csv')); loadTestFile(require.resolve('./job_apis_csv_deprecated')); diff --git a/x-pack/test/reporting_api_integration/services/scenarios.ts b/x-pack/test/reporting_api_integration/services/scenarios.ts index e39a3e2e5954b..54d99d25a6358 100644 --- a/x-pack/test/reporting_api_integration/services/scenarios.ts +++ b/x-pack/test/reporting_api_integration/services/scenarios.ts @@ -36,6 +36,20 @@ export function createScenarios({ getService }: Pick { + // Check task manager health for analyzing test failures. See https://github.com/elastic/kibana/issues/114946 + const tmHealth = await supertest.get(`/api/task_manager/_health`); + const driftValues = tmHealth.body?.stats?.runtime?.value; + + log.info(`Task Manager status: "${tmHealth.body?.status}"`); + log.info(`Task Manager overall drift rankings: "${JSON.stringify(driftValues?.drift)}"`); + log.info( + `Task Manager drift rankings for "report:execute": "${JSON.stringify( + driftValues?.drift_by_type?.['report:execute'] + )}"` + ); + }; + const initEcommerce = async () => { await esArchiver.load('x-pack/test/functional/es_archives/reporting/ecommerce'); await kibanaServer.importExport.load(ecommerceSOPath); @@ -201,6 +215,7 @@ export function createScenarios({ getService }: Pick { + const reportingAPI = context.getService('reportingAPI'); + await reportingAPI.checkTaskManagerHealth(); await createDataAnalystRole(); await createDataAnalyst(); await createReportingUser(); diff --git a/x-pack/test/reporting_functional/reporting_and_security/index.ts b/x-pack/test/reporting_functional/reporting_and_security/index.ts index be0e76a28bd0b..cf59d963a55b5 100644 --- a/x-pack/test/reporting_functional/reporting_and_security/index.ts +++ b/x-pack/test/reporting_functional/reporting_and_security/index.ts @@ -14,6 +14,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { before(async () => { const reportingFunctional = getService('reportingFunctional'); + await reportingFunctional.checkTaskManagerHealth(); await reportingFunctional.createDataAnalystRole(); await reportingFunctional.createDataAnalyst(); await reportingFunctional.createTestReportingUserRole(); diff --git a/x-pack/test/reporting_functional/reporting_without_security/index.ts b/x-pack/test/reporting_functional/reporting_without_security/index.ts index d1801b7e3e2e6..68fa736765489 100644 --- a/x-pack/test/reporting_functional/reporting_without_security/index.ts +++ b/x-pack/test/reporting_functional/reporting_without_security/index.ts @@ -11,6 +11,12 @@ import { FtrProviderContext } from '../ftr_provider_context'; export default function ({ loadTestFile, getService }: FtrProviderContext) { describe('Reporting Functional Tests with Security disabled', function () { this.tags('ciGroup2'); + + before(async () => { + const reportingAPI = getService('reportingAPI'); + await reportingAPI.checkTaskManagerHealth(); + }); + loadTestFile(require.resolve('./management')); }); } From 3539c15bb3ac2d30c85c82256444a4c65fe5467e Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Tue, 2 Nov 2021 11:54:53 -0700 Subject: [PATCH 2/3] re-skip flaky test --- .../reporting_and_security/bwc_generation_urls.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/bwc_generation_urls.ts b/x-pack/test/reporting_api_integration/reporting_and_security/bwc_generation_urls.ts index fd194a1df1f65..03e1592df0818 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/bwc_generation_urls.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/bwc_generation_urls.ts @@ -36,7 +36,8 @@ export default function ({ getService }: FtrProviderContext) { await reportingAPI.deleteAllReports(); }); - describe('Pre 6_2', () => { + // FLAKY: https://github.com/elastic/kibana/issues/93354 + describe.skip('Pre 6_2', () => { // The URL being tested was captured from release 6.4 and then the layout section was removed to test structure before // preserve_layout was introduced. See https://github.com/elastic/kibana/issues/23414 it('job posted successfully', async () => { From 217afd6406f3522fac96d1b7c7a082d65e8c7532 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Tue, 2 Nov 2021 11:56:45 -0700 Subject: [PATCH 3/3] rename method --- .../reporting_api_integration/reporting_and_security/index.ts | 2 +- .../reporting_without_security/index.ts | 2 +- x-pack/test/reporting_api_integration/services/scenarios.ts | 4 ++-- .../reporting_and_deprecated_security/index.ts | 2 +- .../test/reporting_functional/reporting_and_security/index.ts | 2 +- .../reporting_functional/reporting_without_security/index.ts | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts index 4303cbf076920..6ea6de3482501 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts @@ -14,7 +14,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { before(async () => { const reportingAPI = getService('reportingAPI'); - await reportingAPI.checkTaskManagerHealth(); + await reportingAPI.logTaskManagerHealth(); await reportingAPI.createDataAnalystRole(); await reportingAPI.createTestReportingUserRole(); await reportingAPI.createDataAnalyst(); diff --git a/x-pack/test/reporting_api_integration/reporting_without_security/index.ts b/x-pack/test/reporting_api_integration/reporting_without_security/index.ts index eaef51efa48f4..258ae814f5789 100644 --- a/x-pack/test/reporting_api_integration/reporting_without_security/index.ts +++ b/x-pack/test/reporting_api_integration/reporting_without_security/index.ts @@ -12,7 +12,7 @@ export default function ({ loadTestFile, getService }: FtrProviderContext) { describe('Reporting API Integration Tests with Security disabled', function () { before(async () => { const reportingAPI = getService('reportingAPI'); - await reportingAPI.checkTaskManagerHealth(); + await reportingAPI.logTaskManagerHealth(); }); this.tags('ciGroup13'); loadTestFile(require.resolve('./job_apis_csv')); diff --git a/x-pack/test/reporting_api_integration/services/scenarios.ts b/x-pack/test/reporting_api_integration/services/scenarios.ts index 54d99d25a6358..0886deac3a13a 100644 --- a/x-pack/test/reporting_api_integration/services/scenarios.ts +++ b/x-pack/test/reporting_api_integration/services/scenarios.ts @@ -36,7 +36,7 @@ export function createScenarios({ getService }: Pick { + const logTaskManagerHealth = async () => { // Check task manager health for analyzing test failures. See https://github.com/elastic/kibana/issues/114946 const tmHealth = await supertest.get(`/api/task_manager/_health`); const driftValues = tmHealth.body?.stats?.runtime?.value; @@ -215,7 +215,7 @@ export function createScenarios({ getService }: Pick { const reportingAPI = context.getService('reportingAPI'); - await reportingAPI.checkTaskManagerHealth(); + await reportingAPI.logTaskManagerHealth(); await createDataAnalystRole(); await createDataAnalyst(); await createReportingUser(); diff --git a/x-pack/test/reporting_functional/reporting_and_security/index.ts b/x-pack/test/reporting_functional/reporting_and_security/index.ts index cf59d963a55b5..22057c9be77dc 100644 --- a/x-pack/test/reporting_functional/reporting_and_security/index.ts +++ b/x-pack/test/reporting_functional/reporting_and_security/index.ts @@ -14,7 +14,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { before(async () => { const reportingFunctional = getService('reportingFunctional'); - await reportingFunctional.checkTaskManagerHealth(); + await reportingFunctional.logTaskManagerHealth(); await reportingFunctional.createDataAnalystRole(); await reportingFunctional.createDataAnalyst(); await reportingFunctional.createTestReportingUserRole(); diff --git a/x-pack/test/reporting_functional/reporting_without_security/index.ts b/x-pack/test/reporting_functional/reporting_without_security/index.ts index 68fa736765489..fecc0e97daac0 100644 --- a/x-pack/test/reporting_functional/reporting_without_security/index.ts +++ b/x-pack/test/reporting_functional/reporting_without_security/index.ts @@ -14,7 +14,7 @@ export default function ({ loadTestFile, getService }: FtrProviderContext) { before(async () => { const reportingAPI = getService('reportingAPI'); - await reportingAPI.checkTaskManagerHealth(); + await reportingAPI.logTaskManagerHealth(); }); loadTestFile(require.resolve('./management'));