From 2f6a4c786758916def1624cef4c55a9bfe0c0d9d Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Wed, 4 Aug 2021 11:37:09 -0400 Subject: [PATCH] [Reporting] revive "existing indices" functional test (#107537) * [Reporting] revive "existing indices" functional test Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> * fix test service Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../bwc_existing_indexes.ts | 62 +++++++++++++++++++ .../reporting_and_security/index.ts | 1 + .../services/usage.ts | 38 ++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 x-pack/test/reporting_api_integration/reporting_and_security/bwc_existing_indexes.ts diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/bwc_existing_indexes.ts b/x-pack/test/reporting_api_integration/reporting_and_security/bwc_existing_indexes.ts new file mode 100644 index 0000000000000..0da51901f9086 --- /dev/null +++ b/x-pack/test/reporting_api_integration/reporting_and_security/bwc_existing_indexes.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../ftr_provider_context'; + +/** + * This file tests the situation when a reporting index spans releases. By default reporting indexes are created + * on a weekly basis, but this is configurable so it is possible a user has this set to yearly. In that event, it + * is possible report data is getting posted to an index that was created by a very old version. We don't have a + * reporting index migration plan, so this test is important to ensure BWC, or that in the event we decide to make + * a major change in a major release, we handle it properly. + */ + +// eslint-disable-next-line import/no-default-export +export default function ({ getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const reportingAPI = getService('reportingAPI'); + + describe('BWC report generation into existing indexes', () => { + let cleanupIndexAlias: () => Promise; + + describe('existing 6_2 index', () => { + before('load data and add index alias', async () => { + await reportingAPI.deleteAllReports(); + // data to report on + await esArchiver.load('test/functional/fixtures/es_archiver/logstash_functional'); + await esArchiver.load('test/functional/fixtures/es_archiver/discover'); // includes index pattern for logstash_functional + // archive with reporting index mappings v6.2 + await esArchiver.load('x-pack/test/functional/es_archives/reporting/bwc/6_2'); + + // The index name in the reporting/bwc/6_2 archive. + const ARCHIVED_REPORTING_INDEX = '.reporting-2018.03.11'; + // causes reporting to assume the v6.2 index is the one to use for new jobs posted + cleanupIndexAlias = await reportingAPI.coerceReportsIntoExistingIndex( + ARCHIVED_REPORTING_INDEX + ); + }); + + after('remove index alias', async () => { + await esArchiver.load('test/functional/fixtures/es_archiver/logstash_functional'); + await esArchiver.load('test/functional/fixtures/es_archiver/discover'); + + await cleanupIndexAlias(); + await esArchiver.unload('x-pack/test/functional/es_archives/reporting/bwc/6_2'); + }); + + it('single job posted can complete in an index created with an older version', async () => { + const reportPaths = []; + reportPaths.push( + await reportingAPI.postJob( + '/api/reporting/generate/csv_searchsource?jobParams=%28browserTimezone%3AAmerica%2FPhoenix%2Ccolumns%3A%21%28%29%2CobjectType%3Asearch%2CsearchSource%3A%28fields%3A%21%28%28field%3A%27%2A%27%2Cinclude_unmapped%3Atrue%29%29%2Cfilter%3A%21%28%28meta%3A%28index%3A%27logstash-%2A%27%2Cparams%3A%28%29%29%2Crange%3A%28%27%40timestamp%27%3A%28format%3Astrict_date_optional_time%2Cgte%3A%272015-09-20T16%3A00%3A56.290Z%27%2Clte%3A%272015-09-21T10%3A37%3A45.066Z%27%29%29%29%29%2Cindex%3A%27logstash-%2A%27%2Cparent%3A%28filter%3A%21%28%29%2Cindex%3A%27logstash-%2A%27%2Cquery%3A%28language%3Akuery%2Cquery%3A%27%27%29%29%2Csort%3A%21%28%28%27%40timestamp%27%3Adesc%29%29%2CtrackTotalHits%3A%21t%2Cversion%3A%21t%29%2Ctitle%3A%27Discover%20search%20%5B2021-07-30T11%3A47%3A03.731-07%3A00%5D%27%29' + ) + ); + await reportingAPI.expectAllJobsToFinishSuccessfully(reportPaths); + }).timeout(1540000); + }); + }); +} diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts index d279081b5320c..2996f49857ddd 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts @@ -20,6 +20,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { await reportingAPI.createTestReportingUser(); }); + loadTestFile(require.resolve('./bwc_existing_indexes')); loadTestFile(require.resolve('./security_roles_privileges')); loadTestFile(require.resolve('./download_csv_dashboard')); loadTestFile(require.resolve('./generate_csv_discover')); diff --git a/x-pack/test/reporting_api_integration/services/usage.ts b/x-pack/test/reporting_api_integration/services/usage.ts index 4e8d464467191..0a5e0c8761eac 100644 --- a/x-pack/test/reporting_api_integration/services/usage.ts +++ b/x-pack/test/reporting_api_integration/services/usage.ts @@ -6,6 +6,7 @@ */ import expect from '@kbn/expect'; +import { indexTimestamp } from '../../../plugins/reporting/server/lib/store/index_timestamp'; import { FtrProviderContext } from '../ftr_provider_context'; interface PDFAppCounts { @@ -37,6 +38,7 @@ export interface UsageStats { export function createUsageServices({ getService }: FtrProviderContext) { const log = getService('log'); + const esSupertest = getService('esSupertest'); const supertest = getService('supertest'); return { @@ -67,6 +69,42 @@ export function createUsageServices({ getService }: FtrProviderContext) { expect(statusCode).to.be(200); }, + /** + * + * @return {Promise} A function to call to clean up the index alias that was added. + */ + async coerceReportsIntoExistingIndex(indexName: string) { + log.debug(`ReportingAPI.coerceReportsIntoExistingIndex(${indexName})`); + + // Adding an index alias coerces the report to be generated on an existing index which means any new + // index schema won't be applied. This is important if a point release updated the schema. Reports may still + // be inserted into an existing index before the new schema is applied. + const timestampForIndex = indexTimestamp('week', '.'); + await esSupertest + .post('/_aliases') + .send({ + actions: [ + { + add: { index: indexName, alias: `.reporting-${timestampForIndex}` }, + }, + ], + }) + .expect(200); + + return async () => { + await esSupertest + .post('/_aliases') + .send({ + actions: [ + { + remove: { index: indexName, alias: `.reporting-${timestampForIndex}` }, + }, + ], + }) + .expect(200); + }; + }, + async expectAllJobsToFinishSuccessfully(jobPaths: string[]) { await Promise.all( jobPaths.map(async (path) => {