diff --git a/src/core/server/integration_tests/saved_objects/migrations/archives/7.13.0_concurrent_5k_foo.zip b/src/core/server/integration_tests/saved_objects/migrations/archives/7.13.0_concurrent_5k_foo.zip deleted file mode 100644 index d079624d95988..0000000000000 Binary files a/src/core/server/integration_tests/saved_objects/migrations/archives/7.13.0_concurrent_5k_foo.zip and /dev/null differ diff --git a/src/core/server/integration_tests/saved_objects/migrations/archives/7.14.0_xpack_sample_saved_objects.zip b/src/core/server/integration_tests/saved_objects/migrations/archives/7.14.0_xpack_sample_saved_objects.zip deleted file mode 100644 index 7b498c945680c..0000000000000 Binary files a/src/core/server/integration_tests/saved_objects/migrations/archives/7.14.0_xpack_sample_saved_objects.zip and /dev/null differ diff --git a/src/core/server/integration_tests/saved_objects/migrations/archives/7.3.0_xpack_sample_saved_objects.zip b/src/core/server/integration_tests/saved_objects/migrations/archives/7.3.0_xpack_sample_saved_objects.zip deleted file mode 100644 index b79a497d06941..0000000000000 Binary files a/src/core/server/integration_tests/saved_objects/migrations/archives/7.3.0_xpack_sample_saved_objects.zip and /dev/null differ diff --git a/src/core/server/integration_tests/saved_objects/migrations/archives/7.7.2_xpack_100k_obj.zip b/src/core/server/integration_tests/saved_objects/migrations/archives/7.7.2_xpack_100k_obj.zip deleted file mode 100644 index 68d740dd21f69..0000000000000 Binary files a/src/core/server/integration_tests/saved_objects/migrations/archives/7.7.2_xpack_100k_obj.zip and /dev/null differ diff --git a/src/core/server/integration_tests/saved_objects/migrations/archives/8.4.0_with_sample_data_logs.zip b/src/core/server/integration_tests/saved_objects/migrations/archives/8.4.0_with_sample_data_logs.zip index 4304e138f161a..7d81fc53b7477 100644 Binary files a/src/core/server/integration_tests/saved_objects/migrations/archives/8.4.0_with_sample_data_logs.zip and b/src/core/server/integration_tests/saved_objects/migrations/archives/8.4.0_with_sample_data_logs.zip differ diff --git a/src/core/server/integration_tests/saved_objects/migrations/group1/v2_migration.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group1/v2_migration.test.ts index 5981c2759d5ca..b6ee91ceffbb1 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group1/v2_migration.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group1/v2_migration.test.ts @@ -22,18 +22,20 @@ import { readLog, clearLog, currentVersion, + nextMinor, } from '../kibana_migrator_test_kit'; import { + BASELINE_COMPLEX_DOCUMENTS_500K_AFTER, BASELINE_DOCUMENTS_PER_TYPE_500K, BASELINE_TEST_ARCHIVE_500K, } from '../kibana_migrator_archive_utils'; import { - baselineTypes, getReindexingBaselineTypes, getReindexingMigratorTestKit, getUpToDateMigratorTestKit, } from '../kibana_migrator_test_kit.fixtures'; import { delay } from '../test_utils'; +import { expectDocumentsMigratedToHighestVersion } from '../kibana_migrator_test_kit.expect'; const logFilePath = join(__dirname, 'v2_migration.log'); @@ -83,11 +85,11 @@ describe('v2 migration', () => { expect(migrationResults.map((result) => omit(result, 'elapsedMs'))).toMatchInlineSnapshot(` Array [ Object { - "destIndex": ".kibana_migrator_9.0.0_001", + "destIndex": ".kibana_migrator_${currentVersion}_001", "status": "patched", }, Object { - "destIndex": ".kibana_migrator_tasks_9.0.0_001", + "destIndex": ".kibana_migrator_tasks_${currentVersion}_001", "status": "patched", }, ] @@ -95,11 +97,10 @@ describe('v2 migration', () => { }); it('each migrator takes less than 10 seconds', () => { - expect( - (migrationResults as Array<{ elapsedMs?: number }>).every( - ({ elapsedMs }) => !elapsedMs || elapsedMs < 10000 - ) - ).toEqual(true); + const painfulMigrator = (migrationResults as Array<{ elapsedMs?: number }>).find( + ({ elapsedMs }) => elapsedMs && elapsedMs > 10_000 + ); + expect(painfulMigrator).toBeUndefined(); }); }); @@ -214,38 +215,10 @@ describe('v2 migration', () => { }); it('migrates documents to the highest version', async () => { - const typeMigrationVersions: Record = { - basic: '10.1.0', // did not define any model versions - complex: '10.2.0', - task: '10.2.0', - }; - - const resultSets = await Promise.all( - baselineTypes.map(({ name: type }) => - kit.client.search({ - index: [defaultKibanaIndex, defaultKibanaTaskIndex], - query: { - bool: { - should: [ - { - term: { type }, - }, - ], - }, - }, - }) - ) - ); - - expect( - resultSets - .flatMap((result) => result.hits.hits) - .every( - (document) => - document._source.typeMigrationVersion === - typeMigrationVersions[document._source.type] - ) - ).toEqual(true); + await expectDocumentsMigratedToHighestVersion(kit.client, [ + defaultKibanaIndex, + defaultKibanaTaskIndex, + ]); }); describe('a migrator performing a compatible upgrade migration', () => { @@ -338,11 +311,7 @@ describe('v2 migration', () => { }); it('executes the excludeOnUpgrade hook', () => { - // we discard the second half with exclude on upgrade (firstHalf !== true) - // then we discard half all multiples of 100 (1% of them) - expect(primaryIndexCounts.complex).toEqual( - BASELINE_DOCUMENTS_PER_TYPE_500K / 2 - BASELINE_DOCUMENTS_PER_TYPE_500K / 2 / 100 - ); + expect(primaryIndexCounts.complex).toEqual(BASELINE_COMPLEX_DOCUMENTS_500K_AFTER); }); }); @@ -352,13 +321,13 @@ describe('v2 migration', () => { .toMatchInlineSnapshot(` Array [ Object { - "destIndex": ".kibana_migrator_9.1.0_001", - "sourceIndex": ".kibana_migrator_9.0.0_001", + "destIndex": ".kibana_migrator_${nextMinor}_001", + "sourceIndex": ".kibana_migrator_${currentVersion}_001", "status": "migrated", }, Object { - "destIndex": ".kibana_migrator_tasks_9.0.0_001", - "sourceIndex": ".kibana_migrator_tasks_9.0.0_001", + "destIndex": ".kibana_migrator_tasks_${currentVersion}_001", + "sourceIndex": ".kibana_migrator_tasks_${currentVersion}_001", "status": "migrated", }, ] @@ -366,11 +335,10 @@ describe('v2 migration', () => { }); it('each migrator takes less than 60 seconds', () => { - expect( - (migrationResults as Array<{ elapsedMs?: number }>).every( - ({ elapsedMs }) => !elapsedMs || elapsedMs < 60000 - ) - ).toEqual(true); + const painfulMigrator = (migrationResults as Array<{ elapsedMs?: number }>).find( + ({ elapsedMs }) => elapsedMs && elapsedMs > 60_000 + ); + expect(painfulMigrator).toBeUndefined(); }); }); }); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/multiple_kb_nodes.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/multiple_kb_nodes.test.ts new file mode 100644 index 0000000000000..1ad78f5abb666 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/group2/multiple_kb_nodes.test.ts @@ -0,0 +1,290 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { join } from 'path'; +import { omit, sortBy } from 'lodash'; +import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import type { MigrationResult } from '@kbn/core-saved-objects-base-server-internal'; +import type { Client } from '@elastic/elasticsearch'; +import { + clearLog, + defaultKibanaIndex, + defaultKibanaTaskIndex, + getAggregatedTypesCount, + getEsClient, + nextMinor, + startElasticsearch, +} from '../kibana_migrator_test_kit'; +import { + BASELINE_COMPLEX_DOCUMENTS_500K_AFTER, + BASELINE_DOCUMENTS_PER_TYPE_500K, + BASELINE_TEST_ARCHIVE_500K, +} from '../kibana_migrator_archive_utils'; +import { + getRelocatingMigratorTestKit, + kibanaSplitIndex, +} from '../kibana_migrator_test_kit.fixtures'; +import { delay, parseLogFile } from '../test_utils'; +import '../jest_matchers'; +import { expectDocumentsMigratedToHighestVersion } from '../kibana_migrator_test_kit.expect'; + +const PARALLEL_MIGRATORS = 4; +type Job = () => Promise; + +const getLogFile = (node: number) => join(__dirname, `multiple_kb_nodes_${node}.log`); +const logFileSecondRun = join(__dirname, `multiple_kb_nodes_second_run.log`); + +describe('multiple Kibana nodes performing a reindexing migration', () => { + jest.setTimeout(1200000); // costly test + let esServer: TestElasticsearchUtils['es']; + let client: Client; + let results: MigrationResult[][]; + + beforeEach(async () => { + for (let i = 0; i < PARALLEL_MIGRATORS; ++i) { + await clearLog(getLogFile(i)); + } + await clearLog(logFileSecondRun); + + esServer = await startElasticsearch({ dataArchive: BASELINE_TEST_ARCHIVE_500K }); + client = await getEsClient(); + await checkBeforeState(); + }); + + it.each([ + { + case: 'migrate saved objects normally when started at the same time', + delaySeconds: 0, + }, + { + case: 'migrate saved objects normally when started with a small interval', + delaySeconds: 1, + }, + { + case: 'migrate saved objects normally when started with an average interval', + delaySeconds: 5, + }, + { + case: 'migrate saved objects normally when started with a bigger interval', + delaySeconds: 20, + }, + ])('$case', async ({ delaySeconds }) => { + const jobs = await createMigratorJobs(PARALLEL_MIGRATORS); + results = await startWithDelay(jobs, delaySeconds); + checkMigratorsResults(); + await checkIndicesInfo(); + await checkSavedObjectDocuments(); + await checkFirstNodeSteps(); + await checkUpToDateOnRestart(); + }); + + afterEach(async () => { + await esServer?.stop(); + await delay(5); // give it a few seconds... cause we always do ¯\_(ツ)_/¯ + }); + + async function checkBeforeState() { + await expect(getAggregatedTypesCount(client, [defaultKibanaIndex])).resolves.toEqual({ + basic: BASELINE_DOCUMENTS_PER_TYPE_500K, + complex: BASELINE_DOCUMENTS_PER_TYPE_500K, + deprecated: BASELINE_DOCUMENTS_PER_TYPE_500K, + server: BASELINE_DOCUMENTS_PER_TYPE_500K, + }); + await expect(getAggregatedTypesCount(client, [defaultKibanaTaskIndex])).resolves.toEqual({ + task: BASELINE_DOCUMENTS_PER_TYPE_500K, + }); + await expect(getAggregatedTypesCount(client, [kibanaSplitIndex])).resolves.toEqual({}); + } + + function checkMigratorsResults() { + const flatResults = results.flat(); // multiple nodes, multiple migrators each + + // each migrator should take less than 120 seconds + const painfulMigrator = (flatResults as Array<{ elapsedMs?: number }>).find( + ({ elapsedMs }) => elapsedMs && elapsedMs > 120_000 + ); + expect(painfulMigrator).toBeUndefined(); + + // each migrator has either migrated or patched + const failedMigrator = flatResults.find( + ({ status }) => status !== 'migrated' && status !== 'patched' + ); + expect(failedMigrator).toBeUndefined(); + } + + async function checkIndicesInfo() { + const indicesInfo = await client.indices.get({ index: '.kibana*' }); + [defaultKibanaIndex, kibanaSplitIndex].forEach((index) => + expect(indicesInfo[`${index}_${nextMinor}_001`]).toEqual( + expect.objectContaining({ + aliases: expect.objectContaining({ [index]: expect.any(Object) }), + mappings: { + dynamic: 'strict', + _meta: { + mappingVersions: expect.any(Object), + indexTypesMap: expect.any(Object), + }, + properties: expect.any(Object), + }, + settings: { index: expect.any(Object) }, + }) + ) + ); + + const typesMap = + indicesInfo[`${defaultKibanaIndex}_${nextMinor}_001`].mappings?._meta?.indexTypesMap; + expect(typesMap[defaultKibanaIndex]).toEqual(['complex', 'server']); // 'deprecated' no longer present + expect(typesMap[kibanaSplitIndex]).toEqual(['basic', 'task']); + } + + async function checkSavedObjectDocuments() { + // check documents have been migrated + await expect(getAggregatedTypesCount(client, [defaultKibanaIndex])).resolves.toEqual({ + complex: BASELINE_COMPLEX_DOCUMENTS_500K_AFTER, + }); + await expect(getAggregatedTypesCount(client, [defaultKibanaTaskIndex])).resolves.toEqual({}); + await expect(getAggregatedTypesCount(client, [kibanaSplitIndex])).resolves.toEqual({ + basic: BASELINE_DOCUMENTS_PER_TYPE_500K, + task: BASELINE_DOCUMENTS_PER_TYPE_500K, + }); + await expectDocumentsMigratedToHighestVersion(client, [defaultKibanaIndex, kibanaSplitIndex]); + } + + async function checkFirstNodeSteps() { + const logs = await parseLogFile(getLogFile(0)); + // '.kibana_migrator_split' is a new index, all nodes' migrators must attempt to create it + expect(logs).toContainLogEntries( + [ + `[${kibanaSplitIndex}] INIT -> CREATE_REINDEX_TEMP.`, + `[${kibanaSplitIndex}] CREATE_REINDEX_TEMP -> READY_TO_REINDEX_SYNC.`, + // no docs to reindex, as source index did NOT exist + `[${kibanaSplitIndex}] READY_TO_REINDEX_SYNC -> DONE_REINDEXING_SYNC.`, + ], + { ordered: true } + ); + + // '.kibana_migrator' and '.kibana_migrator_tasks' are involved in a relocation + [defaultKibanaIndex, defaultKibanaTaskIndex].forEach((index) => { + expect(logs).toContainLogEntries( + [ + `[${index}] INIT -> WAIT_FOR_YELLOW_SOURCE.`, + `[${index}] WAIT_FOR_YELLOW_SOURCE -> CHECK_CLUSTER_ROUTING_ALLOCATION.`, + `[${index}] CHECK_CLUSTER_ROUTING_ALLOCATION -> CHECK_UNKNOWN_DOCUMENTS.`, + `[${index}] CHECK_UNKNOWN_DOCUMENTS -> SET_SOURCE_WRITE_BLOCK.`, + `[${index}] SET_SOURCE_WRITE_BLOCK -> CALCULATE_EXCLUDE_FILTERS.`, + `[${index}] CALCULATE_EXCLUDE_FILTERS -> CREATE_REINDEX_TEMP.`, + `[${index}] CREATE_REINDEX_TEMP -> READY_TO_REINDEX_SYNC.`, + `[${index}] READY_TO_REINDEX_SYNC -> REINDEX_SOURCE_TO_TEMP_OPEN_PIT.`, + `[${index}] REINDEX_SOURCE_TO_TEMP_OPEN_PIT -> REINDEX_SOURCE_TO_TEMP_READ.`, + `[${index}] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_TRANSFORM.`, + `[${index}] REINDEX_SOURCE_TO_TEMP_TRANSFORM -> REINDEX_SOURCE_TO_TEMP_INDEX_BULK.`, + `[${index}] REINDEX_SOURCE_TO_TEMP_INDEX_BULK`, + // if the index is closed by another node, we will have instead: REINDEX_SOURCE_TO_TEMP_TRANSFORM => REINDEX_SOURCE_TO_TEMP_CLOSE_PIT. + // `[${index}] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_CLOSE_PIT.`, + `[${index}] REINDEX_SOURCE_TO_TEMP_CLOSE_PIT -> DONE_REINDEXING_SYNC.`, + ], + { ordered: true } + ); + }); + + // after the relocation, all migrators share the final part of the flow + [defaultKibanaIndex, defaultKibanaTaskIndex, kibanaSplitIndex].forEach((index) => { + expect(logs).toContainLogEntries( + [ + `[${index}] DONE_REINDEXING_SYNC -> SET_TEMP_WRITE_BLOCK.`, + `[${index}] SET_TEMP_WRITE_BLOCK -> CLONE_TEMP_TO_TARGET.`, + `[${index}] CLONE_TEMP_TO_TARGET -> REFRESH_TARGET.`, + `[${index}] REFRESH_TARGET -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT.`, + `[${index}] OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ.`, + `[${index}] OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT.`, + `[${index}] OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> CHECK_TARGET_MAPPINGS.`, + `[${index}] CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES.`, + `[${index}] UPDATE_TARGET_MAPPINGS_PROPERTIES -> UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK.`, + `[${index}] UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK -> UPDATE_TARGET_MAPPINGS_META.`, + `[${index}] UPDATE_TARGET_MAPPINGS_META -> CHECK_VERSION_INDEX_READY_ACTIONS.`, + `[${index}] CHECK_VERSION_INDEX_READY_ACTIONS -> MARK_VERSION_INDEX_READY_SYNC.`, + `[${index}] MARK_VERSION_INDEX_READY_SYNC`, // all migrators try to update all aliases, all but one will have conclicts + `[${index}] Migration completed after`, + ], + { ordered: true } + ); + }); + + // should NOT retransform anything (we reindexed, thus we transformed already) + [defaultKibanaIndex, defaultKibanaTaskIndex, kibanaSplitIndex].forEach((index) => { + expect(logs).not.toContainLogEntry(`[${index}] OUTDATED_DOCUMENTS_TRANSFORM`); + expect(logs).not.toContainLogEntry( + `[${index}] Kibana is performing a compatible update and it will update the following SO types so that ES can pickup the updated mappings` + ); + }); + } + + async function checkUpToDateOnRestart() { + // run a new migrator to ensure everything is up to date + const { runMigrations } = await getRelocatingMigratorTestKit({ + logFilePath: logFileSecondRun, + // no need to filter deprecated this time, they should not be there anymore + }); + const secondRunResults = await runMigrations(); + expect( + sortBy( + secondRunResults.map((result) => omit(result, 'elapsedMs')), + 'destIndex' + ) + ).toEqual([ + { + destIndex: `.kibana_migrator_${nextMinor}_001`, + status: 'patched', + }, + { + destIndex: `.kibana_migrator_split_${nextMinor}_001`, + status: 'patched', + }, + ]); + const logs = await parseLogFile(logFileSecondRun); + expect(logs).not.toContainLogEntries(['REINDEX', 'CREATE', 'UPDATE_TARGET_MAPPINGS']); + } +}); + +async function createMigratorJobs(nodes: number): Promise>> { + const jobs: Array> = []; + + for (let i = 0; i < nodes; ++i) { + const kit = await getRelocatingMigratorTestKit({ + logFilePath: getLogFile(i), + filterDeprecated: true, + }); + jobs.push(kit.runMigrations); + } + + return jobs; +} + +async function startWithDelay(runnables: Array>, delayInSec: number) { + const promises: Array> = []; + const errors: string[] = []; + for (let i = 0; i < runnables.length; i++) { + promises.push( + runnables[i]().catch((reason) => { + errors.push(reason.message ?? reason); + return reason; + }) + ); + if (i < runnables.length - 2) { + // We wait between instances, but not after the last one + await delay(delayInSec); + } + } + const results = await Promise.all(promises); + if (errors.length) { + throw new Error(`Failed to run all parallel jobs: ${errors.join(',')}`); + } else { + return results; + } +} diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/multiple_kibana_nodes.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/multiple_kibana_nodes.test.ts deleted file mode 100644 index 9bd876166c246..0000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/multiple_kibana_nodes.test.ts +++ /dev/null @@ -1,272 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import Path from 'path'; -import del from 'del'; -import { esTestConfig, kibanaServerTestUser } from '@kbn/test'; -import { kibanaPackageJson as pkg } from '@kbn/repo-info'; -import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; -import { - createTestServers, - createRoot as createkbnTestServerRoot, - type TestElasticsearchUtils, -} from '@kbn/core-test-helpers-kbn-server'; -import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import { Root } from '@kbn/core-root-server-internal'; - -const LOG_FILE_PREFIX = 'migration_test_multiple_kibana_nodes'; - -async function removeLogFiles() { - await del([Path.join(__dirname, `${LOG_FILE_PREFIX}_*.log`)], { force: true }); -} - -function extractSortNumberFromId(id: string): number { - const parsedId = parseInt(id.split(':')[1], 10); // "foo:123" -> 123 - if (isNaN(parsedId)) { - throw new Error(`Failed to parse Saved Object ID [${id}]. Result is NaN`); - } - return parsedId; -} - -async function fetchDocs(esClient: ElasticsearchClient, index: string) { - const body = await esClient.search({ - index, - size: 10000, - body: { - query: { - bool: { - should: [ - { - term: { type: 'foo' }, - }, - ], - }, - }, - }, - }); - - return body.hits.hits - .map((h) => ({ - ...h._source, - id: h._id, - })) - .sort((a, b) => extractSortNumberFromId(a.id) - extractSortNumberFromId(b.id)); -} - -interface CreateRootConfig { - logFileName: string; -} - -async function createRoot({ logFileName }: CreateRootConfig) { - const root = createkbnTestServerRoot({ - elasticsearch: { - hosts: [esTestConfig.getUrl()], - username: kibanaServerTestUser.username, - password: kibanaServerTestUser.password, - }, - migrations: { - skip: false, - batchSize: 100, // fixture contains 5000 docs - }, - logging: { - appenders: { - file: { - type: 'file', - fileName: logFileName, - layout: { - type: 'pattern', - }, - }, - }, - loggers: [ - { - name: 'root', - appenders: ['file'], - level: 'info', - }, - { - name: 'savedobjects-service', - appenders: ['file'], - level: 'debug', - }, - ], - }, - }); - - await root.preboot(); - - return root; -} - -// suite is very long, the 10mins default can cause timeouts -jest.setTimeout(15 * 60 * 1000); - -describe('migration v2', () => { - let esServer: TestElasticsearchUtils; - let rootA: Root; - let rootB: Root; - let rootC: Root; - - const migratedIndexAlias = `.kibana_${pkg.version}`; - const fooType: SavedObjectsType = { - name: 'foo', - hidden: false, - mappings: { properties: { status: { type: 'text' } } }, - namespaceType: 'agnostic', - migrations: { - '7.14.0': (doc) => { - if (doc.attributes?.status) { - doc.attributes.status = doc.attributes.status.replace('unmigrated', 'migrated'); - } - return doc; - }, - }, - }; - - const delay = (timeInMs: number) => new Promise((resolve) => setTimeout(resolve, timeInMs)); - - beforeEach(async () => { - await removeLogFiles(); - - rootA = await createRoot({ - logFileName: Path.join(__dirname, `${LOG_FILE_PREFIX}_A.log`), - }); - rootB = await createRoot({ - logFileName: Path.join(__dirname, `${LOG_FILE_PREFIX}_B.log`), - }); - rootC = await createRoot({ - logFileName: Path.join(__dirname, `${LOG_FILE_PREFIX}_C.log`), - }); - - const { startES } = createTestServers({ - adjustTimeout: (t: number) => jest.setTimeout(t), - settings: { - es: { - license: 'basic', - // original SOs: 5k of `foo` docs with this structure: - // [ - // { id: 'foo:1', type: 'foo', foo: { status: 'unmigrated' }, migrationVersion: { foo: '7.13.0' } }, - // { id: 'foo:2', type: 'foo', foo: { status: 'unmigrated' }, migrationVersion: { foo: '7.13.0' } }, - // { id: 'foo:3', type: 'foo', foo: { status: 'unmigrated' }, migrationVersion: { foo: '7.13.0' } }, - // ]; - dataArchive: Path.join(__dirname, '..', 'archives', '7.13.0_concurrent_5k_foo.zip'), - }, - }, - }); - esServer = await startES(); - }); - - afterEach(async () => { - try { - await Promise.all([rootA.shutdown(), rootB.shutdown(), rootC.shutdown()]); - } catch (e) { - /* trap */ - } - - if (esServer) { - await esServer.stop(); - } - }); - - const startWithDelay = async (instances: Root[], delayInSec: number) => { - const promises: Array> = []; - const errors: string[] = []; - for (let i = 0; i < instances.length; i++) { - promises.push( - instances[i].start().catch((err) => { - errors.push(err.message); - }) - ); - if (i < instances.length - 2) { - // We wait between instances, but not after the last one - await delay(delayInSec * 1000); - } - } - await Promise.all(promises); - if (errors.length) { - throw new Error(`Failed to start all instances: ${errors.join(',')}`); - } - }; - - it('migrates saved objects normally when multiple Kibana instances are started at the same time', async () => { - const setupContracts = await Promise.all([rootA.setup(), rootB.setup(), rootC.setup()]); - - setupContracts.forEach((setup) => setup.savedObjects.registerType(fooType)); - - await startWithDelay([rootA, rootB, rootC], 0); - - const esClient = esServer.es.getClient(); - const migratedDocs = await fetchDocs(esClient, migratedIndexAlias); - - expect(migratedDocs.length).toBe(5000); - - migratedDocs.forEach((doc, i) => { - expect(doc.id).toBe(`foo:${i}`); - expect(doc.foo.status).toBe(`migrated`); - expect(doc.typeMigrationVersion).toBe('7.14.0'); - }); - }); - - it('migrates saved objects normally when multiple Kibana instances are started with a small interval', async () => { - const setupContracts = await Promise.all([rootA.setup(), rootB.setup(), rootC.setup()]); - - setupContracts.forEach((setup) => setup.savedObjects.registerType(fooType)); - - await startWithDelay([rootA, rootB, rootC], 1); - - const esClient = esServer.es.getClient(); - const migratedDocs = await fetchDocs(esClient, migratedIndexAlias); - - expect(migratedDocs.length).toBe(5000); - - migratedDocs.forEach((doc, i) => { - expect(doc.id).toBe(`foo:${i}`); - expect(doc.foo.status).toBe(`migrated`); - expect(doc.typeMigrationVersion).toBe('7.14.0'); - }); - }); - - it('migrates saved objects normally when multiple Kibana instances are started with an average interval', async () => { - const setupContracts = await Promise.all([rootA.setup(), rootB.setup(), rootC.setup()]); - - setupContracts.forEach((setup) => setup.savedObjects.registerType(fooType)); - - await startWithDelay([rootA, rootB, rootC], 5); - - const esClient = esServer.es.getClient(); - const migratedDocs = await fetchDocs(esClient, migratedIndexAlias); - - expect(migratedDocs.length).toBe(5000); - - migratedDocs.forEach((doc, i) => { - expect(doc.id).toBe(`foo:${i}`); - expect(doc.foo.status).toBe(`migrated`); - expect(doc.typeMigrationVersion).toBe('7.14.0'); - }); - }); - - it('migrates saved objects normally when multiple Kibana instances are started with a bigger interval', async () => { - const setupContracts = await Promise.all([rootA.setup(), rootB.setup(), rootC.setup()]); - - setupContracts.forEach((setup) => setup.savedObjects.registerType(fooType)); - - await startWithDelay([rootA, rootB, rootC], 20); - - const esClient = esServer.es.getClient(); - const migratedDocs = await fetchDocs(esClient, migratedIndexAlias); - - expect(migratedDocs.length).toBe(5000); - - migratedDocs.forEach((doc, i) => { - expect(doc.id).toBe(`foo:${i}`); - expect(doc.foo.status).toBe(`migrated`); - expect(doc.typeMigrationVersion).toBe('7.14.0'); - }); - }); -}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/actions/actions.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/actions/actions.test.ts index 0559d059a6cb1..2ee7ea6701e84 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/actions/actions.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/actions/actions.test.ts @@ -7,7 +7,6 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import Path from 'path'; import * as Either from 'fp-ts/lib/Either'; import * as Option from 'fp-ts/lib/Option'; import { errors } from '@elastic/elasticsearch'; @@ -53,15 +52,13 @@ const { startES } = createTestServers({ settings: { es: { license: 'basic', - dataArchive: Path.resolve(__dirname, '../../archives/7.7.2_xpack_100k_obj.zip'), esArgs: ['http.max_content_length=10Kb'], }, }, }); let esServer: TestElasticsearchUtils; -// Failing 9.0 version update: https://github.com/elastic/kibana/issues/192624 -describe.skip('migration actions', () => { +describe('migration actions', () => { let client: ElasticsearchClient; let esCapabilities: ReturnType; diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/read_batch_size.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/read_batch_size.test.ts index e6c22469e1fbe..df809d8c4c173 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/read_batch_size.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/read_batch_size.test.ts @@ -7,33 +7,33 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import Path from 'path'; -import fs from 'fs/promises'; -import { Root } from '@kbn/core-root-server-internal'; +import { join } from 'path'; +import type { Root } from '@kbn/core-root-server-internal'; import { createRootWithCorePlugins, type TestElasticsearchUtils, } from '@kbn/core-test-helpers-kbn-server'; -import { startElasticsearch } from '../kibana_migrator_test_kit'; +import { clearLog, readLog, startElasticsearch } from '../kibana_migrator_test_kit'; +import { delay } from '../test_utils'; -const logFilePath = Path.join(__dirname, 'read_batch_size.log'); +const logFilePath = join(__dirname, 'read_batch_size.log'); -// Failing 9.0 version update: https://github.com/elastic/kibana/issues/192624 -describe.skip('migration v2 - read batch size', () => { +describe('migration v2 - read batch size', () => { let esServer: TestElasticsearchUtils; let root: Root; let logs: string; beforeEach(async () => { esServer = await startElasticsearch({ - dataArchive: Path.join(__dirname, '..', 'archives', '8.4.0_with_sample_data_logs.zip'), + dataArchive: join(__dirname, '..', 'archives', '8.4.0_with_sample_data_logs.zip'), }); - await fs.unlink(logFilePath).catch(() => {}); + await clearLog(logFilePath); }); afterEach(async () => { await root?.shutdown(); await esServer?.stop(); + await delay(5); // give it a few seconds... cause we always do ¯\_(ツ)_/¯ }); it('reduces the read batchSize in half if a batch exceeds maxReadBatchSizeBytes', async () => { @@ -43,7 +43,7 @@ describe.skip('migration v2 - read batch size', () => { await root.start(); // Check for migration steps present in the logs - logs = await fs.readFile(logFilePath, 'utf-8'); + logs = await readLog(logFilePath); expect(logs).toMatch( /Read a batch with a response content length of \d+ bytes which exceeds migrations\.maxReadBatchSizeBytes, retrying by reducing the batch size in half to 15/ @@ -58,7 +58,7 @@ describe.skip('migration v2 - read batch size', () => { await root.start(); // Check for migration steps present in the logs - logs = await fs.readFile(logFilePath, 'utf-8'); + logs = await readLog(logFilePath); expect(logs).not.toMatch('retrying by reducing the batch size in half to'); expect(logs).toMatch('[.kibana] Migration completed'); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/split_failed_to_clone.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/split_failed_to_clone.test.ts index c37640cfe9569..efc6e7bd62cb4 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/split_failed_to_clone.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/split_failed_to_clone.test.ts @@ -7,25 +7,23 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import Path from 'path'; +import { join } from 'path'; import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; -import { - type ISavedObjectTypeRegistry, - type SavedObjectsType, - MAIN_SAVED_OBJECT_INDEX, -} from '@kbn/core-saved-objects-server'; -import { DEFAULT_INDEX_TYPES_MAP } from '@kbn/core-saved-objects-base-server-internal'; import type { CloneIndexParams } from '@kbn/core-saved-objects-migration-server-internal/src/actions'; import { clearLog, startElasticsearch, - getKibanaMigratorTestKit, - getCurrentVersionTypeRegistry, - overrideTypeRegistry, getAggregatedTypesCount, type KibanaMigratorTestKit, + defaultKibanaTaskIndex, + defaultKibanaIndex, } from '../kibana_migrator_test_kit'; +import { BASELINE_TEST_ARCHIVE_1K } from '../kibana_migrator_archive_utils'; +import { + getRelocatingMigratorTestKit, + kibanaSplitIndex, +} from '../kibana_migrator_test_kit.fixtures'; import { delay } from '../test_utils'; import '../jest_matchers'; @@ -38,42 +36,24 @@ jest.mock('@kbn/core-saved-objects-migration-server-internal/src/actions/clone_i ...realModule, cloneIndex: (params: CloneIndexParams) => async () => { // we need to slow down the clone operation for indices other than - // .kibana so that .kibana can completely finish the migration before we + // .kibana_migrator so that .kibana_migrator can completely finish the migration before we // fail - if (params.target.includes('slow_clone')) - await new Promise((resolve) => setTimeout(resolve, 1000)); + if (!params.target.includes('tasks') && !params.target.includes('new')) + await new Promise((resolve) => setTimeout(resolve, 2000)); return realModule.cloneIndex(params)(); }, }; }); -// define a type => index distribution -const RELOCATE_TYPES: Record = { - dashboard: '.kibana_slow_clone_1', - visualization: '.kibana_slow_clone_1', - 'canvas-workpad': '.kibana_slow_clone_1', - search: '.kibana_slow_clone_2', - task: '.kibana_task_manager_new', // force reindex - 'epm-packages-assets': '.kibana_slow_clone_1', - // the remaining types will be forced to go to '.kibana', - // overriding `indexPattern: foo` defined in the registry -}; - -export const logFilePath = Path.join(__dirname, 'split_failed_to_clone.test.log'); - -// Failing 9.0 version update: https://github.com/elastic/kibana/issues/192624 -describe.skip('when splitting .kibana into multiple indices and one clone fails', () => { +export const logFilePath = join(__dirname, 'split_failed_to_clone.test.log'); + +describe('when splitting .kibana into multiple indices and one clone fails', () => { let esServer: TestElasticsearchUtils['es']; - let typeRegistry: ISavedObjectTypeRegistry; let migratorTestKitFactory: () => Promise; beforeAll(async () => { - typeRegistry = await getCurrentVersionTypeRegistry({ oss: false }); await clearLog(logFilePath); - esServer = await startElasticsearch({ - dataArchive: Path.join(__dirname, '..', 'archives', '7.14.0_xpack_sample_saved_objects.zip'), - timeout: 60000, - }); + esServer = await startElasticsearch({ dataArchive: BASELINE_TEST_ARCHIVE_1K }); }); afterAll(async () => { @@ -82,59 +62,33 @@ describe.skip('when splitting .kibana into multiple indices and one clone fails' }); it('after resolving the problem and retrying the migration completes successfully', async () => { - const updatedTypeRegistry = overrideTypeRegistry( - typeRegistry, - (type: SavedObjectsType) => { - return { - ...type, - indexPattern: RELOCATE_TYPES[type.name] ?? MAIN_SAVED_OBJECT_INDEX, - }; - } - ); - migratorTestKitFactory = () => - getKibanaMigratorTestKit({ - types: updatedTypeRegistry.getAllTypes(), - kibanaIndex: '.kibana', + getRelocatingMigratorTestKit({ logFilePath, - defaultIndexTypesMap: DEFAULT_INDEX_TYPES_MAP, + filterDeprecated: true, + relocateTypes: { + // move 'basic' to a new index + basic: kibanaSplitIndex, + }, }); const { runMigrations: runMigrationsWhichFailsWhenCloning, client } = await migratorTestKitFactory(); - // count of types in the legacy index - expect(await getAggregatedTypesCount(client, '.kibana')).toEqual({ - 'apm-telemetry': 1, - application_usage_daily: 4, - 'canvas-workpad': 3, - 'canvas-workpad-template': 5, - config: 1, - 'core-usage-stats': 1, - dashboard: 19, - 'epm-packages': 3, - 'epm-packages-assets': 293, - event_loop_delays_daily: 1, - 'graph-workspace': 3, - 'index-pattern': 5, - 'ingest-agent-policies': 2, - 'ingest-outputs': 1, - 'ingest-package-policies': 2, - ingest_manager_settings: 1, - map: 3, - 'osquery-usage-metric': 1, - 'sample-data-telemetry': 3, - search: 14, - space: 1, - 'spaces-usage-stats': 1, - telemetry: 1, - 'ui-metric': 5, - 'usage-counters': 4, - visualization: 173, + // ensure we have a valid 'before' state + expect(await getAggregatedTypesCount(client, defaultKibanaIndex)).toEqual({ + basic: 200, + complex: 200, + deprecated: 200, + server: 200, + }); + expect(await getAggregatedTypesCount(client, defaultKibanaTaskIndex)).toEqual({ + task: 200, }); + expect(await getAggregatedTypesCount(client, kibanaSplitIndex)).toEqual({}); // cause a failure when cloning .kibana_slow_clone_* indices - await client.cluster.putSettings({ persistent: { 'cluster.max_shards_per_node': 15 } }); + await client.cluster.putSettings({ persistent: { 'cluster.max_shards_per_node': 6 } }); await expect(runMigrationsWhichFailsWhenCloning()).rejects.toThrowError( /cluster_shard_limit_exceeded/ @@ -146,43 +100,16 @@ describe.skip('when splitting .kibana into multiple indices and one clone fails' const { runMigrations: runMigrations2ndTime } = await migratorTestKitFactory(); await runMigrations2ndTime(); - expect(await getAggregatedTypesCount(client, '.kibana')).toMatchInlineSnapshot(` - Object { - "apm-telemetry": 1, - "application_usage_daily": 4, - "canvas-workpad-template": 5, - "config": 1, - "core-usage-stats": 1, - "epm-packages": 3, - "event_loop_delays_daily": 1, - "graph-workspace": 3, - "index-pattern": 5, - "ingest-agent-policies": 2, - "ingest-outputs": 1, - "ingest-package-policies": 2, - "ingest_manager_settings": 1, - "map": 3, - "sample-data-telemetry": 3, - "space": 1, - "spaces-usage-stats": 1, - "telemetry": 1, - "ui-metric": 5, - "usage-counters": 4, - } - `); - expect(await getAggregatedTypesCount(client, '.kibana_slow_clone_1')).toMatchInlineSnapshot(` - Object { - "canvas-workpad": 3, - "dashboard": 19, - "epm-packages-assets": 293, - "visualization": 173, - } - `); - expect(await getAggregatedTypesCount(client, '.kibana_slow_clone_2')).toMatchInlineSnapshot(` - Object { - "search": 14, - } - `); + // ensure we have a valid 'after' state + expect(await getAggregatedTypesCount(client, defaultKibanaIndex)).toEqual({ + complex: 99, + }); + expect(await getAggregatedTypesCount(client, defaultKibanaTaskIndex)).toEqual({ + task: 200, + }); + expect(await getAggregatedTypesCount(client, kibanaSplitIndex)).toEqual({ + basic: 200, + }); // If we run a third time, we should not get any errors const { runMigrations: runMigrations3rdTime } = await migratorTestKitFactory(); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts deleted file mode 100644 index 58dfb5d4e433e..0000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts +++ /dev/null @@ -1,406 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import Path from 'path'; -import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; -import { - type ISavedObjectTypeRegistry, - type SavedObjectsType, - MAIN_SAVED_OBJECT_INDEX, - ALL_SAVED_OBJECT_INDICES, - TASK_MANAGER_SAVED_OBJECT_INDEX, -} from '@kbn/core-saved-objects-server'; -import { DEFAULT_INDEX_TYPES_MAP } from '@kbn/core-saved-objects-base-server-internal'; -import { - clearLog, - startElasticsearch, - getKibanaMigratorTestKit, - getCurrentVersionTypeRegistry, - overrideTypeRegistry, - getAggregatedTypesCount, - currentVersion, - type KibanaMigratorTestKit, - getEsClient, -} from '../kibana_migrator_test_kit'; -import { delay, parseLogFile } from '../test_utils'; -import '../jest_matchers'; - -// define a type => index distribution -const RELOCATE_TYPES: Record = { - dashboard: '.kibana_so_ui', - visualization: '.kibana_so_ui', - 'canvas-workpad': '.kibana_so_ui', - search: '.kibana_so_search', - task: '.kibana_task_manager', - // the remaining types will be forced to go to '.kibana', - // overriding `indexPattern: foo` defined in the registry -}; - -const PARALLEL_MIGRATORS = 6; -export const logFilePathFirstRun = Path.join(__dirname, 'dot_kibana_split_1st_run.test.log'); -export const logFilePathSecondRun = Path.join(__dirname, 'dot_kibana_split_2nd_run.test.log'); - -// Failing 9.0 version update: https://github.com/elastic/kibana/issues/192624 -describe.skip('split .kibana index into multiple system indices', () => { - let esServer: TestElasticsearchUtils['es']; - let typeRegistry: ISavedObjectTypeRegistry; - - beforeAll(async () => { - typeRegistry = await getCurrentVersionTypeRegistry({ oss: false }); - }); - - beforeEach(async () => { - await clearLog(logFilePathFirstRun); - await clearLog(logFilePathSecondRun); - }); - - describe('when migrating from a legacy version', () => { - let migratorTestKitFactory: (logFilePath: string) => Promise; - - beforeAll(async () => { - esServer = await startElasticsearch({ - dataArchive: Path.join(__dirname, '..', 'archives', '7.3.0_xpack_sample_saved_objects.zip'), - timeout: 60000, - }); - }); - - it('performs v1 migration and then relocates saved objects into different indices, depending on their types', async () => { - const updatedTypeRegistry = overrideTypeRegistry( - typeRegistry, - (type: SavedObjectsType) => { - return { - ...type, - indexPattern: RELOCATE_TYPES[type.name] ?? MAIN_SAVED_OBJECT_INDEX, - }; - } - ); - - migratorTestKitFactory = (logFilePath: string) => - getKibanaMigratorTestKit({ - types: updatedTypeRegistry.getAllTypes(), - kibanaIndex: '.kibana', - logFilePath, - defaultIndexTypesMap: DEFAULT_INDEX_TYPES_MAP, - }); - - const { runMigrations, client } = await migratorTestKitFactory(logFilePathFirstRun); - - // count of types in the legacy index - expect(await getAggregatedTypesCount(client, '.kibana_1')).toEqual({ - 'canvas-workpad': 3, - config: 1, - dashboard: 3, - 'index-pattern': 3, - map: 3, - 'maps-telemetry': 1, - 'sample-data-telemetry': 3, - search: 2, - telemetry: 1, - space: 1, - visualization: 39, - }); - - await runMigrations(); - - await client.indices.refresh({ - index: ['.kibana', '.kibana_so_search', '.kibana_so_ui'], - }); - - expect(await getAggregatedTypesCount(client, '.kibana')).toEqual({ - 'index-pattern': 3, - map: 3, - 'sample-data-telemetry': 3, - config: 1, - telemetry: 1, - space: 1, - }); - expect(await getAggregatedTypesCount(client, '.kibana_so_search')).toEqual({ - search: 2, - }); - expect(await getAggregatedTypesCount(client, '.kibana_so_ui')).toEqual({ - visualization: 39, - 'canvas-workpad': 3, - dashboard: 3, - }); - - const indicesInfo = await client.indices.get({ index: '.kibana*' }); - expect(indicesInfo[`.kibana_${currentVersion}_001`]).toEqual( - expect.objectContaining({ - aliases: expect.objectContaining({ '.kibana': expect.any(Object) }), - mappings: { - dynamic: 'strict', - _meta: { - mappingVersions: expect.any(Object), - indexTypesMap: expect.any(Object), - }, - properties: expect.any(Object), - }, - settings: { index: expect.any(Object) }, - }) - ); - - expect(indicesInfo[`.kibana_so_search_${currentVersion}_001`]).toEqual( - expect.objectContaining({ - aliases: expect.objectContaining({ '.kibana_so_search': expect.any(Object) }), - mappings: { - dynamic: 'strict', - _meta: { - mappingVersions: expect.any(Object), - indexTypesMap: expect.any(Object), - }, - properties: expect.any(Object), - }, - settings: { index: expect.any(Object) }, - }) - ); - - expect(indicesInfo[`.kibana_so_ui_${currentVersion}_001`]).toEqual( - expect.objectContaining({ - aliases: expect.objectContaining({ '.kibana_so_ui': expect.any(Object) }), - mappings: { - dynamic: 'strict', - _meta: { - mappingVersions: expect.any(Object), - indexTypesMap: expect.any(Object), - }, - properties: expect.any(Object), - }, - settings: { index: expect.any(Object) }, - }) - ); - - const typesMap = indicesInfo[`.kibana_${currentVersion}_001`].mappings?._meta?.indexTypesMap; - - expect(Array.isArray(typesMap['.kibana'])).toEqual(true); - expect(typesMap['.kibana'].length > 50).toEqual(true); - expect(typesMap['.kibana'].includes('action')).toEqual(true); - expect(typesMap['.kibana'].includes('cases')).toEqual(true); - expect(typesMap['.kibana_so_search']).toEqual(['search']); - expect(typesMap['.kibana_so_ui']).toEqual(['canvas-workpad', 'dashboard', 'visualization']); - expect(typesMap['.kibana_task_manager']).toEqual(['task']); - - const logs = await parseLogFile(logFilePathFirstRun); - - expect(logs).toContainLogEntries( - [ - // .kibana_task_manager index exists and has no aliases => LEGACY_* migration path - '[.kibana_task_manager] INIT -> LEGACY_CHECK_CLUSTER_ROUTING_ALLOCATION.', - '[.kibana_task_manager] LEGACY_CHECK_CLUSTER_ROUTING_ALLOCATION -> LEGACY_SET_WRITE_BLOCK.', - '[.kibana_task_manager] LEGACY_REINDEX_WAIT_FOR_TASK -> LEGACY_DELETE.', - '[.kibana_task_manager] LEGACY_DELETE -> SET_SOURCE_WRITE_BLOCK.', - '[.kibana_task_manager] SET_SOURCE_WRITE_BLOCK -> CALCULATE_EXCLUDE_FILTERS.', - '[.kibana_task_manager] CALCULATE_EXCLUDE_FILTERS -> CREATE_REINDEX_TEMP.', - '[.kibana_task_manager] CREATE_REINDEX_TEMP -> REINDEX_SOURCE_TO_TEMP_OPEN_PIT.', - '[.kibana_task_manager] REINDEX_SOURCE_TO_TEMP_OPEN_PIT -> REINDEX_SOURCE_TO_TEMP_READ.', - '[.kibana_task_manager] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_TRANSFORM.', - '[.kibana_task_manager] REINDEX_SOURCE_TO_TEMP_TRANSFORM -> REINDEX_SOURCE_TO_TEMP_INDEX_BULK.', - '[.kibana_task_manager] REINDEX_SOURCE_TO_TEMP_INDEX_BULK -> REINDEX_SOURCE_TO_TEMP_READ.', - '[.kibana_task_manager] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_CLOSE_PIT.', - '[.kibana_task_manager] REINDEX_SOURCE_TO_TEMP_CLOSE_PIT -> SET_TEMP_WRITE_BLOCK.', - '[.kibana_task_manager] SET_TEMP_WRITE_BLOCK -> CLONE_TEMP_TO_TARGET.', - '[.kibana_task_manager] CLONE_TEMP_TO_TARGET -> REFRESH_TARGET.', - '[.kibana_task_manager] REFRESH_TARGET -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT.', - '[.kibana_task_manager] OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ.', - '[.kibana_task_manager] OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT.', - '[.kibana_task_manager] OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> CHECK_TARGET_MAPPINGS.', - '[.kibana_task_manager] CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES.', - '[.kibana_task_manager] UPDATE_TARGET_MAPPINGS_PROPERTIES -> UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK.', - '[.kibana_task_manager] UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK -> UPDATE_TARGET_MAPPINGS_META.', - '[.kibana_task_manager] UPDATE_TARGET_MAPPINGS_META -> CHECK_VERSION_INDEX_READY_ACTIONS.', - '[.kibana_task_manager] CHECK_VERSION_INDEX_READY_ACTIONS -> MARK_VERSION_INDEX_READY.', - '[.kibana_task_manager] MARK_VERSION_INDEX_READY -> DONE.', - '[.kibana_task_manager] Migration completed after', - ], - { ordered: true } - ); - - expect(logs).not.toContainLogEntries([ - // .kibana_task_manager migrator is NOT involved in relocation, must not sync with other migrators - '[.kibana_task_manager] READY_TO_REINDEX_SYNC', - '[.kibana_task_manager] DONE_REINDEXING_SYNC', - // .kibana_task_manager migrator performed a REINDEX migration, it must update ALL types - '[.kibana_task_manager] Kibana is performing a compatible update and it will update the following SO types so that ES can pickup the updated mappings', - ]); - - // new indices migrators did not exist, so they all have to reindex (create temp index + sync) - ['.kibana_so_ui', '.kibana_so_search'].forEach((newIndex) => { - expect(logs).toContainLogEntries( - [ - `[${newIndex}] INIT -> CREATE_REINDEX_TEMP.`, - `[${newIndex}] CREATE_REINDEX_TEMP -> READY_TO_REINDEX_SYNC.`, - // no docs to reindex, as source index did NOT exist - `[${newIndex}] READY_TO_REINDEX_SYNC -> DONE_REINDEXING_SYNC.`, - ], - { ordered: true } - ); - }); - - // the .kibana migrator is involved in a relocation, it must also reindex - expect(logs).toContainLogEntries( - [ - '[.kibana] INIT -> WAIT_FOR_YELLOW_SOURCE.', - '[.kibana] WAIT_FOR_YELLOW_SOURCE -> CHECK_CLUSTER_ROUTING_ALLOCATION.', - '[.kibana] CHECK_CLUSTER_ROUTING_ALLOCATION -> CHECK_UNKNOWN_DOCUMENTS.', - '[.kibana] CHECK_UNKNOWN_DOCUMENTS -> SET_SOURCE_WRITE_BLOCK.', - '[.kibana] SET_SOURCE_WRITE_BLOCK -> CALCULATE_EXCLUDE_FILTERS.', - '[.kibana] CALCULATE_EXCLUDE_FILTERS -> CREATE_REINDEX_TEMP.', - '[.kibana] CREATE_REINDEX_TEMP -> READY_TO_REINDEX_SYNC.', - '[.kibana] READY_TO_REINDEX_SYNC -> REINDEX_SOURCE_TO_TEMP_OPEN_PIT.', - '[.kibana] REINDEX_SOURCE_TO_TEMP_OPEN_PIT -> REINDEX_SOURCE_TO_TEMP_READ.', - '[.kibana] Starting to process 59 documents.', - '[.kibana] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_TRANSFORM.', - '[.kibana] REINDEX_SOURCE_TO_TEMP_TRANSFORM -> REINDEX_SOURCE_TO_TEMP_INDEX_BULK.', - '[.kibana] REINDEX_SOURCE_TO_TEMP_INDEX_BULK -> REINDEX_SOURCE_TO_TEMP_READ.', - '[.kibana] Processed 59 documents out of 59.', - '[.kibana] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_CLOSE_PIT.', - '[.kibana] REINDEX_SOURCE_TO_TEMP_CLOSE_PIT -> DONE_REINDEXING_SYNC.', - ], - { ordered: true } - ); - - // after .kibana migrator is done relocating documents - // the 3 migrators share the final part of the flow - ['.kibana', '.kibana_so_ui', '.kibana_so_search'].forEach((index) => { - expect(logs).toContainLogEntries( - [ - `[${index}] DONE_REINDEXING_SYNC -> SET_TEMP_WRITE_BLOCK.`, - `[${index}] SET_TEMP_WRITE_BLOCK -> CLONE_TEMP_TO_TARGET.`, - `[${index}] CLONE_TEMP_TO_TARGET -> REFRESH_TARGET.`, - `[${index}] REFRESH_TARGET -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT.`, - `[${index}] OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ.`, - `[${index}] OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT.`, - `[${index}] OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> CHECK_TARGET_MAPPINGS.`, - `[${index}] CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES.`, - `[${index}] UPDATE_TARGET_MAPPINGS_PROPERTIES -> UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK.`, - `[${index}] UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK -> UPDATE_TARGET_MAPPINGS_META.`, - `[${index}] UPDATE_TARGET_MAPPINGS_META -> CHECK_VERSION_INDEX_READY_ACTIONS.`, - `[${index}] CHECK_VERSION_INDEX_READY_ACTIONS -> MARK_VERSION_INDEX_READY_SYNC.`, - `[${index}] MARK_VERSION_INDEX_READY_SYNC`, // all migrators try to update all aliases, all but one will have conclicts - `[${index}] Migration completed after`, - ], - { ordered: true } - ); - }); - - // should NOT retransform anything (we reindexed, thus we transformed already) - ['.kibana', '.kibana_task_manager', '.kibana_so_ui', '.kibana_so_search'].forEach((index) => { - expect(logs).not.toContainLogEntry(`[${index}] OUTDATED_DOCUMENTS_TRANSFORM`); - expect(logs).not.toContainLogEntry( - `[${index}] Kibana is performing a compatible update and it will update the following SO types so that ES can pickup the updated mappings` - ); - }); - }); - - afterEach(async () => { - // we run the migrator again to ensure that the next time state is loaded everything still works as expected - const { runMigrations } = await migratorTestKitFactory(logFilePathSecondRun); - await runMigrations(); - const logs = await parseLogFile(logFilePathSecondRun); - expect(logs).not.toContainLogEntries(['REINDEX', 'CREATE', 'UPDATE_TARGET_MAPPINGS']); - }); - - afterAll(async () => { - await esServer?.stop(); - await delay(2); - }); - }); - - describe('when multiple Kibana migrators run in parallel', () => { - jest.setTimeout(1200000); - it('correctly migrates 7.7.2_xpack_100k_obj.zip archive', async () => { - esServer = await startElasticsearch({ - dataArchive: Path.join(__dirname, '..', 'archives', '7.7.2_xpack_100k_obj.zip'), - }); - const esClient = await getEsClient(); - - const breakdownBefore = await getAggregatedTypesCount(esClient, [ - MAIN_SAVED_OBJECT_INDEX, - TASK_MANAGER_SAVED_OBJECT_INDEX, - ]); - expect(breakdownBefore).toEqual({ - '.kibana': { - 'apm-telemetry': 1, - application_usage_transactional: 4, - config: 1, - dashboard: 52994, - 'index-pattern': 1, - 'maps-telemetry': 1, - search: 1, - space: 1, - 'ui-metric': 5, - visualization: 53004, - }, - '.kibana_task_manager': { - task: 5, - }, - }); - - for (let i = 0; i < PARALLEL_MIGRATORS; ++i) { - await clearLog(Path.join(__dirname, `dot_kibana_split_instance_${i}.log`)); - } - - const testKits = await Promise.all( - new Array(PARALLEL_MIGRATORS).fill(true).map((_, index) => - getKibanaMigratorTestKit({ - settings: { - migrations: { - discardUnknownObjects: currentVersion, - discardCorruptObjects: currentVersion, - }, - }, - kibanaIndex: MAIN_SAVED_OBJECT_INDEX, - types: typeRegistry.getAllTypes(), - defaultIndexTypesMap: DEFAULT_INDEX_TYPES_MAP, - logFilePath: Path.join(__dirname, `dot_kibana_split_instance_${index}.log`), - }) - ) - ); - - const results = await Promise.all(testKits.map((testKit) => testKit.runMigrations())); - expect( - results - .flat() - .every((result) => result.status === 'migrated' || result.status === 'patched') - ).toEqual(true); - - await esClient.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); - - const breakdownAfter = await getAggregatedTypesCount(esClient, [ - MAIN_SAVED_OBJECT_INDEX, - TASK_MANAGER_SAVED_OBJECT_INDEX, - ]); - expect(breakdownAfter).toEqual({ - '.kibana': { - 'apm-telemetry': 1, - config: 1, - space: 1, - 'ui-metric': 5, - }, - '.kibana_alerting_cases': {}, - '.kibana_analytics': { - dashboard: 52994, - 'index-pattern': 1, - search: 1, - visualization: 53004, - }, - '.kibana_ingest': {}, - '.kibana_security_solution': {}, - '.kibana_task_manager': { - task: 5, - }, - '.kibana_usage_counters': {}, - }); - }); - - afterEach(async () => { - await esServer?.stop(); - await delay(2); - }); - }); -}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group6/single_migrator_failures.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group6/single_migrator_failures.test.ts index 358ceea5c006a..81ef3bd237d6d 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group6/single_migrator_failures.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group6/single_migrator_failures.test.ts @@ -15,33 +15,19 @@ import { startElasticsearch, defaultKibanaIndex, defaultKibanaTaskIndex, - getKibanaMigratorTestKit, } from '../kibana_migrator_test_kit'; import { delay } from '../test_utils'; import '../jest_matchers'; import { getElasticsearchClientWrapperFactory } from '../elasticsearch_client_wrapper'; import { BASELINE_TEST_ARCHIVE_1K } from '../kibana_migrator_archive_utils'; import { - baselineIndexTypesMap, - getReindexingBaselineTypes, + getRelocatingMigratorTestKit, + kibanaSplitIndex, } from '../kibana_migrator_test_kit.fixtures'; export const logFilePathFirstRun = join(__dirname, 'single_migrator_failures_1st_run.test.log'); export const logFilePathSecondRun = join(__dirname, 'single_migrator_failures_2nd_run.test.log'); -const kibanaSplitIndex = `${defaultKibanaIndex}_split`; -const tasksToNewIndex = getReindexingBaselineTypes(true).map((type) => { - if (type.name !== 'task' && type.name !== 'simple') { - return type; - } - - // relocate 'simple' and 'task' objects to a new index (forces reindex) - return { - ...type, - indexPattern: kibanaSplitIndex, - }; -}); - describe('split .kibana index into multiple system indices', () => { let esServer: TestElasticsearchUtils['es']; @@ -65,19 +51,7 @@ describe('split .kibana index into multiple system indices', () => { errorDelaySeconds: delaySeconds, }); - return await getKibanaMigratorTestKit({ - types: tasksToNewIndex, - logFilePath, - kibanaVersion: nextMinor, - defaultIndexTypesMap: baselineIndexTypesMap, - clientWrapperFactory, - settings: { - migrations: { - discardUnknownObjects: nextMinor, - discardCorruptObjects: nextMinor, - }, - }, - }); + return await getRelocatingMigratorTestKit({ logFilePath, clientWrapperFactory }); }; beforeEach(async () => { @@ -242,17 +216,8 @@ describe('split .kibana index into multiple system indices', () => { }); afterEach(async () => { - const { runMigrations: secondRun } = await getKibanaMigratorTestKit({ + const { runMigrations: secondRun } = await getRelocatingMigratorTestKit({ logFilePath: logFilePathSecondRun, - types: tasksToNewIndex, - kibanaVersion: nextMinor, - defaultIndexTypesMap: baselineIndexTypesMap, - settings: { - migrations: { - discardUnknownObjects: nextMinor, - discardCorruptObjects: nextMinor, - }, - }, }); const results = await secondRun(); diff --git a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_archive_utils.ts b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_archive_utils.ts index 3e1fa8664fbfd..4361f97ff25a8 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_archive_utils.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_archive_utils.ts @@ -18,7 +18,6 @@ const execPromise = promisify(exec); import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; import { SavedObjectsType } from '@kbn/core-saved-objects-server'; import { - currentVersion, defaultKibanaIndex, getKibanaMigratorTestKit, startElasticsearch, @@ -26,9 +25,13 @@ import { import { delay } from './test_utils'; import { baselineTypes, getBaselineDocuments } from './kibana_migrator_test_kit.fixtures'; -export const BASELINE_ELASTICSEARCH_VERSION = currentVersion; +export const BASELINE_ELASTICSEARCH_VERSION = '9.0.0'; export const BASELINE_DOCUMENTS_PER_TYPE_1K = 200; export const BASELINE_DOCUMENTS_PER_TYPE_500K = 100_000; +// we discard the second half with exclude on upgrade (firstHalf !== true) +// then we discard half all multiples of 100 (1% of them) +export const BASELINE_COMPLEX_DOCUMENTS_500K_AFTER = + BASELINE_DOCUMENTS_PER_TYPE_500K / 2 - BASELINE_DOCUMENTS_PER_TYPE_500K / 2 / 100; export const BASELINE_TEST_ARCHIVE_1K = join( __dirname, diff --git a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.expect.ts b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.expect.ts new file mode 100644 index 0000000000000..2b220aca150e9 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.expect.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { baselineTypes } from './kibana_migrator_test_kit.fixtures'; + +export async function expectDocumentsMigratedToHighestVersion( + client: ElasticsearchClient, + index: string | string[] +) { + const typeMigrationVersions: Record = { + basic: '10.1.0', // did not define any model versions + server: '10.1.0', // did not define any model versions + deprecated: '10.1.0', // did not define any model versions + complex: '10.2.0', + task: '10.2.0', + }; + + const resultSets = await Promise.all( + baselineTypes.map(({ name: type }) => + client.search({ + index, + query: { + bool: { + should: [ + { + term: { type }, + }, + ], + }, + }, + }) + ) + ); + + const notUpgraded = resultSets + .flatMap((result) => result.hits.hits) + .find( + (document) => + document._source.typeMigrationVersion !== typeMigrationVersions[document._source.type] + ); + expect(notUpgraded).toBeUndefined(); +} diff --git a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.fixtures.ts b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.fixtures.ts index 1ec300c075ff2..36887dd02a146 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.fixtures.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.fixtures.ts @@ -10,6 +10,7 @@ import type { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; import type { IndexTypesMap } from '@kbn/core-saved-objects-base-server-internal'; +import type { ElasticsearchClientWrapperFactory } from './elasticsearch_client_wrapper'; import { currentVersion, defaultKibanaIndex, @@ -267,6 +268,7 @@ interface GetMutatedMigratorParams { filterDeprecated?: boolean; types?: Array>; settings?: Record; + clientWrapperFactory?: ElasticsearchClientWrapperFactory; } export const getUpToDateMigratorTestKit = async ({ @@ -304,12 +306,48 @@ export const getReindexingMigratorTestKit = async ({ filterDeprecated = false, types = getReindexingBaselineTypes(filterDeprecated), kibanaVersion = nextMinor, + clientWrapperFactory, settings = {}, }: GetMutatedMigratorParams = {}) => { return await getKibanaMigratorTestKit({ logFilePath, types, kibanaVersion, + clientWrapperFactory, + settings: { + ...settings, + migrations: { + discardUnknownObjects: nextMinor, + discardCorruptObjects: nextMinor, + ...settings.migrations, + }, + }, + }); +}; + +export const kibanaSplitIndex = `${defaultKibanaIndex}_split`; +export const getRelocatingMigratorTestKit = async ({ + logFilePath = defaultLogFilePath, + filterDeprecated = false, + // relocate 'task' and 'basic' objects to a new SO index + relocateTypes = { + task: kibanaSplitIndex, + basic: kibanaSplitIndex, + }, + types = getReindexingBaselineTypes(filterDeprecated).map((type) => ({ + ...type, + ...(relocateTypes[type.name] && { indexPattern: relocateTypes[type.name] }), + })), + kibanaVersion = nextMinor, + clientWrapperFactory, + settings = {}, +}: GetMutatedMigratorParams & { relocateTypes?: Record } = {}) => { + return await getKibanaMigratorTestKit({ + logFilePath, + types, + kibanaVersion, + clientWrapperFactory, + defaultIndexTypesMap: baselineIndexTypesMap, settings: { ...settings, migrations: { diff --git a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts index 7b1459cad1f2d..70b92843add76 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts @@ -94,7 +94,7 @@ export const startElasticsearch = async ({ esVersion, basePath, dataArchive, - timeout, + timeout = 60000, }: { esVersion?: string; basePath?: string;