diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index bf1918e409fb3..d4f5936e2ad57 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -71,6 +71,7 @@ export interface CreateTestEsClusterOptions { * `['key.1=val1', 'key.2=val2']` */ esArgs?: string[]; + esVersion?: string; esFrom?: string; esServerlessOptions?: Pick< ServerlessOptions, @@ -169,6 +170,7 @@ export function createTestEsCluster< log, writeLogsToPath, basePath = Path.resolve(REPO_ROOT, '.es'), + esVersion = esTestConfig.getVersion(), esFrom = esTestConfig.getBuildFrom(), esServerlessOptions, dataArchive, @@ -196,7 +198,7 @@ export function createTestEsCluster< const esArgs = assignArgs(defaultEsArgs, customEsArgs); const config = { - version: esTestConfig.getVersion(), + version: esVersion, installPath: Path.resolve(basePath, clusterName), sourcePath: Path.resolve(REPO_ROOT, '../elasticsearch'), password, diff --git a/src/core/server/integration_tests/saved_objects/migrations/archives/7.13_1.5k_failed_action_tasks.zip b/src/core/server/integration_tests/saved_objects/migrations/archives/7.13_1.5k_failed_action_tasks.zip deleted file mode 100644 index b445dbdfeb435..0000000000000 Binary files a/src/core/server/integration_tests/saved_objects/migrations/archives/7.13_1.5k_failed_action_tasks.zip and /dev/null differ diff --git a/src/core/server/integration_tests/saved_objects/migrations/archives/9.0.0_baseline_1k_docs.zip b/src/core/server/integration_tests/saved_objects/migrations/archives/9.0.0_baseline_1k_docs.zip new file mode 100644 index 0000000000000..4e75d6adf7661 Binary files /dev/null and b/src/core/server/integration_tests/saved_objects/migrations/archives/9.0.0_baseline_1k_docs.zip differ diff --git a/src/core/server/integration_tests/saved_objects/migrations/archives/1m_dummy_so.zip b/src/core/server/integration_tests/saved_objects/migrations/archives/9.0.0_baseline_500k_docs.zip similarity index 54% rename from src/core/server/integration_tests/saved_objects/migrations/archives/1m_dummy_so.zip rename to src/core/server/integration_tests/saved_objects/migrations/archives/9.0.0_baseline_500k_docs.zip index 18b34d8b1ccb3..f744bc91b25cb 100644 Binary files a/src/core/server/integration_tests/saved_objects/migrations/archives/1m_dummy_so.zip and b/src/core/server/integration_tests/saved_objects/migrations/archives/9.0.0_baseline_500k_docs.zip differ diff --git a/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts deleted file mode 100644 index 3cd7451e71014..0000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import path from 'path'; -import { unlink } from 'fs/promises'; -import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import type { InternalCoreStart } from '@kbn/core-lifecycle-server-internal'; -import { Root } from '@kbn/core-root-server-internal'; -import { - createTestServers, - createRootWithCorePlugins, - type TestElasticsearchUtils, -} from '@kbn/core-test-helpers-kbn-server'; -import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; - -const logFilePath = path.join(__dirname, '7.7.2_xpack_100k.log'); - -async function removeLogFile() { - // ignore errors if it doesn't exist - await unlink(logFilePath).catch(() => void 0); -} - -/** Number of SO documents dropped during the migration because they belong to an unused type */ -const UNUSED_SO_COUNT = 5; - -// Failing 9.0 version update: https://github.com/elastic/kibana/issues/192624 -describe.skip('migration from 7.7.2-xpack with 100k objects', () => { - let esServer: TestElasticsearchUtils; - let root: Root; - let coreStart: InternalCoreStart; - let esClient: ElasticsearchClient; - - beforeEach(() => { - jest.setTimeout(600000); - }); - - const startServers = async ({ dataArchive, oss }: { dataArchive: string; oss: boolean }) => { - const { startES } = createTestServers({ - adjustTimeout: (t: number) => jest.setTimeout(600000), - settings: { - es: { - license: 'trial', - dataArchive, - }, - }, - }); - - root = createRootWithCorePlugins( - { - migrations: { - skip: false, - }, - logging: { - appenders: { - file: { - type: 'file', - fileName: logFilePath, - layout: { - type: 'json', - }, - }, - }, - loggers: [ - { - name: 'root', - level: 'info', - appenders: ['file'], - }, - ], - }, - }, - { - oss, - } - ); - - const startEsPromise = startES().then((es) => (esServer = es)); - const startKibanaPromise = root - .preboot() - .then(() => root.setup()) - .then(() => root.start()) - .then((start) => { - coreStart = start; - esClient = coreStart.elasticsearch.client.asInternalUser; - }); - - await Promise.all([startEsPromise, startKibanaPromise]); - }; - - const stopServers = async () => { - if (root) { - await root.shutdown(); - } - if (esServer) { - await esServer.stop(); - } - - await new Promise((resolve) => setTimeout(resolve, 20000)); - }; - - beforeAll(async () => { - await removeLogFile(); - await startServers({ - oss: false, - dataArchive: path.join(__dirname, '..', 'archives', '7.7.2_xpack_100k_obj.zip'), - }); - }); - - afterAll(async () => { - await stopServers(); - }); - - it('copies all the document of the previous index to the new one', async () => { - const migratedIndexResponse = await esClient.count({ - index: ALL_SAVED_OBJECT_INDICES, - }); - const oldIndexResponse = await esClient.count({ - index: '.kibana_1', - }); - - // Use a >= comparison since once Kibana has started it might create new - // documents like telemetry tasks - expect(migratedIndexResponse.count).toBeGreaterThanOrEqual( - oldIndexResponse.count - UNUSED_SO_COUNT - ); - }); -}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group1/7_13_0_failed_action_tasks.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group1/7_13_0_failed_action_tasks.test.ts deleted file mode 100644 index 7575cc4c93dba..0000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/group1/7_13_0_failed_action_tasks.test.ts +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import Path from 'path'; -import fs from 'fs/promises'; -import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import { Root } from '@kbn/core-root-server-internal'; -import { - createRootWithCorePlugins, - createTestServers, - TestElasticsearchUtils, -} from '@kbn/core-test-helpers-kbn-server'; - -const logFilePath = Path.join(__dirname, '7_13_failed_action_tasks.log'); - -async function removeLogFile() { - // ignore errors if it doesn't exist - await fs.unlink(logFilePath).catch(() => void 0); -} - -// Failing 9.0 version update: https://github.com/elastic/kibana/issues/192624 -describe.skip('migration from 7.13 to 7.14+ with many failed action_tasks', () => { - describe('if mappings are incompatible (reindex required)', () => { - let esServer: TestElasticsearchUtils; - let root: Root; - let startES: () => Promise; - - beforeAll(async () => { - await removeLogFile(); - }); - - beforeEach(() => { - ({ startES } = createTestServers({ - adjustTimeout: (t: number) => jest.setTimeout(t), - settings: { - es: { - license: 'basic', - dataArchive: Path.join( - __dirname, - '..', - 'archives', - '7.13_1.5k_failed_action_tasks.zip' - ), - }, - }, - })); - }); - - afterEach(async () => { - if (root) { - await root.shutdown(); - } - if (esServer) { - await esServer.stop(); - } - }); - - const getCounts = async ( - kibanaIndexName = '.kibana', - taskManagerIndexName = '.kibana_task_manager' - ): Promise<{ tasksCount: number; actionTaskParamsCount: number }> => { - const esClient: ElasticsearchClient = esServer.es.getClient(); - - const actionTaskParamsResponse = await esClient.count({ - index: kibanaIndexName, - body: { - query: { - bool: { must: { term: { type: 'action_task_params' } } }, - }, - }, - }); - const tasksResponse = await esClient.count({ - index: taskManagerIndexName, - body: { - query: { - bool: { must: { term: { type: 'task' } } }, - }, - }, - }); - - return { - actionTaskParamsCount: actionTaskParamsResponse.count, - tasksCount: tasksResponse.count, - }; - }; - - it('filters out all outdated action_task_params and action tasks', async () => { - esServer = await startES(); - - // Verify counts in current index before migration starts - expect(await getCounts()).toEqual({ - actionTaskParamsCount: 2010, - tasksCount: 2020, - }); - - root = createRoot(); - await root.preboot(); - await root.setup(); - await root.start(); - - // Bulk of tasks should have been filtered out of current index - const { actionTaskParamsCount, tasksCount } = await getCounts(); - // Use toBeLessThan to avoid flakiness in the case that TM starts manipulating docs before the counts are taken - expect(actionTaskParamsCount).toBeLessThan(1000); - expect(tasksCount).toBeLessThan(1000); - - const { - actionTaskParamsCount: oldIndexActionTaskParamsCount, - tasksCount: oldIndexTasksCount, - } = await getCounts('.kibana_7.13.5_001', '.kibana_task_manager_7.13.5_001'); - - // .kibana mappings changes are NOT compatible, we reindex and preserve old index's documents - expect(oldIndexActionTaskParamsCount).toEqual(2010); - - // ATM .kibana_task_manager mappings changes are compatible, we skip reindex and actively delete unwanted documents - // if the mappings become incompatible in the future, the we will reindex and the old index must still contain all 2020 docs - // if the mappings remain compatible, we reuse the existing index and actively delete unwanted documents from it - expect(oldIndexTasksCount === 2020 || oldIndexTasksCount < 1000).toEqual(true); - }); - }); -}); - -function createRoot() { - return createRootWithCorePlugins( - { - migrations: { - skip: false, - batchSize: 250, - }, - logging: { - appenders: { - file: { - type: 'file', - fileName: logFilePath, - layout: { - type: 'json', - }, - }, - }, - loggers: [ - { - name: 'root', - level: 'info', - appenders: ['file'], - }, - ], - }, - }, - { - oss: false, - } - ); -} diff --git a/src/core/server/integration_tests/saved_objects/migrations/group1/create_test_archives.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group1/create_test_archives.test.ts new file mode 100644 index 0000000000000..924885b1729bd --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/group1/create_test_archives.test.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { getBaselineDocuments } from '../kibana_migrator_test_kit.fixtures'; +import { + BASELINE_DOCUMENTS_PER_TYPE_1K, + BASELINE_DOCUMENTS_PER_TYPE_500K, + BASELINE_ELASTICSEARCH_VERSION, + BASELINE_TEST_ARCHIVE_1K, + BASELINE_TEST_ARCHIVE_500K, + createBaselineArchive, +} from '../kibana_migrator_archive_utils'; + +/** + * Enable and execute this test ONLY IN YOUR DEV MACHINE, in order to build new test packages + */ +describe.skip('migration tests toolkit', () => { + it('can create a 1k documents ZIP archive', async () => { + await createBaselineArchive({ + esVersion: BASELINE_ELASTICSEARCH_VERSION, + documents: getBaselineDocuments({ documentsPerType: BASELINE_DOCUMENTS_PER_TYPE_1K }), + dataArchive: BASELINE_TEST_ARCHIVE_1K, + }); + }); + + it('can create a 400k documents ZIP archive', async () => { + await createBaselineArchive({ + esVersion: BASELINE_ELASTICSEARCH_VERSION, + documents: getBaselineDocuments({ documentsPerType: BASELINE_DOCUMENTS_PER_TYPE_500K }), + dataArchive: BASELINE_TEST_ARCHIVE_500K, + }); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group1/v2_migration.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group1/v2_migration.test.ts new file mode 100644 index 0000000000000..a150a449db7fc --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/group1/v2_migration.test.ts @@ -0,0 +1,157 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { join } from 'path'; +import { omit } from 'lodash'; +import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import type { MigrationResult } from '@kbn/core-saved-objects-base-server-internal'; + +import { + nextMinor, + defaultKibanaIndex, + defaultKibanaTaskIndex, + startElasticsearch, + getAggregatedTypesCount, + type KibanaMigratorTestKit, + readLog, + clearLog, +} from '../kibana_migrator_test_kit'; +import { + BASELINE_DOCUMENTS_PER_TYPE_500K, + BASELINE_TEST_ARCHIVE_500K, +} from '../kibana_migrator_archive_utils'; +import { getReindexingMigratorTestKit } from '../kibana_migrator_test_kit.fixtures'; +import { delay } from '../test_utils'; + +const logFilePath = join(__dirname, 'v2_migration.log'); + +describe('v2 migration', () => { + let esServer: TestElasticsearchUtils; + let kit: KibanaMigratorTestKit; + let migrationResults: MigrationResult[]; + + beforeAll(async () => { + esServer = await startElasticsearch({ dataArchive: BASELINE_TEST_ARCHIVE_500K }); + await clearLog(logFilePath); + kit = await getReindexingMigratorTestKit({ + logFilePath, + filterDeprecated: true, + settings: { + migrations: { + discardUnknownObjects: nextMinor, + }, + }, + }); + migrationResults = await kit.runMigrations(); + }); + + afterAll(async () => { + if (esServer) { + await esServer.stop(); + await delay(5); // give it a few seconds... cause we always do ¯\_(ツ)_/¯ + } + }); + + describe('a migrator performing a reindexing migration', () => { + describe('when an index contains SO types with incompatible mappings', () => { + it('executes the reindexing migration steps', async () => { + const logs = await readLog(logFilePath); + expect(logs).toMatch(`[${defaultKibanaIndex}] INIT -> WAIT_FOR_YELLOW_SOURCE.`); + expect(logs).toMatch( + `[${defaultKibanaIndex}] WAIT_FOR_YELLOW_SOURCE -> UPDATE_SOURCE_MAPPINGS_PROPERTIES.` + ); + expect(logs).toMatch( + `[${defaultKibanaIndex}] UPDATE_SOURCE_MAPPINGS_PROPERTIES -> CHECK_CLUSTER_ROUTING_ALLOCATION.` + ); + expect(logs).toMatch( + `[${defaultKibanaIndex}] CHECK_CLUSTER_ROUTING_ALLOCATION -> CHECK_UNKNOWN_DOCUMENTS.` + ); + expect(logs).toMatch( + `[${defaultKibanaIndex}] CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES.` + ); + expect(logs).toMatch( + `[${defaultKibanaIndex}] UPDATE_TARGET_MAPPINGS_META -> CHECK_VERSION_INDEX_READY_ACTIONS.` + ); + expect(logs).toMatch( + `[${defaultKibanaIndex}] CHECK_VERSION_INDEX_READY_ACTIONS -> MARK_VERSION_INDEX_READY.` + ); + expect(logs).toMatch(`[${defaultKibanaIndex}] MARK_VERSION_INDEX_READY -> DONE.`); + + expect(logs).not.toMatch(`[${defaultKibanaIndex}] CREATE_NEW_TARGET`); + expect(logs).not.toMatch(`[${defaultKibanaIndex}] CLEANUP_UNKNOWN_AND_EXCLUDED`); + expect(logs).not.toMatch(`[${defaultKibanaIndex}] PREPARE_COMPATIBLE_MIGRATION`); + }); + }); + + describe('copies the right documents over to the target indices', () => { + let primaryIndexCounts: Record; + let taskIndexCounts: Record; + + beforeAll(async () => { + primaryIndexCounts = await getAggregatedTypesCount(kit.client, defaultKibanaIndex); + taskIndexCounts = await getAggregatedTypesCount(kit.client, defaultKibanaTaskIndex); + }); + + it('copies documents to the right indices depending on their types', () => { + expect(primaryIndexCounts.basic).toBeDefined(); + expect(primaryIndexCounts.complex).toBeDefined(); + expect(primaryIndexCounts.task).not.toBeDefined(); + + expect(taskIndexCounts.basic).not.toBeDefined(); + expect(taskIndexCounts.complex).not.toBeDefined(); + expect(taskIndexCounts.task).toBeDefined(); + }); + + it('discards REMOVED_TYPES', () => { + expect(primaryIndexCounts.server).not.toBeDefined(); + expect(taskIndexCounts.server).not.toBeDefined(); + }); + + it('discards unknown types', () => { + expect(primaryIndexCounts.deprecated).not.toBeDefined(); + expect(taskIndexCounts.deprecated).not.toBeDefined(); + }); + + it('copies all of the documents', () => { + expect(primaryIndexCounts.basic).toEqual(BASELINE_DOCUMENTS_PER_TYPE_500K); + expect(taskIndexCounts.task).toEqual(BASELINE_DOCUMENTS_PER_TYPE_500K); + }); + + it('executes the excludeOnUpgrade hook', () => { + expect(primaryIndexCounts.complex).toEqual(BASELINE_DOCUMENTS_PER_TYPE_500K / 2); + }); + }); + + it('returns a migrated status for each SO index', () => { + // omit elapsedMs as it varies in each execution + expect(migrationResults.map((result) => omit(result, 'elapsedMs'))).toMatchInlineSnapshot(` + Array [ + Object { + "destIndex": ".kibana_migrator_9.1.0_001", + "sourceIndex": ".kibana_migrator_9.0.0_001", + "status": "migrated", + }, + Object { + "destIndex": ".kibana_migrator_tasks_9.0.0_001", + "sourceIndex": ".kibana_migrator_tasks_9.0.0_001", + "status": "migrated", + }, + ] + `); + }); + + it('each migrator takes less than 60 seconds', () => { + expect( + (migrationResults as Array<{ elapsedMs?: number }>).every( + ({ elapsedMs }) => !elapsedMs || elapsedMs < 60000 + ) + ).toEqual(true); + }); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/batch_size_bytes_exceeds_es_content_length.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/batch_size_bytes_exceeds_es_content_length.test.ts deleted file mode 100644 index f08765a24052c..0000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/batch_size_bytes_exceeds_es_content_length.test.ts +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import Path from 'path'; -import fs from 'fs/promises'; -import JSON5 from 'json5'; -import { - createTestServers, - createRootWithCorePlugins, - type TestElasticsearchUtils, -} from '@kbn/core-test-helpers-kbn-server'; -import { retryAsync } from '@kbn/core-saved-objects-migration-server-mocks'; -import { Root } from '@kbn/core-root-server-internal'; - -const logFilePath = Path.join(__dirname, 'batch_size_bytes_exceeds_es_content_length.log'); - -async function removeLogFile() { - // ignore errors if it doesn't exist - await fs.unlink(logFilePath).catch(() => void 0); -} - -// Failing 9.0 version update: https://github.com/elastic/kibana/issues/192624 -describe.skip('migration v2', () => { - let esServer: TestElasticsearchUtils; - let root: Root; - let startES: () => Promise; - - beforeAll(async () => { - await removeLogFile(); - }); - - beforeEach(() => { - ({ startES } = createTestServers({ - adjustTimeout: (t: number) => jest.setTimeout(t), - settings: { - es: { - license: 'basic', - dataArchive: Path.join( - __dirname, - '..', - 'archives', - '7.14.0_xpack_sample_saved_objects.zip' - ), - esArgs: ['http.max_content_length=1mb'], - }, - }, - })); - }); - - afterEach(async () => { - if (root) { - await root.shutdown(); - } - if (esServer) { - await esServer.stop(); - } - }); - - it('fails with a descriptive message when maxBatchSizeBytes exceeds ES http.max_content_length', async () => { - root = createRoot({ maxBatchSizeBytes: 1715329 }); - esServer = await startES(); - await root.preboot(); - await root.setup(); - await expect(root.start()).rejects.toMatchInlineSnapshot( - `[Error: Unable to complete saved object migrations for the [.kibana] index: While indexing a batch of saved objects, Elasticsearch returned a 413 Request Entity Too Large exception. Ensure that the Kibana configuration option 'migrations.maxBatchSizeBytes' is set to a value that is lower than or equal to the Elasticsearch 'http.max_content_length' configuration option.]` - ); - - await retryAsync( - async () => { - const logFileContent = await fs.readFile(logFilePath, 'utf-8'); - const records = logFileContent - .split('\n') - .filter(Boolean) - .map((str) => JSON5.parse(str)) as any[]; - - expect( - records.find((rec) => - rec.message.startsWith( - `Reason: Unable to complete saved object migrations for the [.kibana] index: While indexing a batch of saved objects, Elasticsearch returned a 413 Request Entity Too Large exception. Ensure that the Kibana configuration option 'migrations.maxBatchSizeBytes' is set to a value that is lower than or equal to the Elasticsearch 'http.max_content_length' configuration option.` - ) - ) - ).toBeDefined(); - }, - { retryAttempts: 10, retryDelayMs: 200 } - ); - }); -}); - -function createRoot(options: { maxBatchSizeBytes?: number }) { - return createRootWithCorePlugins( - { - migrations: { - skip: false, - batchSize: 1000, - maxBatchSizeBytes: options.maxBatchSizeBytes, - }, - logging: { - appenders: { - file: { - type: 'file', - fileName: logFilePath, - layout: { - type: 'json', - }, - }, - }, - loggers: [ - { - name: 'root', - level: 'info', - appenders: ['file'], - }, - ], - }, - }, - { - oss: false, - } - ); -} diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/actions/actions.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/actions/actions.test.ts index 7b90206b5e8ae..0559d059a6cb1 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/actions/actions.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/actions/actions.test.ts @@ -2031,27 +2031,5 @@ describe.skip('migration actions', () => { } `); }); - - it('resolves left request_entity_too_large_exception when the payload is too large', async () => { - const newDocs = new Array(10000).fill({ - _source: { - title: - 'how do I create a document thats large enoug to exceed the limits without typing long sentences', - }, - }) as SavedObjectsRawDoc[]; - const task = bulkOverwriteTransformedDocuments({ - client, - index: 'existing_index_with_docs', - operations: newDocs.map((doc) => createBulkIndexOperationTuple(doc)), - }); - await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Left", - "left": Object { - "type": "request_entity_too_large_exception", - }, - } - `); - }); }); }); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/actions/actions_test_suite.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/actions/actions_test_suite.ts index 32429b3383a31..3b7aae9183a54 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/actions/actions_test_suite.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/actions/actions_test_suite.ts @@ -2086,30 +2086,5 @@ export const runActionTestSuite = ({ } `); }); - - // no way to configure http.max_content_length on the serverless instance for now. - runOnTraditionalOnly(() => { - it('resolves left request_entity_too_large_exception when the payload is too large', async () => { - const newDocs = new Array(10000).fill({ - _source: { - title: - 'how do I create a document thats large enoug to exceed the limits without typing long sentences', - }, - }) as SavedObjectsRawDoc[]; - const task = bulkOverwriteTransformedDocuments({ - client, - index: 'existing_index_with_docs', - operations: newDocs.map((doc) => createBulkIndexOperationTuple(doc)), - }); - await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Left", - "left": Object { - "type": "request_entity_too_large_exception", - }, - } - `); - }); - }); }); }; diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/fail_on_rollback.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/fail_on_rollback.test.ts index 50a2fc8d3ae33..caf9a9189dfbd 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/fail_on_rollback.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/fail_on_rollback.test.ts @@ -15,6 +15,7 @@ import { getKibanaMigratorTestKit, nextMinor, defaultKibanaIndex, + defaultKibanaTaskIndex, } from '../kibana_migrator_test_kit'; import '../jest_matchers'; import { delay, parseLogFile } from '../test_utils'; @@ -51,12 +52,19 @@ describe('when rolling back to an older version', () => { const { runMigrations: rollback } = await getKibanaMigratorTestKit({ types, logFilePath }); await clearLog(logFilePath); - await expect(rollback()).rejects.toThrowError( - `Unable to complete saved object migrations for the [${defaultKibanaIndex}] index: The ${defaultKibanaIndex}_${nextMinor} alias refers to a newer version of Kibana: v${nextMinor}` - ); + + try { + await rollback(); + throw new Error('Rollback should have thrown but it did not'); + } catch (error) { + expect([ + `Unable to complete saved object migrations for the [${defaultKibanaIndex}] index: The ${defaultKibanaIndex}_${nextMinor} alias refers to a newer version of Kibana: v${nextMinor}`, + `Unable to complete saved object migrations for the [${defaultKibanaTaskIndex}] index: The ${defaultKibanaTaskIndex}_${nextMinor} alias refers to a newer version of Kibana: v${nextMinor}`, + ]).toContain(error.message); + } const logs = await parseLogFile(logFilePath); - expect(logs).toContainLogEntry('[.kibana_migrator_tests] INIT -> FATAL.'); + expect(logs).toContainLogEntry(`[${defaultKibanaIndex}] INIT -> FATAL.`); }); afterAll(async () => { diff --git a/src/core/server/integration_tests/saved_objects/migrations/group5/active_delete.fixtures.ts b/src/core/server/integration_tests/saved_objects/migrations/group5/active_delete.fixtures.ts deleted file mode 100644 index c260c01a406ff..0000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/group5/active_delete.fixtures.ts +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import type { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; -import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; - -const defaultType: SavedObjectsType = { - name: 'defaultType', - hidden: false, - namespaceType: 'agnostic', - mappings: { - properties: { - name: { type: 'keyword' }, - }, - }, - migrations: {}, -}; - -export const baselineTypes: Array> = [ - { - ...defaultType, - name: 'server', - }, - { - ...defaultType, - name: 'basic', - }, - { - ...defaultType, - name: 'deprecated', - }, - { - ...defaultType, - name: 'complex', - mappings: { - properties: { - name: { type: 'text' }, - value: { type: 'integer' }, - }, - }, - excludeOnUpgrade: () => { - return { - bool: { - must: [{ term: { type: 'complex' } }, { range: { 'complex.value': { lte: 1 } } }], - }, - }; - }, - }, -]; - -export const baselineDocuments: SavedObjectsBulkCreateObject[] = [ - ...['server-foo', 'server-bar', 'server-baz'].map((name) => ({ - type: 'server', - attributes: { - name, - }, - })), - ...['basic-foo', 'basic-bar', 'basic-baz'].map((name) => ({ - type: 'basic', - attributes: { - name, - }, - })), - ...['deprecated-foo', 'deprecated-bar', 'deprecated-baz'].map((name) => ({ - type: 'deprecated', - attributes: { - name, - }, - })), - ...['complex-foo', 'complex-bar', 'complex-baz', 'complex-lipsum'].map((name, index) => ({ - type: 'complex', - attributes: { - name, - value: index, - }, - })), -]; diff --git a/src/core/server/integration_tests/saved_objects/migrations/group5/active_delete.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group5/active_delete.test.ts index c8bca4869ceed..8cfcb1ea5b745 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group5/active_delete.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group5/active_delete.test.ts @@ -14,16 +14,19 @@ import { readLog, clearLog, nextMinor, - createBaseline, currentVersion, defaultKibanaIndex, startElasticsearch, - getCompatibleMappingsMigrator, - getIdenticalMappingsMigrator, - getIncompatibleMappingsMigrator, - getNonDeprecatedMappingsMigrator, + getAggregatedTypesCount, } from '../kibana_migrator_test_kit'; +import { + createBaseline, + getCompatibleMigratorTestKit, + getUpToDateMigratorTestKit, + getReindexingMigratorTestKit, +} from '../kibana_migrator_test_kit.fixtures'; + describe('when upgrading to a new stack version', () => { let esServer: TestElasticsearchUtils['es']; let esClient: ElasticsearchClient; @@ -41,11 +44,12 @@ describe('when upgrading to a new stack version', () => { let indexContents: SearchResponse<{ type: string }, Record>; beforeAll(async () => { - esClient = await createBaseline(); + esClient = await createBaseline({ documentsPerType: 10 }); await clearLog(); // remove the 'deprecated' type from the mappings, so that it is considered unknown - const { client, runMigrations } = await getNonDeprecatedMappingsMigrator({ + const { client, runMigrations } = await getUpToDateMigratorTestKit({ + filterDeprecated: true, settings: { migrations: { discardUnknownObjects: nextMinor, @@ -88,7 +92,7 @@ describe('when upgrading to a new stack version', () => { describe('CLEANUP_UNKNOWN_AND_EXCLUDED', () => { it('preserves documents with known types', async () => { - expect(countResultsByType(indexContents, 'basic')).toEqual(3); + expect(countResultsByType(indexContents, 'basic')).toEqual(10); }); it('deletes documents with unknown types', async () => { @@ -104,24 +108,19 @@ describe('when upgrading to a new stack version', () => { (result) => result._source?.type === 'complex' ); - expect(complexDocuments.length).toEqual(2); - expect(complexDocuments[0]._source).toEqual( - expect.objectContaining({ - complex: { - name: 'complex-baz', - value: 2, - }, - type: 'complex', - }) - ); - expect(complexDocuments[1]._source).toEqual( - expect.objectContaining({ - complex: { - name: 'complex-lipsum', - value: 3, - }, - type: 'complex', - }) + expect(complexDocuments.length).toEqual(5); + + complexDocuments.forEach(({ _source }, value) => + expect(_source).toEqual( + expect.objectContaining({ + complex: { + name: `complex-${value}`, + firstHalf: true, + value, + }, + type: 'complex', + }) + ) ); }); }); @@ -129,7 +128,7 @@ describe('when upgrading to a new stack version', () => { describe('and discardUnknownObjects = false', () => { beforeAll(async () => { - esClient = await createBaseline(); + esClient = await createBaseline({ documentsPerType: 10 }); }); afterAll(async () => { await esClient?.indices.delete({ index: `${defaultKibanaIndex}_${currentVersion}_001` }); @@ -140,14 +139,16 @@ describe('when upgrading to a new stack version', () => { it('fails if unknown documents exist', async () => { // remove the 'deprecated' type from the mappings, so that it is considered unknown - const { runMigrations } = await getNonDeprecatedMappingsMigrator(); + const { runMigrations } = await getUpToDateMigratorTestKit({ + filterDeprecated: true, + }); try { await runMigrations(); } catch (err) { const errorMessage = err.message; expect(errorMessage).toMatch( - 'Unable to complete saved object migrations for the [.kibana_migrator_tests] index: Migration failed because some documents were found which use unknown saved object types:' + `Unable to complete saved object migrations for the [${defaultKibanaIndex}] index: Migration failed because some documents were found which use unknown saved object types:` ); expect(errorMessage).toMatch( 'To proceed with the migration you can configure Kibana to discard unknown saved objects for this migration.' @@ -163,7 +164,7 @@ describe('when upgrading to a new stack version', () => { }); it('proceeds if there are no unknown documents', async () => { - const { client, runMigrations } = await getIdenticalMappingsMigrator(); + const { client, runMigrations } = await getUpToDateMigratorTestKit(); await runMigrations(); @@ -183,7 +184,7 @@ describe('when upgrading to a new stack version', () => { expect(logs).toMatch('CHECK_VERSION_INDEX_READY_ACTIONS -> DONE.'); const indexContents = await client.search({ index: defaultKibanaIndex, size: 100 }); - expect(indexContents.hits.hits.length).toEqual(8); + expect(indexContents.hits.hits.length).toEqual(25); }); }); }); @@ -193,10 +194,10 @@ describe('when upgrading to a new stack version', () => { let indexContents: SearchResponse<{ type: string }, Record>; beforeAll(async () => { - esClient = await createBaseline(); + esClient = await createBaseline({ documentsPerType: 10 }); await clearLog(); - const { client, runMigrations } = await getCompatibleMappingsMigrator({ + const { client, runMigrations } = await getCompatibleMigratorTestKit({ filterDeprecated: true, // remove the 'deprecated' type from the mappings, so that it is considered unknown settings: { migrations: { @@ -243,7 +244,7 @@ describe('when upgrading to a new stack version', () => { describe('CLEANUP_UNKNOWN_AND_EXCLUDED', () => { it('preserves documents with known types', async () => { - expect(countResultsByType(indexContents, 'basic')).toEqual(3); + expect(countResultsByType(indexContents, 'basic')).toEqual(10); }); it('deletes documents with unknown types', async () => { @@ -259,24 +260,19 @@ describe('when upgrading to a new stack version', () => { (result) => result._source?.type === 'complex' ); - expect(complexDocuments.length).toEqual(2); - expect(complexDocuments[0]._source).toEqual( - expect.objectContaining({ - complex: { - name: 'complex-baz', - value: 2, - }, - type: 'complex', - }) - ); - expect(complexDocuments[1]._source).toEqual( - expect.objectContaining({ - complex: { - name: 'complex-lipsum', - value: 3, - }, - type: 'complex', - }) + expect(complexDocuments.length).toEqual(5); + + complexDocuments.forEach(({ _source }, value) => + expect(_source).toEqual( + expect.objectContaining({ + complex: { + name: `complex-${value}`, + firstHalf: true, + value, + }, + type: 'complex', + }) + ) ); }); }); @@ -284,7 +280,7 @@ describe('when upgrading to a new stack version', () => { describe('and discardUnknownObjects = false', () => { beforeAll(async () => { - esClient = await createBaseline(); + esClient = await createBaseline({ documentsPerType: 10 }); }); afterAll(async () => { await esClient?.indices.delete({ index: `${defaultKibanaIndex}_${currentVersion}_001` }); @@ -294,7 +290,7 @@ describe('when upgrading to a new stack version', () => { }); it('fails if unknown documents exist', async () => { - const { runMigrations } = await getCompatibleMappingsMigrator({ + const { runMigrations } = await getCompatibleMigratorTestKit({ filterDeprecated: true, // remove the 'deprecated' type from the mappings, so that it is considered unknown }); @@ -303,7 +299,7 @@ describe('when upgrading to a new stack version', () => { } catch (err) { const errorMessage = err.message; expect(errorMessage).toMatch( - 'Unable to complete saved object migrations for the [.kibana_migrator_tests] index: Migration failed because some documents were found which use unknown saved object types:' + `Unable to complete saved object migrations for the [${defaultKibanaIndex}] index: Migration failed because some documents were found which use unknown saved object types:` ); expect(errorMessage).toMatch( 'To proceed with the migration you can configure Kibana to discard unknown saved objects for this migration.' @@ -312,14 +308,18 @@ describe('when upgrading to a new stack version', () => { } const logs = await readLog(); - expect(logs).toMatch('INIT -> WAIT_FOR_YELLOW_SOURCE.'); - expect(logs).toMatch('WAIT_FOR_YELLOW_SOURCE -> UPDATE_SOURCE_MAPPINGS_PROPERTIES.'); // this step is run only if mappings are compatible but NOT equal - expect(logs).toMatch('UPDATE_SOURCE_MAPPINGS_PROPERTIES -> CLEANUP_UNKNOWN_AND_EXCLUDED.'); - expect(logs).toMatch('CLEANUP_UNKNOWN_AND_EXCLUDED -> FATAL.'); + expect(logs).toMatch(`[${defaultKibanaIndex}] INIT -> WAIT_FOR_YELLOW_SOURCE.`); + expect(logs).toMatch( + `[${defaultKibanaIndex}] WAIT_FOR_YELLOW_SOURCE -> UPDATE_SOURCE_MAPPINGS_PROPERTIES.` + ); // this step is run only if mappings are compatible but NOT equal + expect(logs).toMatch( + `[${defaultKibanaIndex}] UPDATE_SOURCE_MAPPINGS_PROPERTIES -> CLEANUP_UNKNOWN_AND_EXCLUDED.` + ); + expect(logs).toMatch(`[${defaultKibanaIndex}] CLEANUP_UNKNOWN_AND_EXCLUDED -> FATAL.`); }); it('proceeds if there are no unknown documents', async () => { - const { client, runMigrations } = await getCompatibleMappingsMigrator(); + const { client, runMigrations } = await getCompatibleMigratorTestKit(); await runMigrations(); @@ -341,14 +341,14 @@ describe('when upgrading to a new stack version', () => { const indexContents = await client.search({ index: defaultKibanaIndex, size: 100 }); - expect(indexContents.hits.hits.length).toEqual(8); + expect(indexContents.hits.hits.length).toEqual(25); }); }); }); describe('if the mappings do NOT match (diffMappings() === true) and they are NOT compatible', () => { beforeAll(async () => { - esClient = await createBaseline(); + esClient = await createBaseline({ documentsPerType: 10 }); }); afterAll(async () => { await esClient?.indices.delete({ index: `${defaultKibanaIndex}_${currentVersion}_001` }); @@ -358,7 +358,7 @@ describe('when upgrading to a new stack version', () => { }); it('the migrator does not skip reindexing', async () => { - const { client, runMigrations } = await getIncompatibleMappingsMigrator(); + const { client, runMigrations } = await getReindexingMigratorTestKit(); await runMigrations(); @@ -375,15 +375,15 @@ describe('when upgrading to a new stack version', () => { expect(logs).toMatch('CHECK_VERSION_INDEX_READY_ACTIONS -> MARK_VERSION_INDEX_READY.'); expect(logs).toMatch('MARK_VERSION_INDEX_READY -> DONE'); - const indexContents: SearchResponse< - { type: string }, - Record - > = await client.search({ index: defaultKibanaIndex, size: 100 }); - - expect(indexContents.hits.hits.length).toEqual(8); // we're removing a couple of 'complex' (value < = 1) - - // double-check that the deprecated documents have not been deleted - expect(countResultsByType(indexContents, 'deprecated')).toEqual(3); + const counts = await getAggregatedTypesCount(client); + expect(counts).toMatchInlineSnapshot(` + Object { + "basic": 10, + "complex": 5, + "deprecated": 10, + "task": 10, + } + `); }); }); }); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group5/active_delete_multiple_instances.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group5/active_delete_multiple_instances.test.ts index f74c62734b288..9fc7dc80bf6af 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group5/active_delete_multiple_instances.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group5/active_delete_multiple_instances.test.ts @@ -7,115 +7,64 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import Path from 'path'; -import fs from 'fs/promises'; -import { SemVer } from 'semver'; -import { Env } from '@kbn/config'; -import { getEnvOptions } from '@kbn/config-mocks'; -import { REPO_ROOT } from '@kbn/repo-info'; +import { join } from 'path'; +import { readFile, unlink } from 'fs/promises'; import { type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import type { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; -import { DEFAULT_INDEX_TYPES_MAP } from '@kbn/core-saved-objects-base-server-internal'; import { defaultLogFilePath, + getAggregatedTypesCount, getEsClient, - getKibanaMigratorTestKit, + nextMinor, startElasticsearch, } from '../kibana_migrator_test_kit'; -import { baselineTypes } from './active_delete.fixtures'; -import { createBaselineArchive } from '../kibana_migrator_archive_utils'; +import { getUpToDateMigratorTestKit } from '../kibana_migrator_test_kit.fixtures'; +import { + BASELINE_TEST_ARCHIVE_500K, + BASELINE_DOCUMENTS_PER_TYPE_500K, +} from '../kibana_migrator_archive_utils'; const PARALLEL_MIGRATORS = 6; -const DOCUMENTS_PER_TYPE = 250000; - -const kibanaIndex = '.kibana_migrator_tests'; -const currentVersion = Env.createDefault(REPO_ROOT, getEnvOptions()).packageInfo.version; -const nextMinor = new SemVer(currentVersion).inc('minor').format(); - -const dataArchive = Path.join(__dirname, '..', 'archives', '1m_dummy_so.zip'); - jest.setTimeout(24 * 3600 * 100); -// Failing 9.0 version update: https://github.com/elastic/kibana/issues/192624 -describe.skip('multiple migrator instances running in parallel', () => { - it.skip('enable and focus this test (it.skip => fit), and run it, in order to create a baseline archive', async () => { - // generate DOCUMENTS_PER_TYPE documents of each type - const documents: SavedObjectsBulkCreateObject[] = ['server', 'basic', 'deprecated', 'complex'] - .map((type) => - new Array(DOCUMENTS_PER_TYPE).fill(true).map((_, index) => ({ - type, - attributes: { - name: `${type}-${++index}`, - ...(type === 'complex' && { value: index }), - }, - })) - ) - .flat(); - - await createBaselineArchive({ kibanaIndex, types: baselineTypes, documents, dataArchive }); - }); - +describe('multiple migrator instances running in parallel', () => { describe('when upgrading to a new stack version with matching mappings', () => { let esServer: TestElasticsearchUtils['es']; let esClient: ElasticsearchClient; beforeAll(async () => { - esServer = await startElasticsearch({ dataArchive }); + esServer = await startElasticsearch({ dataArchive: BASELINE_TEST_ARCHIVE_500K }); esClient = await getEsClient(); - await fs.unlink(defaultLogFilePath).catch(() => {}); + await unlink(defaultLogFilePath).catch(() => {}); for (let i = 0; i < PARALLEL_MIGRATORS; ++i) { - await fs.unlink(Path.join(__dirname, `active_delete_instance_${i}.log`)).catch(() => {}); + await unlink(join(__dirname, `active_delete_instance_${i}.log`)).catch(() => {}); } }); - it('will actively delete and successfully complete migration', async () => { + it('will actively delete and sccessfully complete migration', async () => { const startTime = Date.now(); - const types = baselineTypes - .filter((type) => type.name !== 'deprecated') - .map((type) => { - if (type.name !== 'complex') { - return type; - } - return { - ...type, - excludeOnUpgrade: () => { - return { - bool: { - must: [ - { term: { type: 'complex' } }, - { range: { 'complex.value': { lte: 125000 } } }, - ], - }, - }; - }, - }; - }); - - const beforeCleanup = await getAggregatedTypesCount(); - expect(beforeCleanup.server).toEqual(DOCUMENTS_PER_TYPE); - expect(beforeCleanup.basic).toEqual(DOCUMENTS_PER_TYPE); - expect(beforeCleanup.deprecated).toEqual(DOCUMENTS_PER_TYPE); - expect(beforeCleanup.complex).toEqual(DOCUMENTS_PER_TYPE); + const beforeCleanup = await getAggregatedTypesCount(esClient); + expect(beforeCleanup.server).toEqual(BASELINE_DOCUMENTS_PER_TYPE_500K); + expect(beforeCleanup.basic).toEqual(BASELINE_DOCUMENTS_PER_TYPE_500K); + expect(beforeCleanup.deprecated).toEqual(BASELINE_DOCUMENTS_PER_TYPE_500K); + expect(beforeCleanup.complex).toEqual(BASELINE_DOCUMENTS_PER_TYPE_500K); + expect(beforeCleanup.task).toEqual(BASELINE_DOCUMENTS_PER_TYPE_500K); const testKits = await Promise.all( new Array(PARALLEL_MIGRATORS) .fill({ + filterDeprecated: true, settings: { migrations: { discardUnknownObjects: nextMinor, }, }, - kibanaIndex, - types, - kibanaVersion: nextMinor, }) .map((config, index) => - getKibanaMigratorTestKit({ + getUpToDateMigratorTestKit({ ...config, - logFilePath: Path.join(__dirname, `active_delete_instance_${index}.log`), - defaultIndexTypesMap: DEFAULT_INDEX_TYPES_MAP, + logFilePath: join(__dirname, `active_delete_instance_${index}.log`), }) ) ); @@ -124,10 +73,7 @@ describe.skip('multiple migrator instances running in parallel', () => { expect(results.flat().every((result) => result.status === 'migrated')).toEqual(true); for (let i = 0; i < PARALLEL_MIGRATORS; ++i) { - const logs = await fs.readFile( - Path.join(__dirname, `active_delete_instance_${i}.log`), - 'utf-8' - ); + const logs = await readFile(join(__dirname, `active_delete_instance_${i}.log`), 'utf-8'); expect(logs).toMatch('CHECK_VERSION_INDEX_READY_ACTIONS -> DONE'); expect(logs).toMatch('Migration completed'); } @@ -137,52 +83,16 @@ describe.skip('multiple migrator instances running in parallel', () => { console.debug(`Migration took: ${(endTime - startTime) / 1000} seconds`); // After cleanup - const afterCleanup = await getAggregatedTypesCount(); + const afterCleanup = await getAggregatedTypesCount(testKits[0].client); expect(afterCleanup.server).not.toBeDefined(); // 'server' is part of the REMOVED_TYPES - expect(afterCleanup.basic).toEqual(DOCUMENTS_PER_TYPE); // we keep 'basic' SOs + expect(afterCleanup.basic).toEqual(BASELINE_DOCUMENTS_PER_TYPE_500K); // we keep 'basic' SOs expect(afterCleanup.deprecated).not.toBeDefined(); // 'deprecated' is no longer present in nextMinor's mappings - expect(afterCleanup.complex).toEqual(DOCUMENTS_PER_TYPE / 2); // we excludeFromUpgrade half of them with a hook + expect(afterCleanup.complex).toEqual(BASELINE_DOCUMENTS_PER_TYPE_500K / 2); // we excludeFromUpgrade half of them with a hook + expect(afterCleanup.task).toEqual(BASELINE_DOCUMENTS_PER_TYPE_500K); // 'task' SO are on a dedicated index }); afterAll(async () => { - // await esClient?.indices.delete({ index: `${kibanaIndex}_${currentVersion}_001` }); await esServer?.stop(); }); - - const getAggregatedTypesCount = async () => { - await esClient.indices.refresh(); - const response = await esClient.search({ - index: kibanaIndex, - _source: false, - aggs: { - typesAggregation: { - terms: { - // assign type __UNKNOWN__ to those documents that don't define one - missing: '__UNKNOWN__', - field: 'type', - size: 10, - }, - aggs: { - docs: { - top_hits: { - size: 2, - _source: { - excludes: ['*'], - }, - }, - }, - }, - }, - }, - }); - - return (response.aggregations!.typesAggregation.buckets as unknown as any).reduce( - (acc: any, current: any) => { - acc[current.key] = current.doc_count; - return acc; - }, - {} - ); - }; }); }); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts index 3b9d647b25166..58dfb5d4e433e 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts @@ -14,6 +14,7 @@ import { type SavedObjectsType, MAIN_SAVED_OBJECT_INDEX, ALL_SAVED_OBJECT_INDICES, + TASK_MANAGER_SAVED_OBJECT_INDEX, } from '@kbn/core-saved-objects-server'; import { DEFAULT_INDEX_TYPES_MAP } from '@kbn/core-saved-objects-base-server-internal'; import { @@ -26,7 +27,6 @@ import { currentVersion, type KibanaMigratorTestKit, getEsClient, - getAggregatedTypesCountAllIndices, } from '../kibana_migrator_test_kit'; import { delay, parseLogFile } from '../test_utils'; import '../jest_matchers'; @@ -319,7 +319,10 @@ describe.skip('split .kibana index into multiple system indices', () => { }); const esClient = await getEsClient(); - const breakdownBefore = await getAggregatedTypesCountAllIndices(esClient); + const breakdownBefore = await getAggregatedTypesCount(esClient, [ + MAIN_SAVED_OBJECT_INDEX, + TASK_MANAGER_SAVED_OBJECT_INDEX, + ]); expect(breakdownBefore).toEqual({ '.kibana': { 'apm-telemetry': 1, @@ -368,7 +371,10 @@ describe.skip('split .kibana index into multiple system indices', () => { await esClient.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); - const breakdownAfter = await getAggregatedTypesCountAllIndices(esClient); + const breakdownAfter = await getAggregatedTypesCount(esClient, [ + MAIN_SAVED_OBJECT_INDEX, + TASK_MANAGER_SAVED_OBJECT_INDEX, + ]); expect(breakdownAfter).toEqual({ '.kibana': { 'apm-telemetry': 1, diff --git a/src/core/server/integration_tests/saved_objects/migrations/group5/pickup_updated_types_only.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group5/pickup_updated_types_only.test.ts index b9ad71090b132..81cf4e22774a8 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group5/pickup_updated_types_only.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group5/pickup_updated_types_only.test.ts @@ -9,15 +9,14 @@ import Path from 'path'; import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { clearLog, defaultKibanaIndex, startElasticsearch } from '../kibana_migrator_test_kit'; + import { - clearLog, createBaseline, - defaultKibanaIndex, - getCompatibleMappingsMigrator, - getIdenticalMappingsMigrator, - getIncompatibleMappingsMigrator, - startElasticsearch, -} from '../kibana_migrator_test_kit'; + getCompatibleMigratorTestKit, + getUpToDateMigratorTestKit, + getReindexingMigratorTestKit, +} from '../kibana_migrator_test_kit.fixtures'; import '../jest_matchers'; import { delay, parseLogFile } from '../test_utils'; @@ -37,22 +36,24 @@ describe('pickupUpdatedMappings', () => { describe('when performing a reindexing migration', () => { it('should pickup all documents from the index', async () => { - const { runMigrations } = await getIncompatibleMappingsMigrator({ logFilePath }); + const { runMigrations } = await getReindexingMigratorTestKit({ logFilePath }); await runMigrations(); const logs = await parseLogFile(logFilePath); - expect(logs).not.toContainLogEntry('Documents of the following SO types will be updated'); expect(logs).not.toContainLogEntry( - 'There are no changes in the mappings of any of the SO types, skipping UPDATE_TARGET_MAPPINGS steps.' + `[${defaultKibanaIndex}] Documents of the following SO types will be updated` + ); + expect(logs).not.toContainLogEntry( + `[${defaultKibanaIndex}] There are no changes in the mappings of any of the SO types, skipping UPDATE_TARGET_MAPPINGS steps.` ); }); }); describe('when performing a compatible migration', () => { it('should pickup only the types that have been updated', async () => { - const { runMigrations } = await getCompatibleMappingsMigrator({ logFilePath }); + const { runMigrations } = await getCompatibleMigratorTestKit({ logFilePath }); await runMigrations(); @@ -64,7 +65,7 @@ describe('pickupUpdatedMappings', () => { }); it('should NOT pickup any documents if only root fields have been updated', async () => { - const { runMigrations, client } = await getIdenticalMappingsMigrator({ logFilePath }); + const { runMigrations, client } = await getUpToDateMigratorTestKit({ logFilePath }); // we tamper the baseline mappings to simulate some root fields changes const baselineMappings = await client.indices.getMapping({ index: defaultKibanaIndex }); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group5/skip_reindex.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group5/skip_reindex.test.ts index 59de973eb4d30..a921471f08587 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group5/skip_reindex.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group5/skip_reindex.test.ts @@ -12,16 +12,18 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { readLog, clearLog, - createBaseline, currentVersion, defaultKibanaIndex, - getCompatibleMappingsMigrator, - getIdenticalMappingsMigrator, - getIncompatibleMappingsMigrator, startElasticsearch, KibanaMigratorTestKit, } from '../kibana_migrator_test_kit'; +import { + createBaseline, + getCompatibleMigratorTestKit, + getUpToDateMigratorTestKit, +} from '../kibana_migrator_test_kit.fixtures'; + describe('when migrating to a new version', () => { let esServer: TestElasticsearchUtils['es']; let esClient: ElasticsearchClient; @@ -39,7 +41,7 @@ describe('when migrating to a new version', () => { describe('and the mappings remain the same', () => { it('the migrator skips reindexing', async () => { // we run the migrator with the same identic baseline types - migratorTestKitFactory = () => getIdenticalMappingsMigrator(); + migratorTestKitFactory = () => getUpToDateMigratorTestKit(); const testKit = await migratorTestKitFactory(); await testKit.runMigrations(); @@ -68,7 +70,7 @@ describe('when migrating to a new version', () => { describe("and the mappings' changes are still compatible", () => { it('the migrator skips reindexing', async () => { // we run the migrator with altered, compatible mappings - migratorTestKitFactory = () => getCompatibleMappingsMigrator(); + migratorTestKitFactory = () => getCompatibleMigratorTestKit(); const testKit = await migratorTestKitFactory(); await testKit.runMigrations(); @@ -94,31 +96,6 @@ describe('when migrating to a new version', () => { }); }); - describe("and the mappings' changes are NOT compatible", () => { - it('the migrator reindexes documents to a new index', async () => { - // we run the migrator with incompatible mappings - migratorTestKitFactory = () => getIncompatibleMappingsMigrator(); - const testKit = await migratorTestKitFactory(); - await testKit.runMigrations(); - - const logs = await readLog(); - expect(logs).toMatch('INIT -> WAIT_FOR_YELLOW_SOURCE.'); - expect(logs).toMatch('WAIT_FOR_YELLOW_SOURCE -> UPDATE_SOURCE_MAPPINGS_PROPERTIES.'); - expect(logs).toMatch( - 'UPDATE_SOURCE_MAPPINGS_PROPERTIES -> CHECK_CLUSTER_ROUTING_ALLOCATION.' - ); - expect(logs).toMatch('CHECK_CLUSTER_ROUTING_ALLOCATION -> CHECK_UNKNOWN_DOCUMENTS.'); - expect(logs).toMatch('CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES.'); - expect(logs).toMatch('UPDATE_TARGET_MAPPINGS_META -> CHECK_VERSION_INDEX_READY_ACTIONS.'); - expect(logs).toMatch('CHECK_VERSION_INDEX_READY_ACTIONS -> MARK_VERSION_INDEX_READY.'); - expect(logs).toMatch('MARK_VERSION_INDEX_READY -> DONE.'); - - expect(logs).not.toMatch('CREATE_NEW_TARGET'); - expect(logs).not.toMatch('CLEANUP_UNKNOWN_AND_EXCLUDED'); - expect(logs).not.toMatch('PREPARE_COMPATIBLE_MIGRATION'); - }); - }); - afterEach(async () => { // we run the migrator again to ensure that the next time state is loaded everything still works as expected const migratorTestKit = await migratorTestKitFactory(); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group6/single_migrator_failures.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group6/single_migrator_failures.test.ts index bb6b47b3eb6e0..b000f0d775358 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group6/single_migrator_failures.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group6/single_migrator_failures.test.ts @@ -7,35 +7,43 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import Path from 'path'; +import { join } from 'path'; import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; -import { - type ISavedObjectTypeRegistry, - MAIN_SAVED_OBJECT_INDEX, -} from '@kbn/core-saved-objects-server'; -import { DEFAULT_INDEX_TYPES_MAP } from '@kbn/core-saved-objects-base-server-internal'; import { clearLog, + nextMinor, startElasticsearch, + defaultKibanaIndex, + defaultKibanaTaskIndex, getKibanaMigratorTestKit, - getCurrentVersionTypeRegistry, - currentVersion, } from '../kibana_migrator_test_kit'; import { delay } from '../test_utils'; import '../jest_matchers'; import { getElasticsearchClientWrapperFactory } from '../elasticsearch_client_wrapper'; +import { BASELINE_TEST_ARCHIVE_1K } from '../kibana_migrator_archive_utils'; +import { + baselineIndexTypesMap, + getReindexingBaselineTypes, +} from '../kibana_migrator_test_kit.fixtures'; -export const logFilePathFirstRun = Path.join(__dirname, 'dot_kibana_split_1st_run.test.log'); -export const logFilePathSecondRun = Path.join(__dirname, 'dot_kibana_split_2nd_run.test.log'); +export const logFilePathFirstRun = join(__dirname, 'dot_kibana_split_1st_run.test.log'); +export const logFilePathSecondRun = join(__dirname, 'dot_kibana_split_2nd_run.test.log'); -// Failing 9.0 version update: https://github.com/elastic/kibana/issues/192624 -describe.skip('split .kibana index into multiple system indices', () => { - let esServer: TestElasticsearchUtils['es']; - let typeRegistry: ISavedObjectTypeRegistry; +const kibanaSplitIndex = `${defaultKibanaIndex}_split`; +const tasksToNewIndex = getReindexingBaselineTypes(true).map((type) => { + if (type.name !== 'task' && type.name !== 'simple') { + return type; + } - beforeAll(async () => { - typeRegistry = await getCurrentVersionTypeRegistry({ oss: false }); - }); + // relocate 'simple' and 'task' objects to a new index (forces reindex) + return { + ...type, + indexPattern: kibanaSplitIndex, + }; +}); + +describe('split .kibana index into multiple system indices', () => { + let esServer: TestElasticsearchUtils['es']; beforeEach(async () => { await clearLog(logFilePathFirstRun); @@ -58,28 +66,33 @@ describe.skip('split .kibana index into multiple system indices', () => { }); return await getKibanaMigratorTestKit({ - types: typeRegistry.getAllTypes(), - kibanaIndex: MAIN_SAVED_OBJECT_INDEX, - defaultIndexTypesMap: DEFAULT_INDEX_TYPES_MAP, + types: tasksToNewIndex, logFilePath, + kibanaVersion: nextMinor, + defaultIndexTypesMap: baselineIndexTypesMap, clientWrapperFactory, + settings: { + migrations: { + discardUnknownObjects: nextMinor, + }, + }, }); }; beforeEach(async () => { esServer = await startElasticsearch({ - dataArchive: Path.join(__dirname, '..', 'archives', '7.7.2_xpack_100k_obj.zip'), + dataArchive: BASELINE_TEST_ARCHIVE_1K, }); }); - describe('when the .kibana_task_manager migrator fails on the TRANSFORMED_DOCUMENTS_BULK_INDEX state, after the other ones have finished', () => { - it('is capable of completing the .kibana_task_manager migration in subsequent restart', async () => { + describe(`when the ${defaultKibanaTaskIndex} migrator fails on the TRANSFORMED_DOCUMENTS_BULK_INDEX state, after the other ones have finished`, () => { + it(`is capable of completing the ${defaultKibanaTaskIndex} migration in subsequent restart`, async () => { const { runMigrations: firstRun } = await getFailingKibanaMigratorTestKit({ logFilePath: logFilePathFirstRun, failOn: (methodName, methodArgs) => { - // fail on esClient.bulk({ index: '.kibana_task_manager_1' }) which supposedly causes - // the .kibana_task_manager migrator to fail on the TRANSFORMED_DOCUMENTS_BULK_INDEX state - return methodName === 'bulk' && methodArgs[0].index === '.kibana_task_manager_1'; + // fail on esClient.bulk({ index: '.kibana_migrator_tasks' }) which supposedly causes + // the .kibana_migrator_tasks migrator to fail on the TRANSFORMED_DOCUMENTS_BULK_INDEX state + return methodName === 'bulk' && methodArgs[0].index.startsWith(defaultKibanaTaskIndex); }, delaySeconds: 90, // give the other migrators enough time to finish before failing }); @@ -89,25 +102,25 @@ describe.skip('split .kibana index into multiple system indices', () => { throw new Error('First run should have thrown an error but it did not'); } catch (error) { expect(error.message).toEqual( - 'Unable to complete saved object migrations for the [.kibana_task_manager] index. Error: esClient.bulk() failed unexpectedly' + `Unable to complete saved object migrations for the [${defaultKibanaTaskIndex}] index. Error: esClient.bulk() failed unexpectedly` ); } }); }); - describe('when the .kibana migrator fails on the REINDEX_SOURCE_TO_TEMP_INDEX_BULK state', () => { + describe(`when the ${defaultKibanaIndex} migrator fails on the REINDEX_SOURCE_TO_TEMP_INDEX_BULK state`, () => { it('is capable of successfully performing the split migration in subsequent restart', async () => { const { runMigrations: firstRun } = await getFailingKibanaMigratorTestKit({ logFilePath: logFilePathFirstRun, failOn: (methodName, methodArgs) => { - // fail on esClient.bulk({ index: '.kibana_8.11.0_reindex_temp_alias' }) which supposedly causes + // fail on esClient.bulk({ index: '.kibana_migrator_8.11.0_reindex_temp_alias' }) which supposedly causes // the .kibana migrator to fail on the REINDEX_SOURCE_TO_TEMP_INDEX_BULK return ( methodName === 'bulk' && - methodArgs[0].index === `.kibana_${currentVersion}_reindex_temp_alias` + methodArgs[0].index === `${defaultKibanaIndex}_${nextMinor}_reindex_temp_alias` ); }, - delaySeconds: 10, // give the .kibana_task_manager migrator enough time to finish before failing + delaySeconds: 10, // give the .kibana_migrator_tasks migrator enough time to finish before failing }); try { @@ -115,23 +128,23 @@ describe.skip('split .kibana index into multiple system indices', () => { throw new Error('First run should have thrown an error but it did not'); } catch (error) { expect(error.message).toEqual( - 'Unable to complete saved object migrations for the [.kibana] index. Error: esClient.bulk() failed unexpectedly' + `Unable to complete saved object migrations for the [${defaultKibanaIndex}] index. Error: esClient.bulk() failed unexpectedly` ); } }); }); - describe('when the .kibana migrator fails on the CLONE_TEMP_TO_TARGET state', () => { + describe(`when the ${defaultKibanaIndex} migrator fails on the CLONE_TEMP_TO_TARGET state`, () => { it('is capable of successfully performing the split migration in subsequent restart', async () => { const { runMigrations: firstRun } = await getFailingKibanaMigratorTestKit({ logFilePath: logFilePathFirstRun, failOn: (methodName, methodArgs) => { - // fail on esClient.indices.clone({ index: '.kibana_8.11.0_reindex_temp', target: ... }) which supposedly causes - // the .kibana migrator to fail on the CLONE_TEMP_TO_TARGET + // fail on esClient.indices.clone({ index: '.kibana_migrator_8.11.0_reindex_temp', target: ... }) which supposedly causes + // the .kibana_migrator migrator to fail on the CLONE_TEMP_TO_TARGET return ( methodName === 'indices.clone' && - methodArgs[0].index === `.kibana_${currentVersion}_reindex_temp` && - methodArgs[0].target === `.kibana_${currentVersion}_001` + methodArgs[0].index === `${defaultKibanaIndex}_${nextMinor}_reindex_temp` && + methodArgs[0].target === `${defaultKibanaIndex}_${nextMinor}_001` ); }, delaySeconds: 15, // give the other migrators enough time to finish before failing @@ -142,22 +155,22 @@ describe.skip('split .kibana index into multiple system indices', () => { throw new Error('First run should have thrown an error but it did not'); } catch (error) { expect(error.message).toEqual( - 'Unable to complete saved object migrations for the [.kibana] index. Error: esClient.indices.clone() failed unexpectedly' + `Unable to complete saved object migrations for the [${defaultKibanaIndex}] index. Error: esClient.indices.clone() failed unexpectedly` ); } }); }); - describe('when the .kibana migrator fails on the UPDATE_TARGET_MAPPINGS_PROPERTIES state', () => { + describe(`when the ${defaultKibanaIndex} migrator fails on the UPDATE_TARGET_MAPPINGS_PROPERTIES state`, () => { it('is capable of successfully performing the split migration in subsequent restart', async () => { const { runMigrations: firstRun } = await getFailingKibanaMigratorTestKit({ logFilePath: logFilePathFirstRun, failOn: (methodName, methodArgs) => { - // fail on esClient.updateByQuery({ index: '.kibana_8.11.0_001' }) which supposedly causes - // the .kibana migrator to fail on the UPDATE_TARGET_MAPPINGS_PROPERTIES (pickup mappings' changes) + // fail on esClient.updateByQuery({ index: '.kibana_migrator_8.11.0_001' }) which supposedly causes + // the .kibana_migrator migrator to fail on the UPDATE_TARGET_MAPPINGS_PROPERTIES (pickup mappings' changes) return ( methodName === 'updateByQuery' && - methodArgs[0].index === `.kibana_${currentVersion}_001` + methodArgs[0].index === `${defaultKibanaIndex}_${nextMinor}_001` ); }, delaySeconds: 10, // give the other migrators enough time to finish before failing @@ -168,13 +181,13 @@ describe.skip('split .kibana index into multiple system indices', () => { throw new Error('First run should have thrown an error but it did not'); } catch (error) { expect(error.message).toEqual( - 'Unable to complete saved object migrations for the [.kibana] index. Error: esClient.updateByQuery() failed unexpectedly' + `Unable to complete saved object migrations for the [${defaultKibanaIndex}] index. Error: esClient.updateByQuery() failed unexpectedly` ); } }); }); - describe('when the .kibana_analytics migrator fails on the CLONE_TEMP_TO_TARGET state', () => { + describe(`when the ${kibanaSplitIndex} migrator fails on the CLONE_TEMP_TO_TARGET state`, () => { it('is capable of successfully performing the split migration in subsequent restart', async () => { const { runMigrations: firstRun } = await getFailingKibanaMigratorTestKit({ logFilePath: logFilePathFirstRun, @@ -183,8 +196,8 @@ describe.skip('split .kibana index into multiple system indices', () => { // the .kibana migrator to fail on the CLONE_TEMP_TO_TARGET return ( methodName === 'indices.clone' && - methodArgs[0].index === `.kibana_analytics_${currentVersion}_reindex_temp` && - methodArgs[0].target === `.kibana_analytics_${currentVersion}_001` + methodArgs[0].index === `${kibanaSplitIndex}_${nextMinor}_reindex_temp` && + methodArgs[0].target === `${kibanaSplitIndex}_${nextMinor}_001` ); }, delaySeconds: 15, // give the other migrators enough time to finish before failing @@ -195,13 +208,13 @@ describe.skip('split .kibana index into multiple system indices', () => { throw new Error('First run should have thrown an error but it did not'); } catch (error) { expect(error.message).toEqual( - 'Unable to complete saved object migrations for the [.kibana_analytics] index. Error: esClient.indices.clone() failed unexpectedly' + `Unable to complete saved object migrations for the [${kibanaSplitIndex}] index. Error: esClient.indices.clone() failed unexpectedly` ); } }); }); - describe('when the .kibana_analytics migrator fails on the UPDATE_TARGET_MAPPINGS_PROPERTIES state', () => { + describe(`when the ${kibanaSplitIndex} migrator fails on the UPDATE_TARGET_MAPPINGS_PROPERTIES state`, () => { it('is capable of successfully performing the split migration in subsequent restart', async () => { const { runMigrations: firstRun } = await getFailingKibanaMigratorTestKit({ logFilePath: logFilePathFirstRun, @@ -210,7 +223,7 @@ describe.skip('split .kibana index into multiple system indices', () => { // the .kibana migrator to fail on the UPDATE_TARGET_MAPPINGS_PROPERTIES (pickup mappings' changes) return ( methodName === 'updateByQuery' && - methodArgs[0].index === `.kibana_analytics_${currentVersion}_001` + methodArgs[0].index === `${kibanaSplitIndex}_${nextMinor}_001` ); }, delaySeconds: 10, // give the other migrators enough time to finish before failing @@ -221,7 +234,7 @@ describe.skip('split .kibana index into multiple system indices', () => { throw new Error('First run should have thrown an error but it did not'); } catch (error) { expect(error.message).toEqual( - 'Unable to complete saved object migrations for the [.kibana_analytics] index. Error: esClient.updateByQuery() failed unexpectedly' + `Unable to complete saved object migrations for the [${kibanaSplitIndex}] index. Error: esClient.updateByQuery() failed unexpectedly` ); } }); @@ -229,11 +242,17 @@ describe.skip('split .kibana index into multiple system indices', () => { afterEach(async () => { const { runMigrations: secondRun } = await getKibanaMigratorTestKit({ - types: typeRegistry.getAllTypes(), logFilePath: logFilePathSecondRun, - kibanaIndex: MAIN_SAVED_OBJECT_INDEX, - defaultIndexTypesMap: DEFAULT_INDEX_TYPES_MAP, + types: tasksToNewIndex, + kibanaVersion: nextMinor, + defaultIndexTypesMap: baselineIndexTypesMap, + settings: { + migrations: { + discardUnknownObjects: nextMinor, + }, + }, }); + const results = await secondRun(); expect( results diff --git a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_archive_utils.ts b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_archive_utils.ts index 9d35e9b347f05..3e1fa8664fbfd 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_archive_utils.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_archive_utils.ts @@ -9,37 +9,66 @@ /* eslint-disable no-console */ -import Path from 'path'; +import { join } from 'path'; +import fs from 'fs/promises'; import { exec } from 'child_process'; import { promisify } from 'util'; const execPromise = promisify(exec); import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; import { SavedObjectsType } from '@kbn/core-saved-objects-server'; -import { getKibanaMigratorTestKit, startElasticsearch } from './kibana_migrator_test_kit'; +import { + currentVersion, + defaultKibanaIndex, + getKibanaMigratorTestKit, + startElasticsearch, +} from './kibana_migrator_test_kit'; import { delay } from './test_utils'; +import { baselineTypes, getBaselineDocuments } from './kibana_migrator_test_kit.fixtures'; -const DEFAULT_BATCH_SIZE = 100000; +export const BASELINE_ELASTICSEARCH_VERSION = currentVersion; +export const BASELINE_DOCUMENTS_PER_TYPE_1K = 200; +export const BASELINE_DOCUMENTS_PER_TYPE_500K = 100_000; + +export const BASELINE_TEST_ARCHIVE_1K = join( + __dirname, + 'archives', + `${BASELINE_ELASTICSEARCH_VERSION}_baseline_${ + (BASELINE_DOCUMENTS_PER_TYPE_1K * baselineTypes.length) / 1000 + }k_docs.zip` +); + +export const BASELINE_TEST_ARCHIVE_500K = join( + __dirname, + 'archives', + `${BASELINE_ELASTICSEARCH_VERSION}_baseline_${ + (BASELINE_DOCUMENTS_PER_TYPE_500K * baselineTypes.length) / 1000 + }k_docs.zip` +); + +const DEFAULT_BATCH_SIZE = 5000; interface CreateBaselineArchiveParams { - kibanaIndex: string; - types: Array>; - documents: SavedObjectsBulkCreateObject[]; - batchSize?: number; - esBaseFolder?: string; dataArchive: string; + esVersion?: string; + kibanaIndex?: string; + types?: Array>; + documents?: SavedObjectsBulkCreateObject[]; + batchSize?: number; + basePath?: string; } export const createBaselineArchive = async ({ - types, - documents, - kibanaIndex, - batchSize = DEFAULT_BATCH_SIZE, - esBaseFolder = Path.join(__dirname, `target`), dataArchive, + esVersion, + kibanaIndex = defaultKibanaIndex, + types = baselineTypes, + documents = getBaselineDocuments(), + batchSize = DEFAULT_BATCH_SIZE, + basePath = join(__dirname, `target`), }: CreateBaselineArchiveParams) => { const startTime = Date.now(); - const esServer = await startElasticsearch({ basePath: esBaseFolder }); + const esServer = await startElasticsearch({ esVersion, basePath }); const { runMigrations, savedObjectsRepository } = await getKibanaMigratorTestKit({ kibanaIndex, @@ -57,15 +86,22 @@ export const createBaselineArchive = async ({ }); } - await compressBaselineArchive(esBaseFolder, dataArchive); + // wait a bit more to make sure everything's persisted to disk + await delay(30); + + await compressBaselineArchive(basePath, dataArchive); console.log(`Archive created in: ${(Date.now() - startTime) / 1000} seconds`, dataArchive); - await delay(200); + + // leave command line enough time to finish creating + closing ZIP file + await delay(30); + await esServer.stop(); - // await fs.rm(esBaseFolder, { recursive: true }); + await delay(10); + await fs.rm(basePath, { recursive: true, force: true }); }; const compressBaselineArchive = async (esFolder: string, archiveFile: string) => { - const dataFolder = Path.join(esFolder, 'es-test-cluster'); - const cmd = `cd ${dataFolder} && zip -r ${archiveFile} data -x ".DS_Store" -x "__MACOSX"`; + const dataFolder = join(esFolder, 'es-test-cluster', 'data'); + const cmd = `ditto -c -k --sequesterRsrc --keepParent ${dataFolder} ${archiveFile}`; await execPromise(cmd); }; diff --git a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.fixtures.ts b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.fixtures.ts index 21195ed8d8d5a..73437885fc7a7 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.fixtures.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.fixtures.ts @@ -9,6 +9,20 @@ import type { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; +import type { IndexTypesMap } from '@kbn/core-saved-objects-base-server-internal'; +import { + currentVersion, + defaultKibanaIndex, + defaultKibanaTaskIndex, + defaultLogFilePath, + getKibanaMigratorTestKit, + nextMinor, +} from './kibana_migrator_test_kit'; + +export const baselineIndexTypesMap: IndexTypesMap = { + [defaultKibanaIndex]: ['basic', 'complex', 'server', 'deprecated'], + [defaultKibanaTaskIndex]: ['task'], +}; const defaultType: SavedObjectsType = { name: 'defaultType', @@ -41,6 +55,11 @@ export const baselineTypes: Array> = [ ...defaultType, name: 'deprecated', }, + { + ...defaultType, + name: 'task', + indexPattern: `${defaultKibanaIndex}_tasks`, + }, { ...defaultType, name: 'complex', @@ -48,42 +67,214 @@ export const baselineTypes: Array> = [ properties: { name: { type: 'text' }, value: { type: 'integer' }, + firstHalf: { type: 'boolean' }, }, }, excludeOnUpgrade: () => { return { bool: { - must: [{ term: { type: 'complex' } }, { range: { 'complex.value': { lte: 1 } } }], + must: [{ term: { type: 'complex' } }, { term: { 'complex.firstHalf': false } }], }, }; }, }, ]; -export const baselineDocuments: SavedObjectsBulkCreateObject[] = [ - ...['server-foo', 'server-bar', 'server-baz'].map((name) => ({ - type: 'server', - attributes: { - name, - }, - })), - ...['basic-foo', 'basic-bar', 'basic-baz'].map((name) => ({ - type: 'basic', - attributes: { - name, - }, - })), - ...['deprecated-foo', 'deprecated-bar', 'deprecated-baz'].map((name) => ({ - type: 'deprecated', - attributes: { - name, - }, - })), - ...['complex-foo', 'complex-bar', 'complex-baz', 'complex-lipsum'].map((name, index) => ({ - type: 'complex', - attributes: { - name, - value: index, - }, - })), -]; +export const getUpToDateBaselineTypes = (filterDeprecated: boolean) => + baselineTypes.filter((type) => !filterDeprecated || type.name !== 'deprecated'); + +export const getCompatibleBaselineTypes = (filterDeprecated: boolean) => + getUpToDateBaselineTypes(filterDeprecated).map((type) => { + // introduce a compatible change + if (type.name === 'complex') { + return { + ...type, + mappings: { + properties: { + ...type.mappings.properties, + createdAt: { type: 'date' }, + }, + }, + modelVersions: { + ...type.modelVersions, + 2: { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + createdAt: { type: 'date' }, + }, + }, + ], + }, + }, + }; + } else { + return type; + } + }); + +export const getReindexingBaselineTypes = (filterDeprecated: boolean) => + getUpToDateBaselineTypes(filterDeprecated).map((type) => { + // introduce an incompatible change + if (type.name === 'complex') { + return { + ...type, + mappings: { + properties: { + ...type.mappings.properties, + value: { type: 'text' }, // we're forcing an incompatible udpate (number => text) + createdAt: { type: 'date' }, + }, + }, + modelVersions: { + ...type.modelVersions, + 2: { + changes: [ + { + type: 'data_removal', // not true (we're testing reindex migrations, and modelVersions do not support breaking changes) + removedAttributePaths: ['complex.properties.value'], + }, + { + type: 'mappings_addition', + addedMappings: { + createdAt: { type: 'date' }, + }, + }, + ], + }, + }, + }; + } else { + return type; + } + }); + +export interface GetBaselineDocumentsParams { + documentsPerType?: number; +} + +export const getBaselineDocuments = ( + params: GetBaselineDocumentsParams = {} +): SavedObjectsBulkCreateObject[] => { + const documentsPerType = params.documentsPerType ?? 4; + + return [ + ...new Array(documentsPerType).fill(true).map((_, index) => ({ + type: 'server', + attributes: { + name: `server-${index}`, + }, + })), + ...new Array(documentsPerType).fill(true).map((_, index) => ({ + type: 'basic', + attributes: { + name: `basic-${index}`, + }, + })), + ...new Array(documentsPerType).fill(true).map((_, index) => ({ + type: 'deprecated', + attributes: { + name: `deprecated-${index}`, + }, + })), + ...new Array(documentsPerType).fill(true).map((_, index) => ({ + type: 'task', + attributes: { + name: `task-${index}`, + }, + })), + ...new Array(documentsPerType).fill(true).map((_, index) => ({ + type: 'complex', + attributes: { + name: `complex-${index}`, + firstHalf: index < documentsPerType / 2, + value: index, + }, + })), + ]; +}; + +export interface CreateBaselineParams { + documentsPerType?: number; +} + +export const createBaseline = async (params: CreateBaselineParams = {}) => { + const { client, runMigrations, savedObjectsRepository } = await getKibanaMigratorTestKit({ + kibanaIndex: defaultKibanaIndex, + types: baselineTypes, + }); + + // remove the testing indices (current and next minor) + await client.indices.delete({ + index: [ + defaultKibanaIndex, + `${defaultKibanaIndex}_${currentVersion}_001`, + `${defaultKibanaIndex}_${nextMinor}_001`, + defaultKibanaTaskIndex, + `${defaultKibanaTaskIndex}_${currentVersion}_001`, + `${defaultKibanaTaskIndex}_${nextMinor}_001`, + ], + ignore_unavailable: true, + }); + + await runMigrations(); + + await savedObjectsRepository.bulkCreate(getBaselineDocuments(params), { + refresh: 'wait_for', + }); + + return client; +}; + +interface GetMutatedMigratorParams { + logFilePath?: string; + kibanaVersion?: string; + filterDeprecated?: boolean; + types?: Array>; + settings?: Record; +} + +export const getUpToDateMigratorTestKit = async ({ + logFilePath = defaultLogFilePath, + filterDeprecated = false, + kibanaVersion = nextMinor, + settings = {}, +}: GetMutatedMigratorParams = {}) => { + return await getKibanaMigratorTestKit({ + types: getUpToDateBaselineTypes(filterDeprecated), + logFilePath, + kibanaVersion, + settings, + }); +}; + +export const getCompatibleMigratorTestKit = async ({ + logFilePath = defaultLogFilePath, + filterDeprecated = false, + kibanaVersion = nextMinor, + settings = {}, +}: GetMutatedMigratorParams & { + filterDeprecated?: boolean; +} = {}) => { + return await getKibanaMigratorTestKit({ + logFilePath, + types: getCompatibleBaselineTypes(filterDeprecated), + kibanaVersion, + settings, + }); +}; + +export const getReindexingMigratorTestKit = async ({ + logFilePath = defaultLogFilePath, + filterDeprecated = false, + kibanaVersion = nextMinor, + settings = {}, +}: GetMutatedMigratorParams = {}) => { + return await getKibanaMigratorTestKit({ + logFilePath, + types: getReindexingBaselineTypes(filterDeprecated), + kibanaVersion, + settings, + }); +}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts index e8fd86952771f..7b1459cad1f2d 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts @@ -49,7 +49,6 @@ import type { ISavedObjectsRepository } from '@kbn/core-saved-objects-api-server import { getDocLinks, getDocLinksMeta } from '@kbn/doc-links'; import type { DocLinksServiceStart } from '@kbn/core-doc-links-server'; import type { NodeRoles } from '@kbn/core-node-server'; -import { baselineDocuments, baselineTypes } from './kibana_migrator_test_kit.fixtures'; import { delay } from './test_utils'; import type { ElasticsearchClientWrapperFactory } from './elasticsearch_client_wrapper'; @@ -60,7 +59,8 @@ const env = Env.createDefault(REPO_ROOT, getEnvOptions()); export const currentVersion = env.packageInfo.version; export const nextMinor = new SemVer(currentVersion).inc('minor').format(); export const currentBranch = env.packageInfo.branch; -export const defaultKibanaIndex = '.kibana_migrator_tests'; +export const defaultKibanaIndex = '.kibana_migrator'; +export const defaultKibanaTaskIndex = `${defaultKibanaIndex}_tasks`; export const defaultNodeRoles: NodeRoles = { migrator: true, ui: true, backgroundTasks: true }; export interface GetEsClientParams { @@ -91,10 +91,12 @@ export interface KibanaMigratorTestKit { } export const startElasticsearch = async ({ + esVersion, basePath, dataArchive, timeout, }: { + esVersion?: string; basePath?: string; dataArchive?: string; timeout?: number; @@ -106,6 +108,7 @@ export const startElasticsearch = async ({ license: 'basic', basePath, dataArchive, + esVersion, }, }, }); @@ -345,8 +348,8 @@ export const deleteSavedObjectIndices = async ( export const getAggregatedTypesCount = async ( client: ElasticsearchClient, - index: string -): Promise | undefined> => { + index: string | string[] = [defaultKibanaIndex, defaultKibanaTaskIndex] +): Promise> => { try { await client.indices.refresh({ index }); const response = await client.search({ @@ -383,26 +386,12 @@ export const getAggregatedTypesCount = async ( ); } catch (error) { if (error.meta?.statusCode === 404) { - return undefined; + return {}; } throw error; } }; -export const getAggregatedTypesCountAllIndices = async (esClient: ElasticsearchClient) => { - const typeBreakdown = await Promise.all( - ALL_SAVED_OBJECT_INDICES.map((index) => getAggregatedTypesCount(esClient, index)) - ); - - return ALL_SAVED_OBJECT_INDICES.reduce | undefined>>( - (acc, index, pos) => { - acc[index] = typeBreakdown[pos]; - return acc; - }, - {} - ); -}; - const registerTypes = ( typeRegistry: SavedObjectTypeRegistry, types?: Array> @@ -410,157 +399,6 @@ const registerTypes = ( (types || []).forEach((type) => typeRegistry.registerType(type)); }; -export const createBaseline = async () => { - const { client, runMigrations, savedObjectsRepository } = await getKibanaMigratorTestKit({ - kibanaIndex: defaultKibanaIndex, - types: baselineTypes, - }); - - // remove the testing index (current and next minor) - await client.indices.delete({ - index: [ - defaultKibanaIndex, - `${defaultKibanaIndex}_${currentVersion}_001`, - `${defaultKibanaIndex}_${nextMinor}_001`, - ], - ignore_unavailable: true, - }); - - await runMigrations(); - - await savedObjectsRepository.bulkCreate(baselineDocuments, { - refresh: 'wait_for', - }); - - return client; -}; - -interface GetMutatedMigratorParams { - logFilePath?: string; - kibanaVersion?: string; - settings?: Record; -} - -export const getIdenticalMappingsMigrator = async ({ - logFilePath = defaultLogFilePath, - kibanaVersion = nextMinor, - settings = {}, -}: GetMutatedMigratorParams = {}) => { - return await getKibanaMigratorTestKit({ - logFilePath, - types: baselineTypes, - kibanaVersion, - settings, - }); -}; - -export const getNonDeprecatedMappingsMigrator = async ({ - logFilePath = defaultLogFilePath, - kibanaVersion = nextMinor, - settings = {}, -}: GetMutatedMigratorParams = {}) => { - return await getKibanaMigratorTestKit({ - logFilePath, - types: baselineTypes.filter((type) => type.name !== 'deprecated'), - kibanaVersion, - settings, - }); -}; - -export const getCompatibleMappingsMigrator = async ({ - logFilePath = defaultLogFilePath, - filterDeprecated = false, - kibanaVersion = nextMinor, - settings = {}, -}: GetMutatedMigratorParams & { - filterDeprecated?: boolean; -} = {}) => { - const types = baselineTypes - .filter((type) => !filterDeprecated || type.name !== 'deprecated') - .map((type) => { - if (type.name === 'complex') { - return { - ...type, - mappings: { - properties: { - ...type.mappings.properties, - createdAt: { type: 'date' }, - }, - }, - modelVersions: { - ...type.modelVersions, - 2: { - changes: [ - { - type: 'mappings_addition', - addedMappings: { - createdAt: { type: 'date' }, - }, - }, - ], - }, - }, - }; - } else { - return type; - } - }); - - return await getKibanaMigratorTestKit({ - logFilePath, - types, - kibanaVersion, - settings, - }); -}; - -export const getIncompatibleMappingsMigrator = async ({ - logFilePath = defaultLogFilePath, - kibanaVersion = nextMinor, - settings = {}, -}: GetMutatedMigratorParams = {}) => { - const types = baselineTypes.map((type) => { - if (type.name === 'complex') { - return { - ...type, - mappings: { - properties: { - ...type.mappings.properties, - value: { type: 'text' }, // we're forcing an incompatible udpate (number => text) - createdAt: { type: 'date' }, - }, - }, - modelVersions: { - ...type.modelVersions, - 2: { - changes: [ - { - type: 'data_removal', // not true (we're testing reindex migrations, and modelVersions do not support breaking changes) - removedAttributePaths: ['complex.properties.value'], - }, - { - type: 'mappings_addition', - addedMappings: { - createdAt: { type: 'date' }, - }, - }, - ], - }, - }, - }; - } else { - return type; - } - }); - - return await getKibanaMigratorTestKit({ - logFilePath, - types, - kibanaVersion, - settings, - }); -}; - export const getCurrentVersionTypeRegistry = async ({ oss, }: {