diff --git a/src/core/server/integration_tests/saved_objects/migrations/archives/7.13.2_so_with_multiple_namespaces.zip b/src/core/server/integration_tests/saved_objects/migrations/archives/7.13.2_so_with_multiple_namespaces.zip deleted file mode 100644 index bc305de2d4560..0000000000000 Binary files a/src/core/server/integration_tests/saved_objects/migrations/archives/7.13.2_so_with_multiple_namespaces.zip and /dev/null differ diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/batch_size_bytes.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/batch_size_bytes.test.ts deleted file mode 100644 index a360fd4114d91..0000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/batch_size_bytes.test.ts +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import Path from 'path'; -import fs from 'fs/promises'; -import JSON5 from 'json5'; -import { - createTestServers, - createRootWithCorePlugins, - type TestElasticsearchUtils, -} from '@kbn/core-test-helpers-kbn-server'; -import { Root } from '@kbn/core-root-server-internal'; -import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import { Env } from '@kbn/config'; -import { REPO_ROOT } from '@kbn/repo-info'; -import { getEnvOptions } from '@kbn/config-mocks'; -import { LogRecord } from '@kbn/logging'; -import { retryAsync } from '@kbn/core-saved-objects-migration-server-mocks'; - -const kibanaVersion = Env.createDefault(REPO_ROOT, getEnvOptions()).packageInfo.version; -const targetIndex = `.kibana_${kibanaVersion}_001`; -const logFilePath = Path.join(__dirname, 'batch_size_bytes.log'); - -async function removeLogFile() { - // ignore errors if it doesn't exist - await fs.unlink(logFilePath).catch(() => void 0); -} -function sortByTypeAndId(a: { type: string; id: string }, b: { type: string; id: string }) { - return a.type.localeCompare(b.type) || a.id.localeCompare(b.id); -} - -async function fetchDocuments(esClient: ElasticsearchClient, index: string) { - const body = await esClient.search({ - index, - body: { - query: { - match_all: {}, - }, - _source: ['type', 'id'], - }, - }); - - return body.hits.hits - .map((h) => ({ - ...h._source, - id: h._id, - })) - .sort(sortByTypeAndId); -} - -const assertMigratedDocuments = (arr: any[], target: any[]) => target.every((v) => arr.includes(v)); - -// Failing 9.0 version update: https://github.com/elastic/kibana/issues/192624 -describe.skip('migration v2', () => { - let esServer: TestElasticsearchUtils; - let root: Root; - let startES: () => Promise; - - beforeAll(async () => { - await removeLogFile(); - }); - - beforeEach(() => { - ({ startES } = createTestServers({ - adjustTimeout: (t: number) => jest.setTimeout(t), - settings: { - es: { - license: 'basic', - dataArchive: Path.join( - __dirname, - '..', - 'archives', - '7.14.0_xpack_sample_saved_objects.zip' - ), - esArgs: ['http.max_content_length=1715329b'], - }, - }, - })); - }); - - afterEach(async () => { - if (root) { - await root.shutdown(); - } - if (esServer) { - await esServer.stop(); - } - }); - - it('completes the migration even when a full batch would exceed ES http.max_content_length', async () => { - root = createRoot({ maxBatchSizeBytes: 1715329 }); - esServer = await startES(); - await root.preboot(); - await root.setup(); - await expect(root.start()).resolves.toBeTruthy(); - - // After plugins start, some saved objects are deleted/recreated, so we - // wait a bit for the count to settle. - await new Promise((resolve) => setTimeout(resolve, 5000)); - - const esClient: ElasticsearchClient = esServer.es.getClient(); - - // assert that the docs from the original index have been migrated rather than comparing a doc count after startup - const originalDocs = await fetchDocuments(esClient, '.kibana_7.14.0_001'); - const migratedDocs = await fetchDocuments(esClient, targetIndex); - expect(assertMigratedDocuments(migratedDocs, originalDocs)); - }); - - it('fails with a descriptive message when a single document exceeds maxBatchSizeBytes', async () => { - root = createRoot({ maxBatchSizeBytes: 1015275 }); - esServer = await startES(); - await root.preboot(); - await root.setup(); - await expect(root.start()).rejects.toMatchInlineSnapshot( - `[Error: Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715319 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.]` - ); - - await retryAsync( - async () => { - const logFileContent = await fs.readFile(logFilePath, 'utf-8'); - const records = logFileContent - .split('\n') - .filter(Boolean) - .map((str) => JSON5.parse(str)) as LogRecord[]; - expect( - records.find((rec) => - rec.message.startsWith( - `Reason: Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715319 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.` - ) - ) - ).toBeDefined(); - }, - { retryAttempts: 10, retryDelayMs: 200 } - ); - }); -}); - -function createRoot(options: { maxBatchSizeBytes?: number }) { - return createRootWithCorePlugins( - { - migrations: { - skip: false, - batchSize: 1000, - maxBatchSizeBytes: options.maxBatchSizeBytes, - }, - logging: { - appenders: { - file: { - type: 'file', - fileName: logFilePath, - layout: { - type: 'json', - }, - }, - }, - loggers: [ - { - name: 'root', - level: 'info', - appenders: ['file'], - }, - ], - }, - }, - { - oss: false, - } - ); -} diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/migration_from_older_v1.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/migration_from_older_v1.test.ts deleted file mode 100644 index f9918966e9576..0000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/migration_from_older_v1.test.ts +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import Path from 'path'; -import Fs from 'fs'; -import Util from 'util'; -import Semver from 'semver'; -import { REPO_ROOT } from '@kbn/repo-info'; -import { Env } from '@kbn/config'; -import { getEnvOptions } from '@kbn/config-mocks'; -import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import type { SavedObjectsRawDoc } from '@kbn/core-saved-objects-server'; -import { modelVersionToVirtualVersion } from '@kbn/core-saved-objects-base-server-internal'; -import { - createTestServers, - createRootWithCorePlugins, - type TestElasticsearchUtils, -} from '@kbn/core-test-helpers-kbn-server'; -import { InternalCoreStart } from '@kbn/core-lifecycle-server-internal'; -import { Root } from '@kbn/core-root-server-internal'; - -const kibanaVersion = Env.createDefault(REPO_ROOT, getEnvOptions()).packageInfo.version; - -const logFilePath = Path.join(__dirname, 'migration_from_older_v1.log'); - -const asyncUnlink = Util.promisify(Fs.unlink); -async function removeLogFile() { - // ignore errors if it doesn't exist - await asyncUnlink(logFilePath).catch(() => void 0); -} -const assertMigratedDocuments = (arr: any[], target: any[]) => target.every((v) => arr.includes(v)); - -function sortByTypeAndId(a: { type: string; id: string }, b: { type: string; id: string }) { - return a.type.localeCompare(b.type) || a.id.localeCompare(b.id); -} - -async function fetchDocuments(esClient: ElasticsearchClient, index: string) { - const body = await esClient.search({ - index, - body: { - query: { - match_all: {}, - }, - _source: ['type', 'id'], - }, - }); - - return body.hits.hits - .map((h) => ({ - ...h._source, - id: h._id, - })) - .sort(sortByTypeAndId); -} - -// Failing 9.0 version update: https://github.com/elastic/kibana/issues/192624 -describe.skip('migrating from 7.3.0-xpack which used v1 migrations', () => { - const migratedIndex = `.kibana_${kibanaVersion}_001`; - const originalIndex = `.kibana_1`; // v1 migrations index - - let esServer: TestElasticsearchUtils; - let root: Root; - let coreStart: InternalCoreStart; - let esClient: ElasticsearchClient; - - const startServers = async ({ dataArchive, oss }: { dataArchive: string; oss: boolean }) => { - const { startES } = createTestServers({ - adjustTimeout: (t: number) => jest.setTimeout(t), - settings: { - es: { - license: 'basic', - dataArchive, - }, - }, - }); - - root = createRootWithCorePlugins( - { - migrations: { - skip: false, - // There are 40 docs in fixtures. Batch size configured to enforce 3 migration steps. - batchSize: 15, - }, - logging: { - appenders: { - file: { - type: 'file', - fileName: logFilePath, - layout: { - type: 'json', - }, - }, - }, - loggers: [ - { - name: 'root', - appenders: ['file'], - level: 'info', - }, - ], - }, - }, - { - oss, - } - ); - - const startEsPromise = startES().then((es) => (esServer = es)); - const startKibanaPromise = root - .preboot() - .then(() => root.setup()) - .then(() => root.start()) - .then((start) => { - coreStart = start; - esClient = coreStart.elasticsearch.client.asInternalUser; - }); - return await Promise.all([startEsPromise, startKibanaPromise]); - }; - - const getExpectedVersionPerType = () => - coreStart.savedObjects - .getTypeRegistry() - .getAllTypes() - .reduce((versionMap, type) => { - const { name, migrations, convertToMultiNamespaceTypeVersion, modelVersions } = type; - if (migrations || convertToMultiNamespaceTypeVersion) { - const migrationsMap = typeof migrations === 'function' ? migrations() : migrations; - const migrationsKeys = migrationsMap ? Object.keys(migrationsMap) : []; - if (convertToMultiNamespaceTypeVersion) { - // Setting this option registers a conversion migration that is reflected in the object's `typeMigrationVersions` field - migrationsKeys.push(convertToMultiNamespaceTypeVersion); - } - - const modelVersionCreateSchemas = - typeof modelVersions === 'function' ? modelVersions() : modelVersions ?? {}; - - Object.entries(modelVersionCreateSchemas).forEach(([key, modelVersion]) => { - migrationsKeys.push(modelVersionToVirtualVersion(key)); - }); - - const highestVersion = migrationsKeys.sort(Semver.compare).reverse()[0]; - return { - ...versionMap, - [name]: highestVersion, - }; - } else { - return { - ...versionMap, - [name]: undefined, - }; - } - }, {} as Record); - - const assertMigrationVersion = ( - doc: SavedObjectsRawDoc, - expectedVersions: Record - ) => { - const type = doc._source.type; - expect(doc._source.typeMigrationVersion).toEqual(expectedVersions[type]); - }; - - const stopServers = async () => { - if (root) { - await root.shutdown(); - } - if (esServer) { - await esServer.stop(); - } - }; - - beforeAll(async () => { - await removeLogFile(); - await startServers({ - oss: false, - dataArchive: Path.join(__dirname, '..', 'archives', '7.3.0_xpack_sample_saved_objects.zip'), - }); - }); - - afterAll(async () => { - await stopServers(); - }); - - it('creates the new index and the correct aliases', async () => { - const body = await esClient.indices.get( - { - index: migratedIndex, - }, - { ignore: [404] } - ); - - const response = body[migratedIndex]; - - expect(response).toBeDefined(); - expect(Object.keys(response.aliases!).sort()).toEqual(['.kibana', `.kibana_${kibanaVersion}`]); - }); - - it('copies all the document of the previous index to the new one', async () => { - const originalDocs = await fetchDocuments(esClient, originalIndex); - const migratedDocs = await fetchDocuments(esClient, migratedIndex); - expect(assertMigratedDocuments(migratedDocs, originalDocs)); - }); - - it('migrates the documents to the highest version', async () => { - const expectedVersions = getExpectedVersionPerType(); - const res = await esClient.search({ - index: migratedIndex, - body: { - sort: ['_doc'], - }, - size: 10000, - }); - const allDocuments = res.hits.hits as SavedObjectsRawDoc[]; - allDocuments.forEach((doc) => { - assertMigrationVersion(doc, expectedVersions); - }); - }); -}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/rewriting_id.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/rewriting_id.test.ts deleted file mode 100644 index 7e7c6c13465cd..0000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/rewriting_id.test.ts +++ /dev/null @@ -1,258 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import Path from 'path'; -import Fs from 'fs'; -import Util from 'util'; -import { kibanaPackageJson as pkg } from '@kbn/repo-info'; -import { - createRootWithCorePlugins, - createTestServers, - type TestElasticsearchUtils, -} from '@kbn/core-test-helpers-kbn-server'; -import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import { Root } from '@kbn/core-root-server-internal'; -import { deterministicallyRegenerateObjectId } from '@kbn/core-saved-objects-migration-server-internal'; - -const logFilePath = Path.join(__dirname, 'rewriting_id.log'); - -const asyncUnlink = Util.promisify(Fs.unlink); -async function removeLogFile() { - // ignore errors if it doesn't exist - await asyncUnlink(logFilePath).catch(() => void 0); -} - -function sortByTypeAndId(a: { type: string; id: string }, b: { type: string; id: string }) { - return a.type.localeCompare(b.type) || a.id.localeCompare(b.id); -} - -async function fetchDocs(esClient: ElasticsearchClient, index: string) { - const body = await esClient.search({ - index, - body: { - query: { - bool: { - should: [ - { - term: { type: 'foo' }, - }, - { - term: { type: 'bar' }, - }, - { - term: { type: 'legacy-url-alias' }, - }, - ], - }, - }, - }, - }); - - return body.hits.hits - .map((h) => ({ - ...h._source, - id: h._id, - })) - .sort(sortByTypeAndId); -} - -function createRoot() { - return createRootWithCorePlugins( - { - migrations: { - skip: false, - }, - logging: { - appenders: { - file: { - type: 'file', - fileName: logFilePath, - layout: { - type: 'json', - }, - }, - }, - loggers: [ - { - name: 'root', - level: 'info', - appenders: ['file'], - }, - ], - }, - }, - { - oss: true, - } - ); -} - -// Failing 9.0 version update: https://github.com/elastic/kibana/issues/192624 -describe.skip('migration v2', () => { - let esServer: TestElasticsearchUtils; - let root: Root; - - beforeAll(async () => { - await removeLogFile(); - }); - - afterAll(async () => { - if (root) { - await root.shutdown(); - } - if (esServer) { - await esServer.stop(); - } - }); - - it('rewrites id deterministically for SO with namespaceType: "multiple" and "multiple-isolated"', async () => { - const migratedIndexAlias = `.kibana_${pkg.version}`; - const { startES } = createTestServers({ - adjustTimeout: (t: number) => jest.setTimeout(t), - settings: { - es: { - license: 'basic', - // original SO: - // [ - // { id: 'foo:1', type: 'foo', foo: { name: 'Foo 1 default' } }, - // { id: 'spacex:foo:1', type: 'foo', foo: { name: 'Foo 1 spacex' }, namespace: 'spacex' }, - // { - // id: 'bar:1', - // type: 'bar', - // bar: { nomnom: 1 }, - // references: [{ type: 'foo', id: '1', name: 'Foo 1 default' }], - // }, - // { - // id: 'spacex:bar:1', - // type: 'bar', - // bar: { nomnom: 2 }, - // references: [{ type: 'foo', id: '1', name: 'Foo 1 spacex' }], - // namespace: 'spacex', - // }, - // ]; - dataArchive: Path.join( - __dirname, - '..', - 'archives', - '7.13.2_so_with_multiple_namespaces.zip' - ), - }, - }, - }); - - root = createRoot(); - - esServer = await startES(); - await root.preboot(); - const coreSetup = await root.setup(); - - coreSetup.savedObjects.registerType({ - name: 'foo', - hidden: false, - mappings: { properties: { name: { type: 'text' } } }, - namespaceType: 'multiple', - convertToMultiNamespaceTypeVersion: '8.0.0', - }); - - coreSetup.savedObjects.registerType({ - name: 'bar', - hidden: false, - mappings: { properties: { nomnom: { type: 'integer' } } }, - namespaceType: 'multiple-isolated', - convertToMultiNamespaceTypeVersion: '8.0.0', - }); - - const coreStart = await root.start(); - const esClient = coreStart.elasticsearch.client.asInternalUser; - - const migratedDocs = await fetchDocs(esClient, migratedIndexAlias); - - // each newly converted multi-namespace object in a non-default space has its ID deterministically regenerated, and a legacy-url-alias - // object is created which links the old ID to the new ID - const newFooId = deterministicallyRegenerateObjectId('spacex', 'foo', '1'); - const newBarId = deterministicallyRegenerateObjectId('spacex', 'bar', '1'); - - expect(migratedDocs).toEqual( - [ - { - id: 'foo:1', - type: 'foo', - foo: { name: 'Foo 1 default' }, - references: [], - namespaces: ['default'], - coreMigrationVersion: expect.any(String), - typeMigrationVersion: '8.0.0', - managed: false, - }, - { - id: `foo:${newFooId}`, - type: 'foo', - foo: { name: 'Foo 1 spacex' }, - references: [], - namespaces: ['spacex'], - originId: '1', - coreMigrationVersion: expect.any(String), - typeMigrationVersion: '8.0.0', - managed: false, - }, - { - // new object for spacex:foo:1 - id: 'legacy-url-alias:spacex:foo:1', - type: 'legacy-url-alias', - 'legacy-url-alias': { - sourceId: '1', - targetId: newFooId, - targetNamespace: 'spacex', - targetType: 'foo', - purpose: 'savedObjectConversion', - }, - references: [], - coreMigrationVersion: expect.any(String), - typeMigrationVersion: '8.2.0', - }, - { - id: 'bar:1', - type: 'bar', - bar: { nomnom: 1 }, - references: [{ type: 'foo', id: '1', name: 'Foo 1 default' }], - namespaces: ['default'], - coreMigrationVersion: expect.any(String), - typeMigrationVersion: '8.0.0', - managed: false, - }, - { - id: `bar:${newBarId}`, - type: 'bar', - bar: { nomnom: 2 }, - references: [{ type: 'foo', id: newFooId, name: 'Foo 1 spacex' }], - namespaces: ['spacex'], - originId: '1', - coreMigrationVersion: expect.any(String), - typeMigrationVersion: '8.0.0', - managed: false, - }, - { - // new object for spacex:bar:1 - id: 'legacy-url-alias:spacex:bar:1', - type: 'legacy-url-alias', - 'legacy-url-alias': { - sourceId: '1', - targetId: newBarId, - targetNamespace: 'spacex', - targetType: 'bar', - purpose: 'savedObjectConversion', - }, - references: [], - coreMigrationVersion: expect.any(String), - typeMigrationVersion: '8.2.0', - }, - ].sort(sortByTypeAndId) - ); - }); -});