diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/__snapshots__/migrations_state_action_machine.test.ts.snap b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/__snapshots__/migrations_state_action_machine.test.ts.snap index 00f71c4227631..ae825fd8fc51d 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/__snapshots__/migrations_state_action_machine.test.ts.snap +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/__snapshots__/migrations_state_action_machine.test.ts.snap @@ -208,6 +208,7 @@ Object { "properties": Object {}, }, "tempIndex": ".my-so-index_7.11.0_reindex_temp", + "tempIndexAlias": ".my-so-index_7.11.0_reindex_temp_alias", "tempIndexMappings": Object { "dynamic": false, "properties": Object { @@ -436,6 +437,7 @@ Object { "properties": Object {}, }, "tempIndex": ".my-so-index_7.11.0_reindex_temp", + "tempIndexAlias": ".my-so-index_7.11.0_reindex_temp_alias", "tempIndexMappings": Object { "dynamic": false, "properties": Object { @@ -668,6 +670,7 @@ Object { "properties": Object {}, }, "tempIndex": ".my-so-index_7.11.0_reindex_temp", + "tempIndexAlias": ".my-so-index_7.11.0_reindex_temp_alias", "tempIndexMappings": Object { "dynamic": false, "properties": Object { @@ -904,6 +907,7 @@ Object { "properties": Object {}, }, "tempIndex": ".my-so-index_7.11.0_reindex_temp", + "tempIndexAlias": ".my-so-index_7.11.0_reindex_temp_alias", "tempIndexMappings": Object { "dynamic": false, "properties": Object { @@ -1177,6 +1181,7 @@ Object { "properties": Object {}, }, "tempIndex": ".my-so-index_7.11.0_reindex_temp", + "tempIndexAlias": ".my-so-index_7.11.0_reindex_temp_alias", "tempIndexMappings": Object { "dynamic": false, "properties": Object { @@ -1416,6 +1421,7 @@ Object { "properties": Object {}, }, "tempIndex": ".my-so-index_7.11.0_reindex_temp", + "tempIndexAlias": ".my-so-index_7.11.0_reindex_temp_alias", "tempIndexMappings": Object { "dynamic": false, "properties": Object { diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/bulk_overwrite_transformed_documents.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/bulk_overwrite_transformed_documents.ts index 716683c1938fb..76c250a16c17b 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/bulk_overwrite_transformed_documents.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/bulk_overwrite_transformed_documents.ts @@ -26,6 +26,12 @@ export interface BulkOverwriteTransformedDocumentsParams { index: string; operations: BulkOperation[]; refresh?: estypes.Refresh; + /** + * If true, we prevent Elasticsearch from auto-creating the index if it + * doesn't exist. We use the ES paramater require_alias: true so `index` + * must be an alias, otherwise the bulk index will fail. + */ + useAliasToPreventAutoCreate?: boolean; } /** @@ -38,6 +44,7 @@ export const bulkOverwriteTransformedDocuments = index, operations, refresh = false, + useAliasToPreventAutoCreate = false, }: BulkOverwriteTransformedDocumentsParams): TaskEither.TaskEither< | RetryableEsClientError | TargetIndexHadWriteBlock @@ -56,7 +63,7 @@ export const bulkOverwriteTransformedDocuments = // probably unlikely so for now we'll accept this risk and wait till // system indices puts in place a hard control. index, - require_alias: false, + require_alias: useAliasToPreventAutoCreate, wait_for_active_shards: WAIT_FOR_ALL_SHARDS_TO_BE_ACTIVE, refresh, filter_path: ['items.*.error'], diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.test.ts index 1378dec852e2b..00c29f2698469 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.test.ts @@ -265,6 +265,7 @@ describe('createInitialState', () => { }, }, "tempIndex": ".kibana_task_manager_8.1.0_reindex_temp", + "tempIndexAlias": ".kibana_task_manager_8.1.0_reindex_temp_alias", "tempIndexMappings": Object { "dynamic": false, "properties": Object { diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.ts index 6bb6c22e3a7f2..f4a01b43b19e3 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.ts @@ -142,6 +142,7 @@ export const createInitialState = ({ versionAlias: `${indexPrefix}_${kibanaVersion}`, versionIndex: `${indexPrefix}_${kibanaVersion}_001`, tempIndex: getTempIndexName(indexPrefix, kibanaVersion), + tempIndexAlias: getTempIndexName(indexPrefix, kibanaVersion) + '_alias', kibanaVersion, preMigrationScript: Option.fromNullable(preMigrationScript), targetIndexMappings, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.test.ts index 951ee86d92f0d..ca274bc7d9b36 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.test.ts @@ -92,7 +92,7 @@ describe('createBatches', () => { expect( createBatches({ documents, - maxBatchSizeBytes: (DOCUMENT_SIZE_BYTES + 43) * 2, // add extra length for 'index' property + maxBatchSizeBytes: (DOCUMENT_SIZE_BYTES + 49) * 2, // add extra length for 'index' property typeIndexMap: buildTempIndexMap( { '.kibana': ['dashboard'], @@ -108,7 +108,7 @@ describe('createBatches', () => { { index: { _id: '', - _index: '.kibana_8.8.0_reindex_temp', + _index: '.kibana_8.8.0_reindex_temp_alias', }, }, { type: 'dashboard', title: 'my saved object title ¹' }, @@ -117,7 +117,7 @@ describe('createBatches', () => { { index: { _id: '', - _index: '.kibana_8.8.0_reindex_temp', + _index: '.kibana_8.8.0_reindex_temp_alias', }, }, { type: 'dashboard', title: 'my saved object title ²' }, @@ -128,7 +128,7 @@ describe('createBatches', () => { { index: { _id: '', - _index: '.kibana_cases_8.8.0_reindex_temp', + _index: '.kibana_cases_8.8.0_reindex_temp_alias', }, }, { type: 'cases', title: 'a case' }, @@ -137,7 +137,7 @@ describe('createBatches', () => { { index: { _id: '', - _index: '.kibana_cases_8.8.0_reindex_temp', + _index: '.kibana_cases_8.8.0_reindex_temp_alias', }, }, { type: 'cases-comments', title: 'a case comment #1' }, @@ -148,7 +148,7 @@ describe('createBatches', () => { { index: { _id: '', - _index: '.kibana_cases_8.8.0_reindex_temp', + _index: '.kibana_cases_8.8.0_reindex_temp_alias', }, }, { type: 'cases-user-actions', title: 'a case user action' }, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.ts index dc5b09d6c3379..0fcbbc00ce7ea 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.ts @@ -56,7 +56,7 @@ export function buildTempIndexMap( ): Record { return Object.entries(indexTypesMap || {}).reduce>( (acc, [indexAlias, types]) => { - const tempIndex = getTempIndexName(indexAlias, kibanaVersion!); + const tempIndex = getTempIndexName(indexAlias, kibanaVersion!) + '_alias'; types.forEach((type) => { acc[type] = tempIndex; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts index 9763eda1c5b15..a30c90e2da5ee 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts @@ -252,7 +252,9 @@ export const createBulkIndexOperationTuple = ( { index: { _id: doc._id, - ...(typeIndexMap[doc._source.type] && { _index: typeIndexMap[doc._source.type] }), + ...(typeIndexMap[doc._source.type] && { + _index: typeIndexMap[doc._source.type], + }), // use optimistic concurrency control to ensure that outdated // documents are only overwritten once with the latest version ...(typeof doc._seq_no !== 'undefined' && { if_seq_no: doc._seq_no }), diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts index 6390edd9b04cb..f96695af4a07f 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts @@ -102,6 +102,7 @@ describe('migrations v2 model', () => { versionAlias: '.kibana_7.11.0', versionIndex: '.kibana_7.11.0_001', tempIndex: '.kibana_7.11.0_reindex_temp', + tempIndexAlias: '.kibana_7.11.0_reindex_temp_alias', excludeOnUpgradeQuery: { bool: { must_not: [ diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts index 1b5a9fe99fe3a..9c028f150a6d0 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts @@ -139,6 +139,7 @@ export const nextActionMap = ( Actions.createIndex({ client, indexName: state.tempIndex, + aliases: [state.tempIndexAlias], mappings: state.tempIndexMappings, }), READY_TO_REINDEX_SYNC: () => Actions.synchronizeMigrators(readyToReindex), @@ -163,7 +164,13 @@ export const nextActionMap = ( REINDEX_SOURCE_TO_TEMP_INDEX_BULK: (state: ReindexSourceToTempIndexBulk) => Actions.bulkOverwriteTransformedDocuments({ client, - index: state.tempIndex, + /* + * Since other nodes can delete the temp index while we're busy writing + * to it, we use the alias to prevent the auto-creation of the index if + * it doesn't exist. + */ + index: state.tempIndexAlias, + useAliasToPreventAutoCreate: true, operations: state.bulkOperationBatches[state.currentBatch], /** * Since we don't run a search against the target index, we disable "refresh" to speed up diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/state.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/state.ts index 8a6be0269947e..39b7f71ea7415 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/state.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/state.ts @@ -132,10 +132,16 @@ export interface BaseState extends ControlState { */ readonly versionIndex: string; /** - * An alias on the target index used as part of an "reindex block" that + * A temporary index used as part of an "reindex block" that * prevents lost deletes e.g. `.kibana_7.11.0_reindex`. */ readonly tempIndex: string; + /** + * An alias to the tempIndex used to prevent ES from auto-creating the temp + * index if one node deletes it while another writes to it + * e.g. `.kibana_7.11.0_reindex_temp_alias`. + */ + readonly tempIndexAlias: string; /** * When upgrading to a more recent kibana version, some saved object types * might be conflicting or no longer used. diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/batch_size_bytes.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/batch_size_bytes.test.ts index 4a64b26e1125a..340c733f8b00d 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/batch_size_bytes.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group2/batch_size_bytes.test.ts @@ -118,7 +118,7 @@ describe('migration v2', () => { await root.preboot(); await root.setup(); await expect(root.start()).rejects.toMatchInlineSnapshot( - `[Error: Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715312 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.]` + `[Error: Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715318 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.]` ); await retryAsync( @@ -131,7 +131,7 @@ describe('migration v2', () => { expect( records.find((rec) => rec.message.startsWith( - `Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715312 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.` + `Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715318 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.` ) ) ).toBeDefined(); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/actions/actions.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/actions/actions.test.ts index 3b5595377524e..fc53fa54aab57 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/actions/actions.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/actions/actions.test.ts @@ -68,6 +68,7 @@ describe('migration actions', () => { await createIndex({ client, indexName: 'existing_index_with_docs', + aliases: ['existing_index_with_docs_alias'], mappings: { dynamic: true, properties: { @@ -151,7 +152,9 @@ describe('migration actions', () => { expect(res.right).toEqual( expect.objectContaining({ existing_index_with_docs: { - aliases: {}, + aliases: { + existing_index_with_docs_alias: {}, + }, mappings: expect.anything(), settings: expect.anything(), }, @@ -168,7 +171,9 @@ describe('migration actions', () => { expect(res.right).toEqual( expect.objectContaining({ existing_index_with_docs: { - aliases: {}, + aliases: { + existing_index_with_docs_alias: {}, + }, mappings: { // FIXME https://github.com/elastic/elasticsearch-js/issues/1796 dynamic: 'true', @@ -1947,6 +1952,30 @@ describe('migration actions', () => { } `); }); + it('resolves left index_not_found_exception if the index does not exist and useAliasToPreventAutoCreate=true', async () => { + const newDocs = [ + { _source: { title: 'doc 5' } }, + { _source: { title: 'doc 6' } }, + { _source: { title: 'doc 7' } }, + ] as unknown as SavedObjectsRawDoc[]; + await expect( + bulkOverwriteTransformedDocuments({ + client, + index: 'existing_index_with_docs_alias_that_does_not_exist', + useAliasToPreventAutoCreate: true, + operations: newDocs.map((doc) => createBulkIndexOperationTuple(doc)), + refresh: 'wait_for', + })() + ).resolves.toMatchInlineSnapshot(` + Object { + "_tag": "Left", + "left": Object { + "index": "existing_index_with_docs_alias_that_does_not_exist", + "type": "index_not_found_exception", + }, + } + `); + }); it('resolves left target_index_had_write_block if there are write_block errors', async () => { const newDocs = [ { _source: { title: 'doc 5' } },