diff --git a/x-pack/platform/plugins/private/upgrade_assistant/server/lib/reindexing/index_settings.test.ts b/x-pack/platform/plugins/private/upgrade_assistant/server/lib/reindexing/index_settings.test.ts index 685bc443c324c..17eadc16c6229 100644 --- a/x-pack/platform/plugins/private/upgrade_assistant/server/lib/reindexing/index_settings.test.ts +++ b/x-pack/platform/plugins/private/upgrade_assistant/server/lib/reindexing/index_settings.test.ts @@ -8,124 +8,11 @@ import { versionService } from '../version'; import { getMockVersionInfo } from '../__fixtures__/version'; -import { - generateNewIndexName, - getReindexWarnings, - sourceNameForIndex, - transformFlatSettings, -} from './index_settings'; +import { generateNewIndexName, getReindexWarnings, sourceNameForIndex } from './index_settings'; const { currentMajor, prevMajor } = getMockVersionInfo(); -describe('transformFlatSettings', () => { - it('does not blow up for empty mappings', () => { - expect( - transformFlatSettings({ - settings: {}, - mappings: {}, - }) - ).toEqual({ - settings: {}, - mappings: {}, - }); - }); - - it('removes settings that cannot be set on a new index', () => { - expect( - transformFlatSettings({ - settings: { - // Settings that should get preserved - 'index.number_of_replicas': '1', - 'index.number_of_shards': '5', - - // Blacklisted settings - 'index.allocation.existing_shards_allocator': 'gateway_allocator', - 'index.blocks.write': 'true', - 'index.creation_date': '1547052614626', - 'index.frozen': 'true', - 'index.history.uuid': 'i66b9149a-00ee-42d9-8ca1-85ae9279234gh', - 'index.merge.enabled': 'true', - 'index.provided_name': 'test1', - 'index.resize.source.name': 'resizeName', - 'index.resize.source.uuid': 'k34b9149a-00ee-42d9-8ca1-85ae9279234zs', - 'index.routing.allocation.initial_recovery._id': '1', - 'index.search.throttled': 'true', - 'index.source_only': 'true', - 'index.shrink.source.name': 'shrinkSourceName', - 'index.shrink.source.uuid': 'q34b9149a-00ee-42d9-8ca1-85ae234324df', - 'index.store.snapshot.repository_name': 'repoName', - 'index.store.snapshot.snapshot_name': 'snapshotName', - 'index.store.snapshot.snapshot_uuid': 'f345c9149a-00ee-42d9-8ca1-85ae234324df', - 'index.store.snapshot.index_name': 'snapshotIndexName', - 'index.store.snapshot.index_uuid': 'h764f9149a-00ee-42d9-8ca1-85ae234324af', - 'index.uuid': 'i66b9149a-00ee-42d9-8ca1-85ae927924bf', - 'index.verified_before_close': 'true', - 'index.version.created': '123123', - 'index.version.upgraded': '123123', - 'index.mapper.dynamic': 'true', - - // Deprecated settings - 'index.force_memory_term_dictionary': '1024', - 'index.max_adjacency_matrix_filters': 'true', - 'index.soft_deletes.enabled': 'true', - }, - mappings: {}, - }) - ).toEqual({ - settings: { - 'index.number_of_replicas': '1', - 'index.number_of_shards': '5', - }, - mappings: {}, - }); - }); - - it('removes index.translog.retention.size if soft deletes is enabled', () => { - expect( - transformFlatSettings({ - settings: { - // Settings that should get preserved - 'index.number_of_replicas': '1', - 'index.number_of_shards': '5', - - // Deprecated settings - 'index.soft_deletes.enabled': 'true', - 'index.translog.retention.size': '5b', - }, - mappings: {}, - }) - ).toEqual({ - settings: { - 'index.number_of_replicas': '1', - 'index.number_of_shards': '5', - }, - mappings: {}, - }); - }); - - it('removes index.translog.retention.age if soft deletes is enabled', () => { - expect( - transformFlatSettings({ - settings: { - // Settings that should get preserved - 'index.number_of_replicas': '1', - 'index.number_of_shards': '5', - - // Deprecated settings - 'index.soft_deletes.enabled': 'true', - 'index.translog.retention.age': '5d', - }, - mappings: {}, - }) - ).toEqual({ - settings: { - 'index.number_of_replicas': '1', - 'index.number_of_shards': '5', - }, - mappings: {}, - }); - }); - +describe('index settings', () => { describe('sourceNameForIndex', () => { beforeEach(() => { versionService.setup('8.0.0'); diff --git a/x-pack/platform/plugins/private/upgrade_assistant/server/lib/reindexing/index_settings.ts b/x-pack/platform/plugins/private/upgrade_assistant/server/lib/reindexing/index_settings.ts index 3344ecde4ab69..178df91be6096 100644 --- a/x-pack/platform/plugins/private/upgrade_assistant/server/lib/reindexing/index_settings.ts +++ b/x-pack/platform/plugins/private/upgrade_assistant/server/lib/reindexing/index_settings.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { flow, omit } from 'lodash'; import { ReindexWarning } from '../../../common/types'; import { versionService } from '../version'; import { FlatSettings } from './types'; @@ -27,17 +26,6 @@ const deprecatedSettings = [ 'index.soft_deletes.enabled', ]; -/** - * Validates, and updates deprecated settings and mappings to be applied to the - * new updated index. - */ -export const transformFlatSettings = (flatSettings: FlatSettings) => { - const settings = transformSettings(flatSettings.settings); - const mappings = transformMappings(flatSettings.mappings); - - return { settings, mappings }; -}; - /** * Provides the assumed source of the index name stripping any prefixing * introduced by the upgrade assistant @@ -125,63 +113,3 @@ export const getReindexWarnings = (flatSettings: FlatSettings): ReindexWarning[] return warnings; }; - -const removeUnsettableSettings = (settings: FlatSettings['settings']) => - omit(settings, [ - // Private ES settings - 'index.allocation.existing_shards_allocator', - 'index.blocks.write', - 'index.creation_date', - 'index.frozen', - 'index.history.uuid', - 'index.merge.enabled', - 'index.provided_name', - 'index.resize.source.name', - 'index.resize.source.uuid', - 'index.routing.allocation.initial_recovery._id', - 'index.search.throttled', - 'index.source_only', - 'index.shrink.source.name', - 'index.shrink.source.uuid', - 'index.store.snapshot.repository_name', - 'index.store.snapshot.snapshot_name', - 'index.store.snapshot.snapshot_uuid', - 'index.store.snapshot.index_name', - 'index.store.snapshot.index_uuid', - 'index.uuid', - 'index.verified_before_close', - 'index.version.created', - - // Ignored since 6.x and forbidden in 7.x - 'index.mapper.dynamic', - - // Deprecated in 9.0 - 'index.version.upgraded', - ]); - -const removeDeprecatedSettings = (settings: FlatSettings['settings']) => { - const updatedSettings = { ...settings }; - - // Translog settings are only marked as deprecated if soft deletes is enabled - if (updatedSettings['index.soft_deletes.enabled'] === 'true') { - if (updatedSettings['index.translog.retention.size']) { - delete updatedSettings['index.translog.retention.size']; - } - - // @ts-expect-error @elastic/elasticsearch doesn't declare such a setting - if (settings['index.translog.retention.age']) { - delete updatedSettings['index.translog.retention.age']; - } - } - - return omit(updatedSettings, deprecatedSettings); -}; - -// Use `flow` to pipe the settings through each function. -const transformSettings = flow(removeUnsettableSettings, removeDeprecatedSettings); - -const updateFixableMappings = (mappings: FlatSettings['mappings']) => { - return mappings; -}; - -const transformMappings = flow(updateFixableMappings); diff --git a/x-pack/platform/plugins/private/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts b/x-pack/platform/plugins/private/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts index e5c2b6beb4a6f..cef50f37b2994 100644 --- a/x-pack/platform/plugins/private/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts +++ b/x-pack/platform/plugins/private/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts @@ -457,19 +457,21 @@ describe('reindexService', () => { // The more intricate details of how the settings are chosen are test separately. it('creates new index with settings and mappings and updates lastCompletedStep', async () => { actions.getFlatSettings.mockResolvedValueOnce(settingsMappings); - clusterClient.asCurrentUser.indices.create.mockResponse( - // @ts-expect-error not full interface - { acknowledged: true } - ); + clusterClient.asCurrentUser.transport.request.mockResolvedValueOnce({ acknowledged: true }); const updatedOp = await service.processNextStep(reindexOp); expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.newIndexCreated); - expect(clusterClient.asCurrentUser.indices.create).toHaveBeenCalledWith({ - index: 'myIndex-reindex-0', - // index.blocks.write should be removed from the settings for the new index. - // index.number_of_replicas and index.refresh_interval are stored to be set at a later stage. - // Setting to 0 and -1, respectively, right now. - settings: { 'index.number_of_replicas': 0, 'index.refresh_interval': -1 }, - mappings: settingsMappings.mappings, + expect(clusterClient.asCurrentUser.transport.request).toHaveBeenCalledWith({ + method: 'POST', + path: `_create_from/myIndex/myIndex-reindex-0`, + body: { + settings_override: { + 'index.blocks.read_only': null, + 'index.blocks.read_only_allow_delete': null, + 'index.blocks.write': null, + 'index.number_of_replicas': 0, + 'index.refresh_interval': -1, + }, + }, }); }); diff --git a/x-pack/platform/plugins/private/upgrade_assistant/server/lib/reindexing/reindex_service.ts b/x-pack/platform/plugins/private/upgrade_assistant/server/lib/reindexing/reindex_service.ts index f26a9b72252a5..a6f85e45e3676 100644 --- a/x-pack/platform/plugins/private/upgrade_assistant/server/lib/reindexing/reindex_service.ts +++ b/x-pack/platform/plugins/private/upgrade_assistant/server/lib/reindexing/reindex_service.ts @@ -19,12 +19,7 @@ import { import { esIndicesStateCheck } from '../es_indices_state_check'; -import { - generateNewIndexName, - getReindexWarnings, - sourceNameForIndex, - transformFlatSettings, -} from './index_settings'; +import { generateNewIndexName, getReindexWarnings, sourceNameForIndex } from './index_settings'; import { ReindexActions } from './reindex_actions'; @@ -190,7 +185,7 @@ export const reindexServiceFactory = ( throw error.indexNotFound(`Index ${indexName} does not exist.`); } - const { settings, mappings } = transformFlatSettings(flatSettings); + const { settings = {} } = flatSettings; // Backup the current settings to restore them after the reindex // https://github.com/elastic/kibana/issues/201605 @@ -201,16 +196,31 @@ export const reindexServiceFactory = ( let createIndex; try { - createIndex = await esClient.indices.create({ - index: newIndexName, - settings: { - ...settings, - // Reindexing optimizations - 'index.number_of_replicas': 0, - 'index.refresh_interval': -1, + createIndex = await esClient.transport.request<{ acknowledged: boolean }>({ + method: 'POST', + path: `_create_from/${indexName}/${newIndexName}`, + body: { + // Settings overrides copied from ES datastream reindex logic + // https://github.com/elastic/elasticsearch/blob/9c0709f386fee4154e930cb61a02868adebe8572/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java#L195-L210 + settings_override: { + // Remove read-only settings if they exist + 'index.blocks.read_only': null, + 'index.blocks.read_only_allow_delete': null, + 'index.blocks.write': null, + // Reindexing optimizations + 'index.number_of_replicas': 0, + 'index.refresh_interval': -1, + }, }, - mappings, }); + /** + * Response is expected to be: + * { + * "acknowledged": true, + * "shards_acknowledged": true, + * "index": "test-copy" + * } + */ } catch (err) { // If for any reason the new index name generated by the `generateNewIndexName` already // exists (this could happen if kibana is restarted during reindexing), we can just go diff --git a/x-pack/test/api_integration/apis/upgrade_assistant/upgrade_assistant.ts b/x-pack/test/api_integration/apis/upgrade_assistant/upgrade_assistant.ts index f03cb6fb80e37..d7a96333f49e8 100644 --- a/x-pack/test/api_integration/apis/upgrade_assistant/upgrade_assistant.ts +++ b/x-pack/test/api_integration/apis/upgrade_assistant/upgrade_assistant.ts @@ -48,10 +48,8 @@ export default function ({ getService }: FtrProviderContext) { try { await es.indices.create({ index: indexName, - body: { - settings: { - index: indexSettings, - }, + settings: { + index: indexSettings, }, }); } catch (err) {