Skip to content

Commit

Permalink
[UA] Use new _create_from ES API (elastic#207114)
Browse files Browse the repository at this point in the history
  • Loading branch information
@afharo
afharo authored Jan 21, 2025
1 parent 35794a0 commit fb8a17b
Show file tree
Hide file tree
Showing 5 changed files with 42 additions and 217 deletions.
117 changes: 2 additions & 115 deletions ...k/platform/plugins/private/upgrade_assistant/server/lib/reindexing/index_settings.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,124 +8,11 @@
import { versionService } from '../version';
import { getMockVersionInfo } from '../__fixtures__/version';

import {
generateNewIndexName,
getReindexWarnings,
sourceNameForIndex,
transformFlatSettings,
} from './index_settings';
import { generateNewIndexName, getReindexWarnings, sourceNameForIndex } from './index_settings';

const { currentMajor, prevMajor } = getMockVersionInfo();

describe('transformFlatSettings', () => {
it('does not blow up for empty mappings', () => {
expect(
transformFlatSettings({
settings: {},
mappings: {},
})
).toEqual({
settings: {},
mappings: {},
});
});

it('removes settings that cannot be set on a new index', () => {
expect(
transformFlatSettings({
settings: {
// Settings that should get preserved
'index.number_of_replicas': '1',
'index.number_of_shards': '5',

// Blacklisted settings
'index.allocation.existing_shards_allocator': 'gateway_allocator',
'index.blocks.write': 'true',
'index.creation_date': '1547052614626',
'index.frozen': 'true',
'index.history.uuid': 'i66b9149a-00ee-42d9-8ca1-85ae9279234gh',
'index.merge.enabled': 'true',
'index.provided_name': 'test1',
'index.resize.source.name': 'resizeName',
'index.resize.source.uuid': 'k34b9149a-00ee-42d9-8ca1-85ae9279234zs',
'index.routing.allocation.initial_recovery._id': '1',
'index.search.throttled': 'true',
'index.source_only': 'true',
'index.shrink.source.name': 'shrinkSourceName',
'index.shrink.source.uuid': 'q34b9149a-00ee-42d9-8ca1-85ae234324df',
'index.store.snapshot.repository_name': 'repoName',
'index.store.snapshot.snapshot_name': 'snapshotName',
'index.store.snapshot.snapshot_uuid': 'f345c9149a-00ee-42d9-8ca1-85ae234324df',
'index.store.snapshot.index_name': 'snapshotIndexName',
'index.store.snapshot.index_uuid': 'h764f9149a-00ee-42d9-8ca1-85ae234324af',
'index.uuid': 'i66b9149a-00ee-42d9-8ca1-85ae927924bf',
'index.verified_before_close': 'true',
'index.version.created': '123123',
'index.version.upgraded': '123123',
'index.mapper.dynamic': 'true',

// Deprecated settings
'index.force_memory_term_dictionary': '1024',
'index.max_adjacency_matrix_filters': 'true',
'index.soft_deletes.enabled': 'true',
},
mappings: {},
})
).toEqual({
settings: {
'index.number_of_replicas': '1',
'index.number_of_shards': '5',
},
mappings: {},
});
});

it('removes index.translog.retention.size if soft deletes is enabled', () => {
expect(
transformFlatSettings({
settings: {
// Settings that should get preserved
'index.number_of_replicas': '1',
'index.number_of_shards': '5',

// Deprecated settings
'index.soft_deletes.enabled': 'true',
'index.translog.retention.size': '5b',
},
mappings: {},
})
).toEqual({
settings: {
'index.number_of_replicas': '1',
'index.number_of_shards': '5',
},
mappings: {},
});
});

it('removes index.translog.retention.age if soft deletes is enabled', () => {
expect(
transformFlatSettings({
settings: {
// Settings that should get preserved
'index.number_of_replicas': '1',
'index.number_of_shards': '5',

// Deprecated settings
'index.soft_deletes.enabled': 'true',
'index.translog.retention.age': '5d',
},
mappings: {},
})
).toEqual({
settings: {
'index.number_of_replicas': '1',
'index.number_of_shards': '5',
},
mappings: {},
});
});

describe('index settings', () => {
describe('sourceNameForIndex', () => {
beforeEach(() => {
versionService.setup('8.0.0');
Expand Down
72 changes: 0 additions & 72 deletions x-pack/platform/plugins/private/upgrade_assistant/server/lib/reindexing/index_settings.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
* 2.0.
*/

import { flow, omit } from 'lodash';
import { ReindexWarning } from '../../../common/types';
import { versionService } from '../version';
import { FlatSettings } from './types';
Expand All @@ -27,17 +26,6 @@ const deprecatedSettings = [
'index.soft_deletes.enabled',
];

/**
* Validates, and updates deprecated settings and mappings to be applied to the
* new updated index.
*/
export const transformFlatSettings = (flatSettings: FlatSettings) => {
const settings = transformSettings(flatSettings.settings);
const mappings = transformMappings(flatSettings.mappings);

return { settings, mappings };
};

/**
* Provides the assumed source of the index name stripping any prefixing
* introduced by the upgrade assistant
Expand Down Expand Up @@ -125,63 +113,3 @@ export const getReindexWarnings = (flatSettings: FlatSettings): ReindexWarning[]

return warnings;
};

const removeUnsettableSettings = (settings: FlatSettings['settings']) =>
omit(settings, [
// Private ES settings
'index.allocation.existing_shards_allocator',
'index.blocks.write',
'index.creation_date',
'index.frozen',
'index.history.uuid',
'index.merge.enabled',
'index.provided_name',
'index.resize.source.name',
'index.resize.source.uuid',
'index.routing.allocation.initial_recovery._id',
'index.search.throttled',
'index.source_only',
'index.shrink.source.name',
'index.shrink.source.uuid',
'index.store.snapshot.repository_name',
'index.store.snapshot.snapshot_name',
'index.store.snapshot.snapshot_uuid',
'index.store.snapshot.index_name',
'index.store.snapshot.index_uuid',
'index.uuid',
'index.verified_before_close',
'index.version.created',

// Ignored since 6.x and forbidden in 7.x
'index.mapper.dynamic',

// Deprecated in 9.0
'index.version.upgraded',
]);

const removeDeprecatedSettings = (settings: FlatSettings['settings']) => {
const updatedSettings = { ...settings };

// Translog settings are only marked as deprecated if soft deletes is enabled
if (updatedSettings['index.soft_deletes.enabled'] === 'true') {
if (updatedSettings['index.translog.retention.size']) {
delete updatedSettings['index.translog.retention.size'];
}

// @ts-expect-error @elastic/elasticsearch doesn't declare such a setting
if (settings['index.translog.retention.age']) {
delete updatedSettings['index.translog.retention.age'];
}
}

return omit(updatedSettings, deprecatedSettings);
};

// Use `flow` to pipe the settings through each function.
const transformSettings = flow(removeUnsettableSettings, removeDeprecatedSettings);

const updateFixableMappings = (mappings: FlatSettings['mappings']) => {
return mappings;
};

const transformMappings = flow(updateFixableMappings);
24 changes: 13 additions & 11 deletions .../platform/plugins/private/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -457,19 +457,21 @@ describe('reindexService', () => {
// The more intricate details of how the settings are chosen are test separately.
it('creates new index with settings and mappings and updates lastCompletedStep', async () => {
actions.getFlatSettings.mockResolvedValueOnce(settingsMappings);
clusterClient.asCurrentUser.indices.create.mockResponse(
// @ts-expect-error not full interface
{ acknowledged: true }
);
clusterClient.asCurrentUser.transport.request.mockResolvedValueOnce({ acknowledged: true });
const updatedOp = await service.processNextStep(reindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.newIndexCreated);
expect(clusterClient.asCurrentUser.indices.create).toHaveBeenCalledWith({
index: 'myIndex-reindex-0',
// index.blocks.write should be removed from the settings for the new index.
// index.number_of_replicas and index.refresh_interval are stored to be set at a later stage.
// Setting to 0 and -1, respectively, right now.
settings: { 'index.number_of_replicas': 0, 'index.refresh_interval': -1 },
mappings: settingsMappings.mappings,
expect(clusterClient.asCurrentUser.transport.request).toHaveBeenCalledWith({
method: 'POST',
path: `_create_from/myIndex/myIndex-reindex-0`,
body: {
settings_override: {
'index.blocks.read_only': null,
'index.blocks.read_only_allow_delete': null,
'index.blocks.write': null,
'index.number_of_replicas': 0,
'index.refresh_interval': -1,
},
},
});
});

Expand Down
40 changes: 25 additions & 15 deletions x-pack/platform/plugins/private/upgrade_assistant/server/lib/reindexing/reindex_service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,7 @@ import {

import { esIndicesStateCheck } from '../es_indices_state_check';

import {
generateNewIndexName,
getReindexWarnings,
sourceNameForIndex,
transformFlatSettings,
} from './index_settings';
import { generateNewIndexName, getReindexWarnings, sourceNameForIndex } from './index_settings';

import { ReindexActions } from './reindex_actions';

Expand Down Expand Up @@ -190,7 +185,7 @@ export const reindexServiceFactory = (
throw error.indexNotFound(`Index ${indexName} does not exist.`);
}

const { settings, mappings } = transformFlatSettings(flatSettings);
const { settings = {} } = flatSettings;

// Backup the current settings to restore them after the reindex
// https://github.com/elastic/kibana/issues/201605
Expand All @@ -201,16 +196,31 @@ export const reindexServiceFactory = (

let createIndex;
try {
createIndex = await esClient.indices.create({
index: newIndexName,
settings: {
...settings,
// Reindexing optimizations
'index.number_of_replicas': 0,
'index.refresh_interval': -1,
createIndex = await esClient.transport.request<{ acknowledged: boolean }>({
method: 'POST',
path: `_create_from/${indexName}/${newIndexName}`,
body: {
// Settings overrides copied from ES datastream reindex logic
// https://github.com/elastic/elasticsearch/blob/9c0709f386fee4154e930cb61a02868adebe8572/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java#L195-L210
settings_override: {
// Remove read-only settings if they exist
'index.blocks.read_only': null,
'index.blocks.read_only_allow_delete': null,
'index.blocks.write': null,
// Reindexing optimizations
'index.number_of_replicas': 0,
'index.refresh_interval': -1,
},
},
mappings,
});
/**
* Response is expected to be:
* {
* "acknowledged": true,
* "shards_acknowledged": true,
* "index": "test-copy"
* }
*/
} catch (err) {
// If for any reason the new index name generated by the `generateNewIndexName` already
// exists (this could happen if kibana is restarted during reindexing), we can just go
Expand Down
6 changes: 2 additions & 4 deletions x-pack/test/api_integration/apis/upgrade_assistant/upgrade_assistant.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,8 @@ export default function ({ getService }: FtrProviderContext) {
try {
await es.indices.create({
index: indexName,
body: {
settings: {
index: indexSettings,
},
settings: {
index: indexSettings,
},
});
} catch (err) {
Expand Down

0 comments on commit fb8a17b

Please sign in to comment.