diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/check_target_mappings.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/check_target_mappings.test.ts index 94e5ca0234535..50a20dcf2fb81 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/check_target_mappings.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/check_target_mappings.test.ts @@ -175,8 +175,9 @@ describe('checkTargetTypesMappings', () => { const result = await task(); expect(result).toEqual( - Either.right({ - type: 'types_match' as const, + Either.left({ + type: 'types_added' as const, + newTypes: ['type3'], }) ); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/check_target_mappings.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/check_target_mappings.ts index 029989d89935b..e115c65d8d6bd 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/check_target_mappings.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/check_target_mappings.ts @@ -9,7 +9,7 @@ import * as Either from 'fp-ts/lib/Either'; import * as TaskEither from 'fp-ts/lib/TaskEither'; import type { IndexMapping, VirtualVersionMap } from '@kbn/core-saved-objects-base-server-internal'; -import { getUpdatedTypes } from '../core/compare_mappings'; +import { getNewAndUpdatedTypes } from '../core/compare_mappings'; /** @internal */ export interface CheckTargetTypesMappingsParams { @@ -36,6 +36,12 @@ export interface TypesChanged { updatedTypes: string[]; } +/** @internal */ +export interface TypesAdded { + type: 'types_added'; + newTypes: string[]; +} + export const checkTargetTypesMappings = ({ indexTypes, @@ -44,7 +50,7 @@ export const checkTargetTypesMappings = latestMappingsVersions, hashToVersionMap = {}, }: CheckTargetTypesMappingsParams): TaskEither.TaskEither< - IndexMappingsIncomplete | TypesChanged, + IndexMappingsIncomplete | TypesChanged | TypesAdded, TypesMatch > => async () => { @@ -56,7 +62,7 @@ export const checkTargetTypesMappings = return Either.left({ type: 'index_mappings_incomplete' as const }); } - const updatedTypes = getUpdatedTypes({ + const { newTypes, updatedTypes } = getNewAndUpdatedTypes({ indexTypes, indexMeta: indexMappings?._meta, latestMappingsVersions, @@ -68,6 +74,11 @@ export const checkTargetTypesMappings = type: 'types_changed' as const, updatedTypes, }); + } else if (newTypes.length) { + return Either.left({ + type: 'types_added' as const, + newTypes, + }); } else { return Either.right({ type: 'types_match' as const }); } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts index 35d05e1374667..03d59e6f2fbbd 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts @@ -111,7 +111,7 @@ import type { UnknownDocsFound } from './check_for_unknown_docs'; import type { IncompatibleClusterRoutingAllocation } from './check_cluster_routing_allocation'; import type { ClusterShardLimitExceeded } from './create_index'; import type { SynchronizationFailed } from './synchronize_migrators'; -import type { IndexMappingsIncomplete, TypesChanged } from './check_target_mappings'; +import type { IndexMappingsIncomplete, TypesAdded, TypesChanged } from './check_target_mappings'; export type { CheckForUnknownDocsParams, @@ -192,6 +192,7 @@ export interface ActionErrorTypeMap { synchronization_failed: SynchronizationFailed; index_mappings_incomplete: IndexMappingsIncomplete; types_changed: TypesChanged; + types_added: TypesAdded; operation_not_supported: OperationNotSupported; source_equals_target: SourceEqualsTarget; } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/update_source_mappings_properties.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/update_source_mappings_properties.test.ts index 43fbd60a192a5..28bec95691982 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/update_source_mappings_properties.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/update_source_mappings_properties.test.ts @@ -46,6 +46,7 @@ describe('updateSourceMappingsProperties', () => { appMappings: { properties: { a: { type: 'keyword' }, + b: { type: 'long' }, c: { type: 'long' }, ...getBaseMappings().properties, }, @@ -67,8 +68,10 @@ describe('updateSourceMappingsProperties', () => { it('should not update mappings when there are no changes', async () => { // we overwrite the app mappings to have the "unchanged" values with respect to the index mappings const sameMappingsParams = chain(params) + // let's not introduce 'c' for now + .set('indexTypes', ['a', 'b']) // even if the app versions are more recent, we emulate a scenario where mappings haven NOT changed - .set('latestMappingsVersions', { a: '10.1.0', b: '10.1.0', c: '10.1.0' }) + .set('latestMappingsVersions', { a: '10.1.0', b: '10.1.0' }) .value(); const result = await updateSourceMappingsProperties(sameMappingsParams)(); @@ -77,6 +80,28 @@ describe('updateSourceMappingsProperties', () => { expect(result).toHaveProperty('right', 'update_mappings_succeeded'); }); + it('should update mappings if there are new types', async () => { + // we overwrite the app mappings to have the "unchanged" values with respect to the index mappings + const sameMappingsParams = chain(params) + // even if the app versions are more recent, we emulate a scenario where mappings haven NOT changed + .set('latestMappingsVersions', { a: '10.1.0', b: '10.1.0', c: '10.1.0' }) + .value(); + const result = await updateSourceMappingsProperties(sameMappingsParams)(); + + expect(client.indices.putMapping).toHaveBeenCalledTimes(1); + expect(client.indices.putMapping).toHaveBeenCalledWith( + expect.objectContaining({ + properties: expect.objectContaining({ + a: { type: 'keyword' }, + b: { type: 'long' }, + c: { type: 'long' }, + }), + }) + ); + expect(Either.isRight(result)).toEqual(true); + expect(result).toHaveProperty('right', 'update_mappings_succeeded'); + }); + it('should return that mappings are updated when changes are compatible', async () => { const result = await updateSourceMappingsProperties(params)(); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/compare_mappings.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/compare_mappings.test.ts index 29a1d6cfe4849..e0700f4c42481 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/compare_mappings.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/compare_mappings.test.ts @@ -8,17 +8,20 @@ import type { IndexMappingMeta } from '@kbn/core-saved-objects-base-server-internal'; import { getBaseMappings } from './build_active_mappings'; -import { getUpdatedTypes, getUpdatedRootFields } from './compare_mappings'; +import { getUpdatedRootFields, getNewAndUpdatedTypes } from './compare_mappings'; -describe('getUpdatedTypes', () => { +describe('getNewAndUpdatedTypes', () => { test('returns all types if _meta is missing in indexMappings', () => { const indexTypes = ['foo', 'bar']; const latestMappingsVersions = {}; - expect(getUpdatedTypes({ indexTypes, indexMeta: undefined, latestMappingsVersions })).toEqual([ - 'foo', - 'bar', - ]); + const { newTypes, updatedTypes } = getNewAndUpdatedTypes({ + indexTypes, + indexMeta: undefined, + latestMappingsVersions, + }); + expect(newTypes).toEqual([]); + expect(updatedTypes).toEqual(['foo', 'bar']); }); test('returns all types if migrationMappingPropertyHashes and mappingVersions are missing in indexMappings', () => { @@ -26,14 +29,17 @@ describe('getUpdatedTypes', () => { const indexMeta: IndexMappingMeta = {}; const latestMappingsVersions = {}; - expect(getUpdatedTypes({ indexTypes, indexMeta, latestMappingsVersions })).toEqual([ - 'foo', - 'bar', - ]); + const { newTypes, updatedTypes } = getNewAndUpdatedTypes({ + indexTypes, + indexMeta, + latestMappingsVersions, + }); + expect(newTypes).toEqual([]); + expect(updatedTypes).toEqual(['foo', 'bar']); }); describe('when ONLY migrationMappingPropertyHashes exists in indexMappings', () => { - test('uses the provided hashToVersionMap to compare changes and return only the types that have changed', async () => { + test('uses the provided hashToVersionMap to compare changes and return new types and types that have changed', async () => { const indexTypes = ['type1', 'type2', 'type4']; const indexMeta: IndexMappingMeta = { migrationMappingPropertyHashes: { @@ -55,14 +61,19 @@ describe('getUpdatedTypes', () => { type4: '10.5.0', // new type, no need to pick it up }; - expect( - getUpdatedTypes({ indexTypes, indexMeta, latestMappingsVersions, hashToVersionMap }) - ).toEqual(['type2']); + const { newTypes, updatedTypes } = getNewAndUpdatedTypes({ + indexTypes, + indexMeta, + latestMappingsVersions, + hashToVersionMap, + }); + expect(newTypes).toEqual(['type4']); + expect(updatedTypes).toEqual(['type2']); }); }); describe('when mappingVersions exist in indexMappings', () => { - test('compares the modelVersions and returns only the types that have changed', async () => { + test('compares the modelVersions and returns new types and types that have changed', async () => { const indexTypes = ['type1', 'type2', 'type4']; const indexMeta: IndexMappingMeta = { @@ -89,9 +100,14 @@ describe('getUpdatedTypes', () => { // empty on purpose, not used as mappingVersions is present in indexMappings }; - expect( - getUpdatedTypes({ indexTypes, indexMeta, latestMappingsVersions, hashToVersionMap }) - ).toEqual(['type2']); + const { newTypes, updatedTypes } = getNewAndUpdatedTypes({ + indexTypes, + indexMeta, + latestMappingsVersions, + hashToVersionMap, + }); + expect(newTypes).toEqual(['type4']); + expect(updatedTypes).toEqual(['type2']); }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/compare_mappings.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/compare_mappings.ts index 6e2ac42dee2a3..d91635db0cd7a 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/compare_mappings.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/compare_mappings.ts @@ -32,43 +32,56 @@ export const getUpdatedRootFields = (indexMappings: IndexMapping): string[] => { .map(([propertyName]) => propertyName); }; +interface GetUpdatedTypesParams { + indexMeta?: IndexMappingMeta; + indexTypes: string[]; + latestMappingsVersions: VirtualVersionMap; + hashToVersionMap?: Record; +} + /** * Compares the current vs stored mappings' hashes or modelVersions. - * Returns a list with all the types that have been updated. + * Returns 2 lists: one with all the new types and one with the types that have been updated. * @param indexMeta The meta information stored in the SO index * @param knownTypes The list of SO types that belong to the index and are enabled * @param latestMappingsVersions A map holding [type => version] with the latest versions where mappings have changed for each type * @param hashToVersionMap A map holding information about [md5 => modelVersion] equivalence - * @returns the list of types that have been updated (in terms of their mappings) + * @returns the lists of new types and updated types */ -export const getUpdatedTypes = ({ +export const getNewAndUpdatedTypes = ({ indexMeta, indexTypes, latestMappingsVersions, hashToVersionMap = {}, -}: { - indexMeta?: IndexMappingMeta; - indexTypes: string[]; - latestMappingsVersions: VirtualVersionMap; - hashToVersionMap?: Record; -}): string[] => { +}: GetUpdatedTypesParams) => { if (!indexMeta || (!indexMeta.mappingVersions && !indexMeta.migrationMappingPropertyHashes)) { // if we currently do NOT have meta information stored in the index // we consider that all types have been updated - return indexTypes; + return { newTypes: [], updatedTypes: indexTypes }; } // If something exists in stored, but is missing in current // we don't care, as it could be a disabled plugin, etc // and keeping stale stuff around is better than migrating unecessesarily. - return indexTypes.filter((type) => - isTypeUpdated({ + const newTypes: string[] = []; + const updatedTypes: string[] = []; + + indexTypes.forEach((type) => { + const status = checkTypeStatus({ type, mappingVersion: latestMappingsVersions[type], indexMeta, hashToVersionMap, - }) - ); + }); + + if (status === 'new') { + newTypes.push(type); + } else if (status === 'updated') { + updatedTypes.push(type); + } + }); + + return { newTypes, updatedTypes }; }; /** @@ -77,9 +90,9 @@ export const getUpdatedTypes = ({ * @param mappingVersion The most recent model version that includes mappings changes * @param indexMeta The meta information stored in the SO index * @param hashToVersionMap A map holding information about [md5 => modelVersion] equivalence - * @returns true if the mappings for the given type have changed since Kibana was last started + * @returns 'new' | 'updated' | 'unchanged' depending on whether the type has changed */ -function isTypeUpdated({ +function checkTypeStatus({ type, mappingVersion, indexMeta, @@ -89,7 +102,7 @@ function isTypeUpdated({ mappingVersion: string; indexMeta: IndexMappingMeta; hashToVersionMap: Record; -}): boolean { +}): 'new' | 'updated' | 'unchanged' { const latestMappingsVersion = Semver.parse(mappingVersion); if (!latestMappingsVersion) { throw new Error( @@ -103,26 +116,28 @@ function isTypeUpdated({ if (!indexVersion) { // either a new type, and thus there's not need to update + pickup any docs // or an old re-enabled type, which will be updated on OUTDATED_DOCUMENTS_TRANSFORM - return false; + return 'new'; } // if the last version where mappings have changed is more recent than the one stored in the index // it means that the type has been updated - return latestMappingsVersion.compare(indexVersion) === 1; + return latestMappingsVersion.compare(indexVersion) === 1 ? 'updated' : 'unchanged'; } else if (indexMeta.migrationMappingPropertyHashes) { const latestHash = indexMeta.migrationMappingPropertyHashes?.[type]; if (!latestHash) { // either a new type, and thus there's not need to update + pickup any docs // or an old re-enabled type, which will be updated on OUTDATED_DOCUMENTS_TRANSFORM - return false; + return 'new'; } const indexEquivalentVersion = hashToVersionMap[`${type}|${latestHash}`]; - return !indexEquivalentVersion || latestMappingsVersion.compare(indexEquivalentVersion) === 1; + return !indexEquivalentVersion || latestMappingsVersion.compare(indexEquivalentVersion) === 1 + ? 'updated' + : 'unchanged'; } // at this point, the mappings do not contain any meta informataion // we consider the type has been updated, out of caution - return true; + return 'updated'; } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/diff_mappings.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/diff_mappings.test.ts index d7036f1264a99..3b7391cb4661a 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/diff_mappings.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/diff_mappings.test.ts @@ -8,12 +8,14 @@ import type { IndexMapping } from '@kbn/core-saved-objects-base-server-internal'; import { getBaseMappings } from './build_active_mappings'; -import { getUpdatedRootFields, getUpdatedTypes } from './compare_mappings'; +import { getUpdatedRootFields, getNewAndUpdatedTypes } from './compare_mappings'; import { diffMappings } from './diff_mappings'; jest.mock('./compare_mappings'); const getUpdatedRootFieldsMock = getUpdatedRootFields as jest.MockedFn; -const getUpdatedTypesMock = getUpdatedTypes as jest.MockedFn; +const getNewAndUpdatedTypesMock = getNewAndUpdatedTypes as jest.MockedFn< + typeof getNewAndUpdatedTypes +>; const dummyMappings: IndexMapping = { _meta: { @@ -55,7 +57,7 @@ const dummyHashToVersionMap = { describe('diffMappings', () => { beforeEach(() => { getUpdatedRootFieldsMock.mockReset(); - getUpdatedTypesMock.mockReset(); + getNewAndUpdatedTypesMock.mockReset(); }); test('is different if dynamic is different', () => { @@ -113,14 +115,17 @@ describe('diffMappings', () => { expect(getUpdatedRootFieldsMock).toHaveBeenCalledTimes(1); expect(getUpdatedRootFieldsMock).toHaveBeenCalledWith(initialMappings); - expect(getUpdatedTypesMock).not.toHaveBeenCalled(); + expect(getNewAndUpdatedTypesMock).not.toHaveBeenCalled(); }); }); - describe('if some types have changed', () => { + describe('if there are new or updated types', () => { test('returns a changed type', () => { getUpdatedRootFieldsMock.mockReturnValueOnce([]); - getUpdatedTypesMock.mockReturnValueOnce(['foo', 'bar']); + getNewAndUpdatedTypesMock.mockReturnValueOnce({ + newTypes: ['baz'], + updatedTypes: ['foo'], + }); expect( diffMappings({ @@ -136,8 +141,8 @@ describe('diffMappings', () => { expect(getUpdatedRootFieldsMock).toHaveBeenCalledTimes(1); expect(getUpdatedRootFieldsMock).toHaveBeenCalledWith(initialMappings); - expect(getUpdatedTypesMock).toHaveBeenCalledTimes(1); - expect(getUpdatedTypesMock).toHaveBeenCalledWith({ + expect(getNewAndUpdatedTypesMock).toHaveBeenCalledTimes(1); + expect(getNewAndUpdatedTypesMock).toHaveBeenCalledWith({ indexTypes: ['foo', 'bar', 'baz'], indexMeta: initialMappings._meta, latestMappingsVersions: { @@ -151,7 +156,10 @@ describe('diffMappings', () => { describe('if no root field or types have changed', () => { test('returns undefined', () => { getUpdatedRootFieldsMock.mockReturnValueOnce([]); - getUpdatedTypesMock.mockReturnValueOnce([]); + getNewAndUpdatedTypesMock.mockReturnValueOnce({ + newTypes: [], + updatedTypes: [], + }); expect( diffMappings({ diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/diff_mappings.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/diff_mappings.ts index 788ad9e282b6e..12247d7562bf7 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/diff_mappings.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/diff_mappings.ts @@ -7,7 +7,7 @@ */ import type { IndexMapping, VirtualVersionMap } from '@kbn/core-saved-objects-base-server-internal'; -import { getUpdatedRootFields, getUpdatedTypes } from './compare_mappings'; +import { getNewAndUpdatedTypes, getUpdatedRootFields } from './compare_mappings'; /** * Diffs the stored vs app mappings. @@ -55,8 +55,9 @@ export function diffMappings({ } /** - * Finds a property that has changed its schema with respect to the mappings stored in the SO index - * It can either be a root field or a SO type + * Finds a property (either a root field or a SO type) that either: + * - is new (did not exist in the current mappings) + * - has changed its schema with respect to the mappings stored in the SO index * @returns the name of the property (if any) */ function findChangedProp({ @@ -75,7 +76,7 @@ function findChangedProp({ return updatedFields[0]; } - const updatedTypes = getUpdatedTypes({ + const { newTypes, updatedTypes } = getNewAndUpdatedTypes({ indexMeta: indexMappings._meta, indexTypes, latestMappingsVersions, @@ -83,6 +84,8 @@ function findChangedProp({ }); if (updatedTypes.length) { return updatedTypes[0]; + } else if (newTypes.length) { + return newTypes[0]; } return undefined; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts index 961bc08e735ee..3a3d0747a8f16 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts @@ -2688,6 +2688,18 @@ describe('migrations v2 model', () => { }); }); + it('CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_META if ONLY new SO types have been added', () => { + const res: ResponseType<'CHECK_TARGET_MAPPINGS'> = Either.left({ + type: 'types_added' as const, + updatedFields: [], + newTypes: ['newFeatureType'], + }); + const newState = model(checkTargetTypesMappingsState, res) as UpdateTargetMappingsMeta; + expect(newState.controlState).toEqual('UPDATE_TARGET_MAPPINGS_META'); + expect(newState.retryCount).toEqual(0); + expect(newState.retryDelay).toEqual(0); + }); + it('CHECK_TARGET_MAPPINGS -> CHECK_VERSION_INDEX_READY_ACTIONS if types match (there might be additions in core fields)', () => { const res: ResponseType<'CHECK_TARGET_MAPPINGS'> = Either.right({ type: 'types_match' as const, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts index cf0fe5fa3396b..33e16efa59564 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts @@ -1497,6 +1497,12 @@ export const model = (currentState: State, resW: ResponseType): }, ], }; + } else if (isTypeof(left, 'types_added')) { + // compatible migration: ONLY new SO types have been introduced, skip directly to UPDATE_TARGET_MAPPINGS_META + return { + ...stateP, + controlState: 'UPDATE_TARGET_MAPPINGS_META', + }; } else { throwBadResponse(stateP, res as never); }