From e9dcf029ccdb86c95d80427bedac42ba6dadaa5f Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Fri, 16 Jul 2021 10:44:22 -0400 Subject: [PATCH 01/17] Updating unit tests --- .../server/saved_objects/migrations.test.ts | 242 ++- .../server/saved_objects/migrations.ts | 27 +- .../server/saved_objects/migrations.test.ts | 1541 +++++++++-------- .../server/saved_objects/migrations.ts | 39 +- .../server/create_migration.test.ts | 326 ++++ .../server/create_migration.ts | 47 +- 6 files changed, 1359 insertions(+), 863 deletions(-) diff --git a/x-pack/plugins/actions/server/saved_objects/migrations.test.ts b/x-pack/plugins/actions/server/saved_objects/migrations.test.ts index 4c30925e61894..17ba57e602723 100644 --- a/x-pack/plugins/actions/server/saved_objects/migrations.test.ts +++ b/x-pack/plugins/actions/server/saved_objects/migrations.test.ts @@ -15,121 +15,185 @@ import { migrationMocks } from 'src/core/server/mocks'; const context = migrationMocks.createContext(); const encryptedSavedObjectsSetup = encryptedSavedObjectsMock.createSetup(); -describe('7.10.0', () => { +describe('successful migrations', () => { beforeEach(() => { jest.resetAllMocks(); - encryptedSavedObjectsSetup.createMigration.mockImplementation( - (shouldMigrateWhenPredicate, migration) => migration - ); + encryptedSavedObjectsSetup.createMigration.mockImplementation((_, migration) => migration); }); - test('add hasAuth config property for .email actions', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const action = getMockDataForEmail({}); - const migratedAction = migration710(action, context); - expect(migratedAction.attributes.config).toEqual({ - hasAuth: true, - }); - expect(migratedAction).toEqual({ - ...action, - attributes: { - ...action.attributes, - config: { - hasAuth: true, + describe('7.10.0', () => { + test('add hasAuth config property for .email actions', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const action = getMockDataForEmail({}); + const migratedAction = migration710(action, context); + expect(migratedAction.attributes.config).toEqual({ + hasAuth: true, + }); + expect(migratedAction).toEqual({ + ...action, + attributes: { + ...action.attributes, + config: { + hasAuth: true, + }, }, - }, + }); }); - }); - test('rename cases configuration object', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const action = getCasesMockData({}); - const migratedAction = migration710(action, context); - expect(migratedAction.attributes.config).toEqual({ - incidentConfiguration: { mapping: [] }, - }); - expect(migratedAction).toEqual({ - ...action, - attributes: { - ...action.attributes, - config: { - incidentConfiguration: { mapping: [] }, + test('rename cases configuration object', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const action = getCasesMockData({}); + const migratedAction = migration710(action, context); + expect(migratedAction.attributes.config).toEqual({ + incidentConfiguration: { mapping: [] }, + }); + expect(migratedAction).toEqual({ + ...action, + attributes: { + ...action.attributes, + config: { + incidentConfiguration: { mapping: [] }, + }, }, - }, + }); }); }); -}); - -describe('7.11.0', () => { - beforeEach(() => { - jest.resetAllMocks(); - encryptedSavedObjectsSetup.createMigration.mockImplementation( - (shouldMigrateWhenPredicate, migration) => migration - ); - }); - test('add hasAuth = true for .webhook actions with user and password', () => { - const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; - const action = getMockDataForWebhook({}, true); - expect(migration711(action, context)).toMatchObject({ - ...action, - attributes: { - ...action.attributes, - config: { - hasAuth: true, + describe('7.11.0', () => { + test('add hasAuth = true for .webhook actions with user and password', () => { + const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; + const action = getMockDataForWebhook({}, true); + expect(migration711(action, context)).toMatchObject({ + ...action, + attributes: { + ...action.attributes, + config: { + hasAuth: true, + }, }, - }, + }); }); - }); - test('add hasAuth = false for .webhook actions without user and password', () => { - const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; - const action = getMockDataForWebhook({}, false); - expect(migration711(action, context)).toMatchObject({ - ...action, - attributes: { - ...action.attributes, - config: { - hasAuth: false, + test('add hasAuth = false for .webhook actions without user and password', () => { + const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; + const action = getMockDataForWebhook({}, false); + expect(migration711(action, context)).toMatchObject({ + ...action, + attributes: { + ...action.attributes, + config: { + hasAuth: false, + }, }, - }, + }); }); - }); - test('remove cases mapping object', () => { - const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; - const action = getMockData({ - config: { incidentConfiguration: { mapping: [] }, isCaseOwned: true, another: 'value' }, + test('remove cases mapping object', () => { + const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; + const action = getMockData({ + config: { incidentConfiguration: { mapping: [] }, isCaseOwned: true, another: 'value' }, + }); + expect(migration711(action, context)).toEqual({ + ...action, + attributes: { + ...action.attributes, + config: { + another: 'value', + }, + }, + }); }); - expect(migration711(action, context)).toEqual({ - ...action, - attributes: { - ...action.attributes, - config: { - another: 'value', + }); + + describe('7.14.0', () => { + test('add isMissingSecrets property for actions', () => { + const migration714 = getMigrations(encryptedSavedObjectsSetup)['7.14.0']; + const action = getMockData({ isMissingSecrets: undefined }); + const migratedAction = migration714(action, context); + expect(migratedAction).toEqual({ + ...action, + attributes: { + ...action.attributes, + isMissingSecrets: false, }, - }, + }); }); }); }); -describe('7.14.0', () => { +describe('handles errors during migrations', () => { beforeEach(() => { jest.resetAllMocks(); - encryptedSavedObjectsSetup.createMigration.mockImplementation( - (shouldMigrateWhenPredicate, migration) => migration - ); + encryptedSavedObjectsSetup.createMigration.mockImplementation(() => () => { + throw new Error(`Can't migrate!`); + }); + }); + + describe('7.10.0 throws if migration fails', () => { + test('should show the proper exception', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const action = getMockDataForEmail({}); + expect(() => { + migration710(action, context); + }).toThrowError(`Can't migrate!`); + expect(context.log.error).toHaveBeenCalledWith( + `encryptedSavedObject 7.10.0 migration failed for action ${action.id} with error: Can't migrate!`, + { + migrations: { + actionDocument: { + ...action, + attributes: { + ...action.attributes, + }, + }, + }, + } + ); + }); + }); + + describe('7.11.0 throws if migration fails', () => { + test('should show the proper exception', () => { + const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; + const action = getMockDataForEmail({}); + expect(() => { + migration711(action, context); + }).toThrowError(`Can't migrate!`); + expect(context.log.error).toHaveBeenCalledWith( + `encryptedSavedObject 7.11.0 migration failed for action ${action.id} with error: Can't migrate!`, + { + migrations: { + actionDocument: { + ...action, + attributes: { + ...action.attributes, + }, + }, + }, + } + ); + }); }); - test('add isMissingSecrets property for actions', () => { - const migration714 = getMigrations(encryptedSavedObjectsSetup)['7.14.0']; - const action = getMockData({ isMissingSecrets: undefined }); - const migratedAction = migration714(action, context); - expect(migratedAction).toEqual({ - ...action, - attributes: { - ...action.attributes, - isMissingSecrets: false, - }, + describe('7.14.0 throws if migration fails', () => { + test('should show the proper exception', () => { + const migration714 = getMigrations(encryptedSavedObjectsSetup)['7.14.0']; + const action = getMockDataForEmail({}); + expect(() => { + migration714(action, context); + }).toThrowError(`Can't migrate!`); + expect(context.log.error).toHaveBeenCalledWith( + `encryptedSavedObject 7.14.0 migration failed for action ${action.id} with error: Can't migrate!`, + { + migrations: { + actionDocument: { + ...action, + attributes: { + ...action.attributes, + }, + }, + }, + } + ); }); }); }); diff --git a/x-pack/plugins/actions/server/saved_objects/migrations.ts b/x-pack/plugins/actions/server/saved_objects/migrations.ts index 17932b6b90f97..e672dd4921e5d 100644 --- a/x-pack/plugins/actions/server/saved_objects/migrations.ts +++ b/x-pack/plugins/actions/server/saved_objects/migrations.ts @@ -23,17 +23,35 @@ type ActionMigration = ( doc: SavedObjectUnsanitizedDoc ) => SavedObjectUnsanitizedDoc; +type IsMigrationNeededPredicate = ( + doc: SavedObjectUnsanitizedDoc +) => doc is SavedObjectUnsanitizedDoc; + +function createEsoMigration( + encryptedSavedObjects: EncryptedSavedObjectsPluginSetup, + isMigrationNeededPredicate: IsMigrationNeededPredicate, + migrationFunc: ActionMigration +) { + return encryptedSavedObjects.createMigration( + isMigrationNeededPredicate, + migrationFunc, + true // shouldMigrateIfDecryptionFails flag that applies the migration to undecrypted document if decryption fails + ); +} + export function getMigrations( encryptedSavedObjects: EncryptedSavedObjectsPluginSetup ): SavedObjectMigrationMap { - const migrationActionsTen = encryptedSavedObjects.createMigration( + const migrationActionsTen = createEsoMigration( + encryptedSavedObjects, (doc): doc is SavedObjectUnsanitizedDoc => doc.attributes.config?.hasOwnProperty('casesConfiguration') || doc.attributes.actionTypeId === '.email', pipeMigrations(renameCasesConfigurationObject, addHasAuthConfigurationObject) ); - const migrationActionsEleven = encryptedSavedObjects.createMigration( + const migrationActionsEleven = createEsoMigration( + encryptedSavedObjects, (doc): doc is SavedObjectUnsanitizedDoc => doc.attributes.config?.hasOwnProperty('isCaseOwned') || doc.attributes.config?.hasOwnProperty('incidentConfiguration') || @@ -41,7 +59,8 @@ export function getMigrations( pipeMigrations(removeCasesFieldMappings, addHasAuthConfigurationObject) ); - const migrationActionsFourteen = encryptedSavedObjects.createMigration( + const migrationActionsFourteen = createEsoMigration( + encryptedSavedObjects, (doc): doc is SavedObjectUnsanitizedDoc => true, pipeMigrations(addisMissingSecretsField) ); @@ -69,8 +88,8 @@ function executeMigrationWithErrorHandling( }, } ); + throw ex; } - return doc; }; } diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts b/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts index 4888116e43602..5c4af3dfcfd16 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts @@ -15,99 +15,80 @@ import { migrationMocks } from 'src/core/server/mocks'; const migrationContext = migrationMocks.createContext(); const encryptedSavedObjectsSetup = encryptedSavedObjectsMock.createSetup(); -describe('7.10.0', () => { +describe('successful migrations', () => { beforeEach(() => { jest.resetAllMocks(); - encryptedSavedObjectsSetup.createMigration.mockImplementation( - (shouldMigrateWhenPredicate, migration) => migration - ); + encryptedSavedObjectsSetup.createMigration.mockImplementation((_, migration) => migration); }); - - test('marks alerts as legacy', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const alert = getMockData({}); - expect(migration710(alert, migrationContext)).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, - meta: { - versionApiKeyLastmodified: 'pre-7.10.0', - }, - }, - }); - }); - - test('migrates the consumer for metrics', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const alert = getMockData({ - consumer: 'metrics', - }); - expect(migration710(alert, migrationContext)).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, - consumer: 'infrastructure', - meta: { - versionApiKeyLastmodified: 'pre-7.10.0', + describe('7.10.0', () => { + test('marks alerts as legacy', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const alert = getMockData({}); + expect(migration710(alert, migrationContext)).toMatchObject({ + ...alert, + attributes: { + ...alert.attributes, + meta: { + versionApiKeyLastmodified: 'pre-7.10.0', + }, }, - }, + }); }); - }); - test('migrates the consumer for siem', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const alert = getMockData({ - consumer: 'securitySolution', - }); - expect(migration710(alert, migrationContext)).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, - consumer: 'siem', - meta: { - versionApiKeyLastmodified: 'pre-7.10.0', + test('migrates the consumer for metrics', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const alert = getMockData({ + consumer: 'metrics', + }); + expect(migration710(alert, migrationContext)).toMatchObject({ + ...alert, + attributes: { + ...alert.attributes, + consumer: 'infrastructure', + meta: { + versionApiKeyLastmodified: 'pre-7.10.0', + }, }, - }, + }); }); - }); - test('migrates the consumer for alerting', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const alert = getMockData({ - consumer: 'alerting', - }); - expect(migration710(alert, migrationContext)).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, - consumer: 'alerts', - meta: { - versionApiKeyLastmodified: 'pre-7.10.0', + test('migrates the consumer for siem', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const alert = getMockData({ + consumer: 'securitySolution', + }); + expect(migration710(alert, migrationContext)).toMatchObject({ + ...alert, + attributes: { + ...alert.attributes, + consumer: 'siem', + meta: { + versionApiKeyLastmodified: 'pre-7.10.0', + }, }, - }, + }); }); - }); - test('migrates PagerDuty actions to set a default dedupkey of the AlertId', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const alert = getMockData({ - actions: [ - { - actionTypeId: '.pagerduty', - group: 'default', - params: { - summary: 'fired {{alertInstanceId}}', - eventAction: 'resolve', - component: '', + test('migrates the consumer for alerting', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const alert = getMockData({ + consumer: 'alerting', + }); + expect(migration710(alert, migrationContext)).toMatchObject({ + ...alert, + attributes: { + ...alert.attributes, + consumer: 'alerts', + meta: { + versionApiKeyLastmodified: 'pre-7.10.0', }, - id: 'b62ea790-5366-4abc-a7df-33db1db78410', }, - ], + }); }); - expect(migration710(alert, migrationContext)).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, + + test('migrates PagerDuty actions to set a default dedupkey of the AlertId', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const alert = getMockData({ actions: [ { actionTypeId: '.pagerduty', @@ -115,37 +96,36 @@ describe('7.10.0', () => { params: { summary: 'fired {{alertInstanceId}}', eventAction: 'resolve', - dedupKey: '{{alertId}}', component: '', }, id: 'b62ea790-5366-4abc-a7df-33db1db78410', }, ], - }, - }); - }); - - test('skips PagerDuty actions with a specified dedupkey', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const alert = getMockData({ - actions: [ - { - actionTypeId: '.pagerduty', - group: 'default', - params: { - summary: 'fired {{alertInstanceId}}', - eventAction: 'trigger', - dedupKey: '{{alertInstanceId}}', - component: '', - }, - id: 'b62ea790-5366-4abc-a7df-33db1db78410', + }); + expect(migration710(alert, migrationContext)).toMatchObject({ + ...alert, + attributes: { + ...alert.attributes, + actions: [ + { + actionTypeId: '.pagerduty', + group: 'default', + params: { + summary: 'fired {{alertInstanceId}}', + eventAction: 'resolve', + dedupKey: '{{alertId}}', + component: '', + }, + id: 'b62ea790-5366-4abc-a7df-33db1db78410', + }, + ], }, - ], + }); }); - expect(migration710(alert, migrationContext)).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, + + test('skips PagerDuty actions with a specified dedupkey', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const alert = getMockData({ actions: [ { actionTypeId: '.pagerduty', @@ -159,33 +139,31 @@ describe('7.10.0', () => { id: 'b62ea790-5366-4abc-a7df-33db1db78410', }, ], - }, - }); - }); - - test('skips PagerDuty actions with an eventAction of "trigger"', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const alert = getMockData({ - actions: [ - { - actionTypeId: '.pagerduty', - group: 'default', - params: { - summary: 'fired {{alertInstanceId}}', - eventAction: 'trigger', - component: '', - }, - id: 'b62ea790-5366-4abc-a7df-33db1db78410', + }); + expect(migration710(alert, migrationContext)).toMatchObject({ + ...alert, + attributes: { + ...alert.attributes, + actions: [ + { + actionTypeId: '.pagerduty', + group: 'default', + params: { + summary: 'fired {{alertInstanceId}}', + eventAction: 'trigger', + dedupKey: '{{alertInstanceId}}', + component: '', + }, + id: 'b62ea790-5366-4abc-a7df-33db1db78410', + }, + ], }, - ], + }); }); - expect(migration710(alert, migrationContext)).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, - meta: { - versionApiKeyLastmodified: 'pre-7.10.0', - }, + + test('skips PagerDuty actions with an eventAction of "trigger"', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const alert = getMockData({ actions: [ { actionTypeId: '.pagerduty', @@ -198,214 +176,115 @@ describe('7.10.0', () => { id: 'b62ea790-5366-4abc-a7df-33db1db78410', }, ], - }, - }); - }); - - test('creates execution status', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const alert = getMockData(); - const dateStart = Date.now(); - const migratedAlert = migration710(alert, migrationContext); - const dateStop = Date.now(); - const dateExecutionStatus = Date.parse( - migratedAlert.attributes.executionStatus.lastExecutionDate - ); - - expect(dateStart).toBeLessThanOrEqual(dateExecutionStatus); - expect(dateStop).toBeGreaterThanOrEqual(dateExecutionStatus); - - expect(migratedAlert).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, - executionStatus: { - lastExecutionDate: migratedAlert.attributes.executionStatus.lastExecutionDate, - status: 'pending', - error: null, + }); + expect(migration710(alert, migrationContext)).toMatchObject({ + ...alert, + attributes: { + ...alert.attributes, + meta: { + versionApiKeyLastmodified: 'pre-7.10.0', + }, + actions: [ + { + actionTypeId: '.pagerduty', + group: 'default', + params: { + summary: 'fired {{alertInstanceId}}', + eventAction: 'trigger', + component: '', + }, + id: 'b62ea790-5366-4abc-a7df-33db1db78410', + }, + ], }, - }, + }); }); - }); -}); -describe('7.10.0 migrates with failure', () => { - beforeEach(() => { - jest.resetAllMocks(); - encryptedSavedObjectsSetup.createMigration.mockImplementationOnce(() => () => { - throw new Error(`Can't migrate!`); - }); - }); + test('creates execution status', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const alert = getMockData(); + const dateStart = Date.now(); + const migratedAlert = migration710(alert, migrationContext); + const dateStop = Date.now(); + const dateExecutionStatus = Date.parse( + migratedAlert.attributes.executionStatus.lastExecutionDate + ); - test('should show the proper exception', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const alert = getMockData({ - consumer: 'alerting', - }); - const res = migration710(alert, migrationContext); - expect(res).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, - }, - }); - expect(migrationContext.log.error).toHaveBeenCalledWith( - `encryptedSavedObject 7.10.0 migration failed for alert ${alert.id} with error: Can't migrate!`, - { - migrations: { - alertDocument: { - ...alert, - attributes: { - ...alert.attributes, - }, + expect(dateStart).toBeLessThanOrEqual(dateExecutionStatus); + expect(dateStop).toBeGreaterThanOrEqual(dateExecutionStatus); + + expect(migratedAlert).toMatchObject({ + ...alert, + attributes: { + ...alert.attributes, + executionStatus: { + lastExecutionDate: migratedAlert.attributes.executionStatus.lastExecutionDate, + status: 'pending', + error: null, }, }, - } - ); - }); -}); - -describe('7.11.0', () => { - beforeEach(() => { - jest.resetAllMocks(); - encryptedSavedObjectsSetup.createMigration.mockImplementation( - (shouldMigrateWhenPredicate, migration) => migration - ); - }); - - test('add updatedAt field to alert - set to SavedObject updated_at attribute', () => { - const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; - const alert = getMockData({}, true); - expect(migration711(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, - updatedAt: alert.updated_at, - notifyWhen: 'onActiveAlert', - }, + }); }); }); - test('add updatedAt field to alert - set to createdAt when SavedObject updated_at is not defined', () => { - const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; - const alert = getMockData({}); - expect(migration711(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, - updatedAt: alert.attributes.createdAt, - notifyWhen: 'onActiveAlert', - }, + describe('7.11.0', () => { + test('add updatedAt field to alert - set to SavedObject updated_at attribute', () => { + const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; + const alert = getMockData({}, true); + expect(migration711(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + updatedAt: alert.updated_at, + notifyWhen: 'onActiveAlert', + }, + }); }); - }); - test('add notifyWhen=onActiveAlert when throttle is null', () => { - const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; - const alert = getMockData({}); - expect(migration711(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, - updatedAt: alert.attributes.createdAt, - notifyWhen: 'onActiveAlert', - }, + test('add updatedAt field to alert - set to createdAt when SavedObject updated_at is not defined', () => { + const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; + const alert = getMockData({}); + expect(migration711(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + updatedAt: alert.attributes.createdAt, + notifyWhen: 'onActiveAlert', + }, + }); }); - }); - test('add notifyWhen=onActiveAlert when throttle is set', () => { - const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; - const alert = getMockData({ throttle: '5m' }); - expect(migration711(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, - updatedAt: alert.attributes.createdAt, - notifyWhen: 'onThrottleInterval', - }, + test('add notifyWhen=onActiveAlert when throttle is null', () => { + const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; + const alert = getMockData({}); + expect(migration711(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + updatedAt: alert.attributes.createdAt, + notifyWhen: 'onActiveAlert', + }, + }); }); - }); -}); - -describe('7.11.2', () => { - beforeEach(() => { - jest.resetAllMocks(); - encryptedSavedObjectsSetup.createMigration.mockImplementation( - (shouldMigrateWhenPredicate, migration) => migration - ); - }); - test('transforms connectors that support incident correctly', () => { - const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; - const alert = getMockData({ - actions: [ - { - actionTypeId: '.jira', - group: 'threshold met', - params: { - subAction: 'pushToService', - subActionParams: { - title: 'Jira summary', - issueType: '10001', - comments: [ - { - commentId: '1', - comment: 'jira comment', - }, - ], - description: 'Jira description', - savedObjectId: '{{alertId}}', - priority: 'Highest', - parent: 'CASES-78', - labels: ['test'], - }, - }, - id: 'b1abe42d-ae1a-4a6a-b5ec-482ce0492c14', - }, - { - actionTypeId: '.resilient', - group: 'threshold met', - params: { - subAction: 'pushToService', - subActionParams: { - savedObjectId: '{{alertId}}', - incidentTypes: ['17', '21'], - severityCode: '5', - title: 'IBM name', - description: 'IBM description', - comments: [ - { - commentId: 'alert-comment', - comment: 'IBM comment', - }, - ], - }, - }, - id: '75d63268-9a83-460f-9026-0028f4f7dac4', - }, - { - actionTypeId: '.servicenow', - group: 'threshold met', - params: { - subAction: 'pushToService', - subActionParams: { - severity: '2', - impact: '2', - urgency: '2', - savedObjectId: '{{alertId}}', - title: 'SN short desc', - description: 'SN desc', - comment: 'sn comment', - }, - }, - id: '1266562a-4e1f-4305-99ca-1b44c469b26e', + test('add notifyWhen=onActiveAlert when throttle is set', () => { + const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; + const alert = getMockData({ throttle: '5m' }); + expect(migration711(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + updatedAt: alert.attributes.createdAt, + notifyWhen: 'onThrottleInterval', }, - ], + }); }); + }); - expect(migration7112(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, + describe('7.11.2', () => { + test('transforms connectors that support incident correctly', () => { + const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; + const alert = getMockData({ actions: [ { actionTypeId: '.jira', @@ -413,20 +292,19 @@ describe('7.11.2', () => { params: { subAction: 'pushToService', subActionParams: { - incident: { - summary: 'Jira summary', - description: 'Jira description', - issueType: '10001', - priority: 'Highest', - parent: 'CASES-78', - labels: ['test'], - }, + title: 'Jira summary', + issueType: '10001', comments: [ { commentId: '1', comment: 'jira comment', }, ], + description: 'Jira description', + savedObjectId: '{{alertId}}', + priority: 'Highest', + parent: 'CASES-78', + labels: ['test'], }, }, id: 'b1abe42d-ae1a-4a6a-b5ec-482ce0492c14', @@ -437,12 +315,11 @@ describe('7.11.2', () => { params: { subAction: 'pushToService', subActionParams: { - incident: { - name: 'IBM name', - description: 'IBM description', - incidentTypes: ['17', '21'], - severityCode: '5', - }, + savedObjectId: '{{alertId}}', + incidentTypes: ['17', '21'], + severityCode: '5', + title: 'IBM name', + description: 'IBM description', comments: [ { commentId: 'alert-comment', @@ -459,81 +336,205 @@ describe('7.11.2', () => { params: { subAction: 'pushToService', subActionParams: { - incident: { - short_description: 'SN short desc', - description: 'SN desc', - severity: '2', - impact: '2', - urgency: '2', - }, - comments: [{ commentId: '1', comment: 'sn comment' }], + severity: '2', + impact: '2', + urgency: '2', + savedObjectId: '{{alertId}}', + title: 'SN short desc', + description: 'SN desc', + comment: 'sn comment', }, }, id: '1266562a-4e1f-4305-99ca-1b44c469b26e', }, ], - }, - }); - }); + }); - test('it transforms only subAction=pushToService', () => { - const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; - const alert = getMockData({ - actions: [ - { - actionTypeId: '.jira', - group: 'threshold met', - params: { - subAction: 'issues', - subActionParams: { issues: 'Task' }, - }, - id: '1266562a-4e1f-4305-99ca-1b44c469b26e', + expect(migration7112(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + actions: [ + { + actionTypeId: '.jira', + group: 'threshold met', + params: { + subAction: 'pushToService', + subActionParams: { + incident: { + summary: 'Jira summary', + description: 'Jira description', + issueType: '10001', + priority: 'Highest', + parent: 'CASES-78', + labels: ['test'], + }, + comments: [ + { + commentId: '1', + comment: 'jira comment', + }, + ], + }, + }, + id: 'b1abe42d-ae1a-4a6a-b5ec-482ce0492c14', + }, + { + actionTypeId: '.resilient', + group: 'threshold met', + params: { + subAction: 'pushToService', + subActionParams: { + incident: { + name: 'IBM name', + description: 'IBM description', + incidentTypes: ['17', '21'], + severityCode: '5', + }, + comments: [ + { + commentId: 'alert-comment', + comment: 'IBM comment', + }, + ], + }, + }, + id: '75d63268-9a83-460f-9026-0028f4f7dac4', + }, + { + actionTypeId: '.servicenow', + group: 'threshold met', + params: { + subAction: 'pushToService', + subActionParams: { + incident: { + short_description: 'SN short desc', + description: 'SN desc', + severity: '2', + impact: '2', + urgency: '2', + }, + comments: [{ commentId: '1', comment: 'sn comment' }], + }, + }, + id: '1266562a-4e1f-4305-99ca-1b44c469b26e', + }, + ], }, - ], + }); }); - expect(migration7112(alert, migrationContext)).toEqual(alert); - }); + test('it transforms only subAction=pushToService', () => { + const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; + const alert = getMockData({ + actions: [ + { + actionTypeId: '.jira', + group: 'threshold met', + params: { + subAction: 'issues', + subActionParams: { issues: 'Task' }, + }, + id: '1266562a-4e1f-4305-99ca-1b44c469b26e', + }, + ], + }); - test('it does not transforms other connectors', () => { - const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; - const alert = getMockData({ - actions: [ - { - actionTypeId: '.server-log', - group: 'threshold met', - params: { - level: 'info', - message: 'log message', + expect(migration7112(alert, migrationContext)).toEqual(alert); + }); + + test('it does not transforms other connectors', () => { + const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; + const alert = getMockData({ + actions: [ + { + actionTypeId: '.server-log', + group: 'threshold met', + params: { + level: 'info', + message: 'log message', + }, + id: '99257478-e591-4560-b264-441bdd4fe1d9', }, - id: '99257478-e591-4560-b264-441bdd4fe1d9', - }, - { - actionTypeId: '.servicenow', - group: 'threshold met', - params: { - subAction: 'pushToService', - subActionParams: { - severity: '2', - impact: '2', - urgency: '2', - savedObjectId: '{{alertId}}', - title: 'SN short desc', - description: 'SN desc', - comment: 'sn comment', + { + actionTypeId: '.servicenow', + group: 'threshold met', + params: { + subAction: 'pushToService', + subActionParams: { + severity: '2', + impact: '2', + urgency: '2', + savedObjectId: '{{alertId}}', + title: 'SN short desc', + description: 'SN desc', + comment: 'sn comment', + }, }, + id: '1266562a-4e1f-4305-99ca-1b44c469b26e', }, - id: '1266562a-4e1f-4305-99ca-1b44c469b26e', + ], + }); + + expect(migration7112(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + actions: [ + alert.attributes.actions![0], + { + actionTypeId: '.servicenow', + group: 'threshold met', + params: { + subAction: 'pushToService', + subActionParams: { + incident: { + short_description: 'SN short desc', + description: 'SN desc', + severity: '2', + impact: '2', + urgency: '2', + }, + comments: [{ commentId: '1', comment: 'sn comment' }], + }, + }, + id: '1266562a-4e1f-4305-99ca-1b44c469b26e', + }, + ], }, - ], + }); + }); + + test.each(['.jira', '.servicenow', '.resilient'])( + 'isAnyActionSupportIncidents should return true when %s is in actions', + (actionTypeId) => { + const doc = { + attributes: { actions: [{ actionTypeId }, { actionTypeId: '.server-log' }] }, + } as SavedObjectUnsanitizedDoc; + expect(isAnyActionSupportIncidents(doc)).toBe(true); + } + ); + + test('isAnyActionSupportIncidents should return false when there is no connector that supports incidents', () => { + const doc = { + attributes: { actions: [{ actionTypeId: '.server-log' }] }, + } as SavedObjectUnsanitizedDoc; + expect(isAnyActionSupportIncidents(doc)).toBe(false); }); - expect(migration7112(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, + test('it does not transforms alerts when the right structure connectors is already applied', () => { + const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; + const alert = getMockData({ actions: [ - alert.attributes.actions![0], + { + actionTypeId: '.server-log', + group: 'threshold met', + params: { + level: 'info', + message: 'log message', + }, + id: '99257478-e591-4560-b264-441bdd4fe1d9', + }, { actionTypeId: '.servicenow', group: 'threshold met', @@ -553,218 +554,109 @@ describe('7.11.2', () => { id: '1266562a-4e1f-4305-99ca-1b44c469b26e', }, ], - }, - }); - }); - - test.each(['.jira', '.servicenow', '.resilient'])( - 'isAnyActionSupportIncidents should return true when %s is in actions', - (actionTypeId) => { - const doc = { - attributes: { actions: [{ actionTypeId }, { actionTypeId: '.server-log' }] }, - } as SavedObjectUnsanitizedDoc; - expect(isAnyActionSupportIncidents(doc)).toBe(true); - } - ); + }); - test('isAnyActionSupportIncidents should return false when there is no connector that supports incidents', () => { - const doc = { - attributes: { actions: [{ actionTypeId: '.server-log' }] }, - } as SavedObjectUnsanitizedDoc; - expect(isAnyActionSupportIncidents(doc)).toBe(false); - }); + expect(migration7112(alert, migrationContext)).toEqual(alert); + }); - test('it does not transforms alerts when the right structure connectors is already applied', () => { - const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; - const alert = getMockData({ - actions: [ - { - actionTypeId: '.server-log', - group: 'threshold met', - params: { - level: 'info', - message: 'log message', + test('if incident attribute is an empty object, copy back the related attributes from subActionParams back to incident', () => { + const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; + const alert = getMockData({ + actions: [ + { + actionTypeId: '.server-log', + group: 'threshold met', + params: { + level: 'info', + message: 'log message', + }, + id: '99257478-e591-4560-b264-441bdd4fe1d9', }, - id: '99257478-e591-4560-b264-441bdd4fe1d9', - }, - { - actionTypeId: '.servicenow', - group: 'threshold met', - params: { - subAction: 'pushToService', - subActionParams: { - incident: { + { + actionTypeId: '.servicenow', + group: 'threshold met', + params: { + subAction: 'pushToService', + subActionParams: { short_description: 'SN short desc', description: 'SN desc', severity: '2', impact: '2', urgency: '2', + incident: {}, + comments: [{ commentId: '1', comment: 'sn comment' }], }, - comments: [{ commentId: '1', comment: 'sn comment' }], }, + id: '1266562a-4e1f-4305-99ca-1b44c469b26e', }, - id: '1266562a-4e1f-4305-99ca-1b44c469b26e', - }, - ], - }); - - expect(migration7112(alert, migrationContext)).toEqual(alert); - }); + ], + }); - test('if incident attribute is an empty object, copy back the related attributes from subActionParams back to incident', () => { - const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; - const alert = getMockData({ - actions: [ - { - actionTypeId: '.server-log', - group: 'threshold met', - params: { - level: 'info', - message: 'log message', - }, - id: '99257478-e591-4560-b264-441bdd4fe1d9', - }, - { - actionTypeId: '.servicenow', - group: 'threshold met', - params: { - subAction: 'pushToService', - subActionParams: { - short_description: 'SN short desc', - description: 'SN desc', - severity: '2', - impact: '2', - urgency: '2', - incident: {}, - comments: [{ commentId: '1', comment: 'sn comment' }], + expect(migration7112(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + actions: [ + alert.attributes.actions![0], + { + actionTypeId: '.servicenow', + group: 'threshold met', + params: { + subAction: 'pushToService', + subActionParams: { + incident: { + short_description: 'SN short desc', + description: 'SN desc', + severity: '2', + impact: '2', + urgency: '2', + }, + comments: [{ commentId: '1', comment: 'sn comment' }], + }, + }, + id: '1266562a-4e1f-4305-99ca-1b44c469b26e', }, - }, - id: '1266562a-4e1f-4305-99ca-1b44c469b26e', + ], }, - ], + }); }); - expect(migration7112(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, + test('custom action does not get migrated/loss', () => { + const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; + const alert = getMockData({ actions: [ - alert.attributes.actions![0], { - actionTypeId: '.servicenow', + actionTypeId: '.mike', group: 'threshold met', params: { subAction: 'pushToService', subActionParams: { - incident: { - short_description: 'SN short desc', - description: 'SN desc', - severity: '2', - impact: '2', - urgency: '2', - }, + short_description: 'SN short desc', + description: 'SN desc', + severity: '2', + impact: '2', + urgency: '2', + incident: {}, comments: [{ commentId: '1', comment: 'sn comment' }], }, }, id: '1266562a-4e1f-4305-99ca-1b44c469b26e', }, ], - }, - }); - }); + }); - test('custom action does not get migrated/loss', () => { - const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; - const alert = getMockData({ - actions: [ - { - actionTypeId: '.mike', - group: 'threshold met', - params: { - subAction: 'pushToService', - subActionParams: { - short_description: 'SN short desc', - description: 'SN desc', - severity: '2', - impact: '2', - urgency: '2', - incident: {}, - comments: [{ commentId: '1', comment: 'sn comment' }], - }, - }, - id: '1266562a-4e1f-4305-99ca-1b44c469b26e', - }, - ], + expect(migration7112(alert, migrationContext)).toEqual(alert); }); - - expect(migration7112(alert, migrationContext)).toEqual(alert); - }); -}); - -describe('7.13.0', () => { - beforeEach(() => { - jest.resetAllMocks(); - encryptedSavedObjectsSetup.createMigration.mockImplementation( - (shouldMigrateWhenPredicate, migration) => migration - ); }); - test('security solution alerts get migrated and remove null values', () => { - const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; - const alert = getMockData({ - alertTypeId: 'siem.signals', - params: { - author: ['Elastic'], - buildingBlockType: null, - description: - "This rule detects a known command and control pattern in network events. The FIN7 threat group is known to use this command and control technique, while maintaining persistence in their target's network.", - ruleId: '4a4e23cf-78a2-449c-bac3-701924c269d3', - index: ['packetbeat-*'], - falsePositives: [ - "This rule could identify benign domains that are formatted similarly to FIN7's command and control algorithm. Alerts should be investigated by an analyst to assess the validity of the individual observations.", - ], - from: 'now-6m', - immutable: true, - query: - 'event.category:(network OR network_traffic) AND type:(tls OR http) AND network.transport:tcp AND destination.domain:/[a-zA-Z]{4,5}.(pw|us|club|info|site|top)/ AND NOT destination.domain:zoom.us', - language: 'lucene', - license: 'Elastic License', - outputIndex: '.siem-signals-rylandherrick_2-default', - savedId: null, - timelineId: null, - timelineTitle: null, - meta: null, - filters: null, - maxSignals: 100, - riskScore: 73, - riskScoreMapping: [], - ruleNameOverride: null, - severity: 'high', - severityMapping: null, - threat: null, - threatFilters: null, - timestampOverride: null, - to: 'now', - type: 'query', - references: [ - 'https://www.fireeye.com/blog/threat-research/2018/08/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation.html', - ], - note: - 'In the event this rule identifies benign domains in your environment, the `destination.domain` field in the rule can be modified to include those domains. Example: `...AND NOT destination.domain:(zoom.us OR benign.domain1 OR benign.domain2)`.', - version: 1, - exceptionsList: null, - threshold: { - field: null, - value: 5, - }, - }, - }); - expect(migration713(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, + describe('7.13.0', () => { + test('security solution alerts get migrated and remove null values', () => { + const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; + const alert = getMockData({ + alertTypeId: 'siem.signals', params: { author: ['Elastic'], + buildingBlockType: null, description: "This rule detects a known command and control pattern in network events. The FIN7 threat group is known to use this command and control technique, while maintaining persistence in their target's network.", ruleId: '4a4e23cf-78a2-449c-bac3-701924c269d3', @@ -779,12 +671,20 @@ describe('7.13.0', () => { language: 'lucene', license: 'Elastic License', outputIndex: '.siem-signals-rylandherrick_2-default', + savedId: null, + timelineId: null, + timelineTitle: null, + meta: null, + filters: null, maxSignals: 100, riskScore: 73, riskScoreMapping: [], + ruleNameOverride: null, severity: 'high', - severityMapping: [], - threat: [], + severityMapping: null, + threat: null, + threatFilters: null, + timestampOverride: null, to: 'now', type: 'query', references: [ @@ -793,169 +693,193 @@ describe('7.13.0', () => { note: 'In the event this rule identifies benign domains in your environment, the `destination.domain` field in the rule can be modified to include those domains. Example: `...AND NOT destination.domain:(zoom.us OR benign.domain1 OR benign.domain2)`.', version: 1, - exceptionsList: [], + exceptionsList: null, threshold: { - field: [], + field: null, value: 5, - cardinality: [], }, }, - }, - }); - }); + }); - test('non-null values in security solution alerts are not modified', () => { - const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; - const alert = getMockData({ - alertTypeId: 'siem.signals', - params: { - author: ['Elastic'], - buildingBlockType: 'default', - description: - "This rule detects a known command and control pattern in network events. The FIN7 threat group is known to use this command and control technique, while maintaining persistence in their target's network.", - ruleId: '4a4e23cf-78a2-449c-bac3-701924c269d3', - index: ['packetbeat-*'], - falsePositives: [ - "This rule could identify benign domains that are formatted similarly to FIN7's command and control algorithm. Alerts should be investigated by an analyst to assess the validity of the individual observations.", - ], - from: 'now-6m', - immutable: true, - query: - 'event.category:(network OR network_traffic) AND type:(tls OR http) AND network.transport:tcp AND destination.domain:/[a-zA-Z]{4,5}.(pw|us|club|info|site|top)/ AND NOT destination.domain:zoom.us', - language: 'lucene', - license: 'Elastic License', - outputIndex: '.siem-signals-rylandherrick_2-default', - savedId: 'saved-id', - timelineId: 'timeline-id', - timelineTitle: 'timeline-title', - meta: { - field: 'value', - }, - filters: ['filters'], - maxSignals: 100, - riskScore: 73, - riskScoreMapping: ['risk-score-mapping'], - ruleNameOverride: 'field.name', - severity: 'high', - severityMapping: ['severity-mapping'], - threat: [ - { - framework: 'MITRE ATT&CK', - tactic: { - id: 'TA0011', - name: 'Command and Control', - reference: 'https://attack.mitre.org/tactics/TA0011/', - }, - technique: [ - { - id: 'T1483', - name: 'Domain Generation Algorithms', - reference: 'https://attack.mitre.org/techniques/T1483/', - }, + expect(migration713(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + params: { + author: ['Elastic'], + description: + "This rule detects a known command and control pattern in network events. The FIN7 threat group is known to use this command and control technique, while maintaining persistence in their target's network.", + ruleId: '4a4e23cf-78a2-449c-bac3-701924c269d3', + index: ['packetbeat-*'], + falsePositives: [ + "This rule could identify benign domains that are formatted similarly to FIN7's command and control algorithm. Alerts should be investigated by an analyst to assess the validity of the individual observations.", + ], + from: 'now-6m', + immutable: true, + query: + 'event.category:(network OR network_traffic) AND type:(tls OR http) AND network.transport:tcp AND destination.domain:/[a-zA-Z]{4,5}.(pw|us|club|info|site|top)/ AND NOT destination.domain:zoom.us', + language: 'lucene', + license: 'Elastic License', + outputIndex: '.siem-signals-rylandherrick_2-default', + maxSignals: 100, + riskScore: 73, + riskScoreMapping: [], + severity: 'high', + severityMapping: [], + threat: [], + to: 'now', + type: 'query', + references: [ + 'https://www.fireeye.com/blog/threat-research/2018/08/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation.html', ], + note: + 'In the event this rule identifies benign domains in your environment, the `destination.domain` field in the rule can be modified to include those domains. Example: `...AND NOT destination.domain:(zoom.us OR benign.domain1 OR benign.domain2)`.', + version: 1, + exceptionsList: [], + threshold: { + field: [], + value: 5, + cardinality: [], + }, }, - ], - threatFilters: ['threat-filter'], - timestampOverride: 'event.ingested', - to: 'now', - type: 'query', - references: [ - 'https://www.fireeye.com/blog/threat-research/2018/08/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation.html', - ], - note: - 'In the event this rule identifies benign domains in your environment, the `destination.domain` field in the rule can be modified to include those domains. Example: `...AND NOT destination.domain:(zoom.us OR benign.domain1 OR benign.domain2)`.', - version: 1, - exceptionsList: ['exceptions-list'], - }, + }, + }); }); - expect(migration713(alert, migrationContext)).toEqual(alert); - }); - - test('security solution threshold alert with string in threshold.field is migrated to array', () => { - const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; - const alert = getMockData({ - alertTypeId: 'siem.signals', - params: { - threshold: { - field: 'host.id', - value: 5, + test('non-null values in security solution alerts are not modified', () => { + const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; + const alert = getMockData({ + alertTypeId: 'siem.signals', + params: { + author: ['Elastic'], + buildingBlockType: 'default', + description: + "This rule detects a known command and control pattern in network events. The FIN7 threat group is known to use this command and control technique, while maintaining persistence in their target's network.", + ruleId: '4a4e23cf-78a2-449c-bac3-701924c269d3', + index: ['packetbeat-*'], + falsePositives: [ + "This rule could identify benign domains that are formatted similarly to FIN7's command and control algorithm. Alerts should be investigated by an analyst to assess the validity of the individual observations.", + ], + from: 'now-6m', + immutable: true, + query: + 'event.category:(network OR network_traffic) AND type:(tls OR http) AND network.transport:tcp AND destination.domain:/[a-zA-Z]{4,5}.(pw|us|club|info|site|top)/ AND NOT destination.domain:zoom.us', + language: 'lucene', + license: 'Elastic License', + outputIndex: '.siem-signals-rylandherrick_2-default', + savedId: 'saved-id', + timelineId: 'timeline-id', + timelineTitle: 'timeline-title', + meta: { + field: 'value', + }, + filters: ['filters'], + maxSignals: 100, + riskScore: 73, + riskScoreMapping: ['risk-score-mapping'], + ruleNameOverride: 'field.name', + severity: 'high', + severityMapping: ['severity-mapping'], + threat: [ + { + framework: 'MITRE ATT&CK', + tactic: { + id: 'TA0011', + name: 'Command and Control', + reference: 'https://attack.mitre.org/tactics/TA0011/', + }, + technique: [ + { + id: 'T1483', + name: 'Domain Generation Algorithms', + reference: 'https://attack.mitre.org/techniques/T1483/', + }, + ], + }, + ], + threatFilters: ['threat-filter'], + timestampOverride: 'event.ingested', + to: 'now', + type: 'query', + references: [ + 'https://www.fireeye.com/blog/threat-research/2018/08/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation.html', + ], + note: + 'In the event this rule identifies benign domains in your environment, the `destination.domain` field in the rule can be modified to include those domains. Example: `...AND NOT destination.domain:(zoom.us OR benign.domain1 OR benign.domain2)`.', + version: 1, + exceptionsList: ['exceptions-list'], }, - }, + }); + + expect(migration713(alert, migrationContext)).toEqual(alert); }); - expect(migration713(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, + test('security solution threshold alert with string in threshold.field is migrated to array', () => { + const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; + const alert = getMockData({ + alertTypeId: 'siem.signals', params: { threshold: { - field: ['host.id'], + field: 'host.id', value: 5, - cardinality: [], }, - exceptionsList: [], - riskScoreMapping: [], - severityMapping: [], - threat: [], }, - }, - }); - }); + }); - test('security solution threshold alert with empty string in threshold.field is migrated to empty array', () => { - const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; - const alert = getMockData({ - alertTypeId: 'siem.signals', - params: { - threshold: { - field: '', - value: 5, + expect(migration713(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + params: { + threshold: { + field: ['host.id'], + value: 5, + cardinality: [], + }, + exceptionsList: [], + riskScoreMapping: [], + severityMapping: [], + threat: [], + }, }, - }, + }); }); - expect(migration713(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, + test('security solution threshold alert with empty string in threshold.field is migrated to empty array', () => { + const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; + const alert = getMockData({ + alertTypeId: 'siem.signals', params: { threshold: { - field: [], + field: '', value: 5, - cardinality: [], }, - exceptionsList: [], - riskScoreMapping: [], - severityMapping: [], - threat: [], }, - }, - }); - }); + }); - test('security solution threshold alert with array in threshold.field and cardinality is left alone', () => { - const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; - const alert = getMockData({ - alertTypeId: 'siem.signals', - params: { - threshold: { - field: ['host.id'], - value: 5, - cardinality: [ - { - field: 'source.ip', - value: 10, + expect(migration713(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + params: { + threshold: { + field: [], + value: 5, + cardinality: [], }, - ], + exceptionsList: [], + riskScoreMapping: [], + severityMapping: [], + threat: [], + }, }, - }, + }); }); - expect(migration713(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, + test('security solution threshold alert with array in threshold.field and cardinality is left alone', () => { + const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; + const alert = getMockData({ + alertTypeId: 'siem.signals', params: { threshold: { field: ['host.id'], @@ -967,64 +891,191 @@ describe('7.13.0', () => { }, ], }, - exceptionsList: [], - riskScoreMapping: [], - severityMapping: [], - threat: [], }, - }, - }); - }); + }); - test('security solution ML alert with string in machineLearningJobId is converted to an array', () => { - const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; - const alert = getMockData({ - alertTypeId: 'siem.signals', - params: { - anomalyThreshold: 20, - machineLearningJobId: 'my_job_id', - }, + expect(migration713(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + params: { + threshold: { + field: ['host.id'], + value: 5, + cardinality: [ + { + field: 'source.ip', + value: 10, + }, + ], + }, + exceptionsList: [], + riskScoreMapping: [], + severityMapping: [], + threat: [], + }, + }, + }); }); - expect(migration713(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, + test('security solution ML alert with string in machineLearningJobId is converted to an array', () => { + const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; + const alert = getMockData({ + alertTypeId: 'siem.signals', params: { anomalyThreshold: 20, - machineLearningJobId: ['my_job_id'], - exceptionsList: [], - riskScoreMapping: [], - severityMapping: [], - threat: [], + machineLearningJobId: 'my_job_id', }, - }, - }); - }); + }); - test('security solution ML alert with an array in machineLearningJobId is preserved', () => { - const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; - const alert = getMockData({ - alertTypeId: 'siem.signals', - params: { - anomalyThreshold: 20, - machineLearningJobId: ['my_job_id', 'my_other_job_id'], - }, + expect(migration713(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + params: { + anomalyThreshold: 20, + machineLearningJobId: ['my_job_id'], + exceptionsList: [], + riskScoreMapping: [], + severityMapping: [], + threat: [], + }, + }, + }); }); - expect(migration713(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, + test('security solution ML alert with an array in machineLearningJobId is preserved', () => { + const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; + const alert = getMockData({ + alertTypeId: 'siem.signals', params: { anomalyThreshold: 20, machineLearningJobId: ['my_job_id', 'my_other_job_id'], - exceptionsList: [], - riskScoreMapping: [], - severityMapping: [], - threat: [], }, - }, + }); + + expect(migration713(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + params: { + anomalyThreshold: 20, + machineLearningJobId: ['my_job_id', 'my_other_job_id'], + exceptionsList: [], + riskScoreMapping: [], + severityMapping: [], + threat: [], + }, + }, + }); + }); + }); +}); + +describe('handles errors during migrations', () => { + beforeEach(() => { + jest.resetAllMocks(); + encryptedSavedObjectsSetup.createMigration.mockImplementation(() => () => { + throw new Error(`Can't migrate!`); + }); + }); + describe('7.10.0 throws if migration fails', () => { + test('should show the proper exception', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const alert = getMockData({ + consumer: 'alerting', + }); + expect(() => { + migration710(alert, migrationContext); + }).toThrowError(`Can't migrate!`); + expect(migrationContext.log.error).toHaveBeenCalledWith( + `encryptedSavedObject 7.10.0 migration failed for alert ${alert.id} with error: Can't migrate!`, + { + migrations: { + alertDocument: { + ...alert, + attributes: { + ...alert.attributes, + }, + }, + }, + } + ); + }); + }); + + describe('7.11.0 throws if migration fails', () => { + test('should show the proper exception', () => { + const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; + const alert = getMockData({ + consumer: 'alerting', + }); + expect(() => { + migration711(alert, migrationContext); + }).toThrowError(`Can't migrate!`); + expect(migrationContext.log.error).toHaveBeenCalledWith( + `encryptedSavedObject 7.11.0 migration failed for alert ${alert.id} with error: Can't migrate!`, + { + migrations: { + alertDocument: { + ...alert, + attributes: { + ...alert.attributes, + }, + }, + }, + } + ); + }); + }); + + describe('7.11.2 throws if migration fails', () => { + test('should show the proper exception', () => { + const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; + const alert = getMockData({ + consumer: 'alerting', + }); + expect(() => { + migration7112(alert, migrationContext); + }).toThrowError(`Can't migrate!`); + expect(migrationContext.log.error).toHaveBeenCalledWith( + `encryptedSavedObject 7.11.2 migration failed for alert ${alert.id} with error: Can't migrate!`, + { + migrations: { + alertDocument: { + ...alert, + attributes: { + ...alert.attributes, + }, + }, + }, + } + ); + }); + }); + + describe('7.13.0 throws if migration fails', () => { + test('should show the proper exception', () => { + const migration7130 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; + const alert = getMockData({ + consumer: 'alerting', + }); + expect(() => { + migration7130(alert, migrationContext); + }).toThrowError(`Can't migrate!`); + expect(migrationContext.log.error).toHaveBeenCalledWith( + `encryptedSavedObject 7.13.0 migration failed for alert ${alert.id} with error: Can't migrate!`, + { + migrations: { + alertDocument: { + ...alert, + attributes: { + ...alert.attributes, + }, + }, + }, + } + ); }); }); }); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations.ts b/x-pack/plugins/alerting/server/saved_objects/migrations.ts index 8969e3ad0fdef..ae1e9828336cd 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations.ts @@ -29,6 +29,22 @@ type AlertMigration = ( doc: SavedObjectUnsanitizedDoc ) => SavedObjectUnsanitizedDoc; +type IsMigrationNeededPredicate = ( + doc: SavedObjectUnsanitizedDoc +) => doc is SavedObjectUnsanitizedDoc; + +function createEsoMigration( + encryptedSavedObjects: EncryptedSavedObjectsPluginSetup, + isMigrationNeededPredicate: IsMigrationNeededPredicate, + migrationFunc: AlertMigration +) { + return encryptedSavedObjects.createMigration( + isMigrationNeededPredicate, + migrationFunc, + true // shouldMigrateIfDecryptionFails flag that applies the migration to undecrypted document if decryption fails + ); +} + const SUPPORT_INCIDENTS_ACTION_TYPES = ['.servicenow', '.jira', '.resilient']; export const isAnyActionSupportIncidents = (doc: SavedObjectUnsanitizedDoc): boolean => @@ -42,11 +58,10 @@ export const isSecuritySolutionRule = (doc: SavedObjectUnsanitizedDoc) export function getMigrations( encryptedSavedObjects: EncryptedSavedObjectsPluginSetup ): SavedObjectMigrationMap { - const migrationWhenRBACWasIntroduced = encryptedSavedObjects.createMigration( - function shouldBeMigrated(doc): doc is SavedObjectUnsanitizedDoc { - // migrate all documents in 7.10 in order to add the "meta" RBAC field - return true; - }, + const migrationWhenRBACWasIntroduced = createEsoMigration( + encryptedSavedObjects, + // migrate all documents in 7.10 in order to add the "meta" RBAC field + (doc): doc is SavedObjectUnsanitizedDoc => true, pipeMigrations( markAsLegacyAndChangeConsumer, setAlertIdAsDefaultDedupkeyOnPagerDutyActions, @@ -54,21 +69,21 @@ export function getMigrations( ) ); - const migrationAlertUpdatedAtAndNotifyWhen = encryptedSavedObjects.createMigration< - RawAlert, - RawAlert - >( + const migrationAlertUpdatedAtAndNotifyWhen = createEsoMigration( + encryptedSavedObjects, // migrate all documents in 7.11 in order to add the "updatedAt" and "notifyWhen" fields (doc): doc is SavedObjectUnsanitizedDoc => true, pipeMigrations(setAlertUpdatedAtDate, setNotifyWhen) ); - const migrationActions7112 = encryptedSavedObjects.createMigration( + const migrationActions7112 = createEsoMigration( + encryptedSavedObjects, (doc): doc is SavedObjectUnsanitizedDoc => isAnyActionSupportIncidents(doc), pipeMigrations(restructureConnectorsThatSupportIncident) ); - const migrationSecurityRules713 = encryptedSavedObjects.createMigration( + const migrationSecurityRules713 = createEsoMigration( + encryptedSavedObjects, (doc): doc is SavedObjectUnsanitizedDoc => isSecuritySolutionRule(doc), pipeMigrations(removeNullsFromSecurityRules) ); @@ -97,8 +112,8 @@ function executeMigrationWithErrorHandling( }, } ); + throw ex; } - return doc; }; } diff --git a/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts b/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts index 548340fbb6463..9163ab3f4af95 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts @@ -44,6 +44,7 @@ describe('createMigration()', () => { return true; }, (doc) => doc, + false, { type: 'known-type-1', attributesToEncrypt: new Set(), @@ -111,6 +112,329 @@ describe('createMigration()', () => { attributes ); }); + + it('throws error on decryption failure if shouldMigrateIfDecryptionFails is false', () => { + const instantiateServiceWithLegacyType = jest.fn(() => + encryptedSavedObjectsServiceMock.create() + ); + const migrationFunc = jest.fn((doc) => doc); + + const migrationCreator = getCreateMigration( + encryptionSavedObjectService, + instantiateServiceWithLegacyType + ); + const noopMigration = migrationCreator(function ( + doc + ): doc is SavedObjectUnsanitizedDoc { + return true; + }, + migrationFunc); + + const attributes = { + firstAttr: 'first_attr', + }; + + encryptionSavedObjectService.decryptAttributesSync.mockImplementationOnce(() => { + throw new Error('decryption failed!'); + }); + + expect(() => { + noopMigration( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + attributes, + }, + migrationContext + ); + }).toThrowError(`decryption failed!`); + + expect(encryptionSavedObjectService.decryptAttributesSync).toHaveBeenCalledWith( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + }, + attributes, + { convertToMultiNamespaceType: false } + ); + + expect(migrationFunc).not.toHaveBeenCalled(); + expect(encryptionSavedObjectService.encryptAttributesSync).not.toHaveBeenCalled(); + }); + + it('runs migration function on decryption failure if shouldMigrateIfDecryptionFails is true', () => { + const instantiateServiceWithLegacyType = jest.fn(() => + encryptedSavedObjectsServiceMock.create() + ); + const migrationFunc = jest.fn((doc) => doc); + + const migrationCreator = getCreateMigration( + encryptionSavedObjectService, + instantiateServiceWithLegacyType + ); + const noopMigration = migrationCreator( + function (doc): doc is SavedObjectUnsanitizedDoc { + return true; + }, + migrationFunc, + true + ); + + const attributes = { + firstAttr: 'first_attr', + }; + + encryptionSavedObjectService.decryptAttributesSync.mockImplementationOnce(() => { + throw new Error('decryption failed!'); + }); + + noopMigration( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + attributes, + }, + migrationContext + ); + + expect(encryptionSavedObjectService.decryptAttributesSync).toHaveBeenCalledWith( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + }, + attributes, + { convertToMultiNamespaceType: false } + ); + + expect(migrationFunc).toHaveBeenCalled(); + expect(encryptionSavedObjectService.encryptAttributesSync).not.toHaveBeenCalled(); + }); + + it('throws error on migration failure', () => { + const instantiateServiceWithLegacyType = jest.fn(() => + encryptedSavedObjectsServiceMock.create() + ); + const migrationFunc = jest.fn((doc) => { + throw new Error('migration failed!'); + }); + + const migrationCreator = getCreateMigration( + encryptionSavedObjectService, + instantiateServiceWithLegacyType + ); + const noopMigration = migrationCreator(function ( + doc + ): doc is SavedObjectUnsanitizedDoc { + return true; + }, + migrationFunc); + + const attributes = { + firstAttr: 'first_attr', + }; + + encryptionSavedObjectService.decryptAttributesSync.mockReturnValueOnce(attributes); + + expect(() => { + noopMigration( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + attributes, + }, + migrationContext + ); + }).toThrowError(`migration failed!`); + + expect(encryptionSavedObjectService.decryptAttributesSync).toHaveBeenCalledWith( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + }, + attributes, + { convertToMultiNamespaceType: false } + ); + + expect(migrationFunc).toHaveBeenCalled(); + expect(encryptionSavedObjectService.encryptAttributesSync).not.toHaveBeenCalled(); + }); + + it('throws error on migration failure even if shouldMigrateIfDecryptionFails is true', () => { + const instantiateServiceWithLegacyType = jest.fn(() => + encryptedSavedObjectsServiceMock.create() + ); + const migrationFunc = jest.fn((doc) => { + throw new Error('migration failed!'); + }); + + const migrationCreator = getCreateMigration( + encryptionSavedObjectService, + instantiateServiceWithLegacyType + ); + const noopMigration = migrationCreator( + function (doc): doc is SavedObjectUnsanitizedDoc { + return true; + }, + migrationFunc, + true + ); + + const attributes = { + firstAttr: 'first_attr', + }; + + encryptionSavedObjectService.decryptAttributesSync.mockReturnValueOnce(attributes); + + expect(() => { + noopMigration( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + attributes, + }, + migrationContext + ); + }).toThrowError(`migration failed!`); + + expect(encryptionSavedObjectService.decryptAttributesSync).toHaveBeenCalledWith( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + }, + attributes, + { convertToMultiNamespaceType: false } + ); + + expect(migrationFunc).toHaveBeenCalled(); + expect(encryptionSavedObjectService.encryptAttributesSync).not.toHaveBeenCalled(); + }); + + it('throws error on encryption failure', () => { + const instantiateServiceWithLegacyType = jest.fn(() => + encryptedSavedObjectsServiceMock.create() + ); + const migrationFunc = jest.fn((doc) => doc); + + const migrationCreator = getCreateMigration( + encryptionSavedObjectService, + instantiateServiceWithLegacyType + ); + const noopMigration = migrationCreator(function ( + doc + ): doc is SavedObjectUnsanitizedDoc { + return true; + }, + migrationFunc); + + const attributes = { + firstAttr: 'first_attr', + }; + + encryptionSavedObjectService.decryptAttributesSync.mockReturnValueOnce(attributes); + encryptionSavedObjectService.encryptAttributesSync.mockImplementationOnce(() => { + throw new Error('encryption failed!'); + }); + + expect(() => { + noopMigration( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + attributes, + }, + migrationContext + ); + }).toThrowError(`encryption failed!`); + + expect(encryptionSavedObjectService.decryptAttributesSync).toHaveBeenCalledWith( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + }, + attributes, + { convertToMultiNamespaceType: false } + ); + + expect(migrationFunc).toHaveBeenCalled(); + expect(encryptionSavedObjectService.encryptAttributesSync).toHaveBeenCalledWith( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + }, + attributes + ); + }); + + it('throws error on encryption failure even if shouldMigrateIfDecryptionFails is true', () => { + const instantiateServiceWithLegacyType = jest.fn(() => + encryptedSavedObjectsServiceMock.create() + ); + const migrationFunc = jest.fn((doc) => doc); + + const migrationCreator = getCreateMigration( + encryptionSavedObjectService, + instantiateServiceWithLegacyType + ); + const noopMigration = migrationCreator( + function (doc): doc is SavedObjectUnsanitizedDoc { + return true; + }, + migrationFunc, + true + ); + + const attributes = { + firstAttr: 'first_attr', + }; + + encryptionSavedObjectService.decryptAttributesSync.mockReturnValueOnce(attributes); + encryptionSavedObjectService.encryptAttributesSync.mockImplementationOnce(() => { + throw new Error('encryption failed!'); + }); + + expect(() => { + noopMigration( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + attributes, + }, + migrationContext + ); + }).toThrowError(`encryption failed!`); + + expect(encryptionSavedObjectService.decryptAttributesSync).toHaveBeenCalledWith( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + }, + attributes, + { convertToMultiNamespaceType: false } + ); + + expect(migrationFunc).toHaveBeenCalled(); + expect(encryptionSavedObjectService.encryptAttributesSync).toHaveBeenCalledWith( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + }, + attributes + ); + }); }); describe('migration of a single legacy type', () => { @@ -127,6 +451,7 @@ describe('createMigration()', () => { return true; }, (doc) => doc, + false, inputType ); @@ -274,6 +599,7 @@ describe('createMigration()', () => { }, ...doc, }), + false, inputType, migrationType ); diff --git a/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts b/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts index beace2b17fe08..54ab27c3d198a 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts @@ -31,6 +31,7 @@ export type CreateEncryptedSavedObjectsMigrationFn = < >( isMigrationNeededPredicate: IsMigrationNeededPredicate, migration: SavedObjectMigrationFn, + shouldMigrateIfDecryptionFails?: boolean, inputType?: EncryptedSavedObjectTypeRegistration, migratedType?: EncryptedSavedObjectTypeRegistration ) => SavedObjectOptionalMigrationFn; @@ -43,6 +44,7 @@ export const getCreateMigration = ( ): CreateEncryptedSavedObjectsMigrationFn => ( isMigrationNeededPredicate, migration, + shouldMigrateIfDecryptionFails, inputType, migratedType ) => { @@ -79,21 +81,40 @@ export const getCreateMigration = ( const decryptDescriptor = { id, type, namespace: decryptDescriptorNamespace }; const encryptDescriptor = { id, type, namespace: encryptedDoc.namespace }; + let shouldEncrypt = true; + // decrypt the attributes using the input type definition - // then migrate the document - // then encrypt the attributes using the migration type definition - return mapAttributes( - migration( - mapAttributes(encryptedDoc, (inputAttributes) => - inputService.decryptAttributesSync(decryptDescriptor, inputAttributes, { + // if an error occurs during decryption, use the shouldMigrateIfDecryptionFails flag + // to determine whether to throw the error or return the un-decrypted document for migration + // if we are migrating the un-decrypted document, set `shouldEncrypt` to false to avoid encrypting + // an un-decrypted document + const documentToMigrate = mapAttributes(encryptedDoc, (inputAttributes) => { + try { + const decryptedDoc = inputService.decryptAttributesSync( + decryptDescriptor, + inputAttributes, + { convertToMultiNamespaceType, - }) - ), - context - ), - (migratedAttributes) => - migratedService.encryptAttributesSync(encryptDescriptor, migratedAttributes) - ); + } + ); + shouldEncrypt = true; + return decryptedDoc; + } catch (err) { + if (!shouldMigrateIfDecryptionFails) { + throw err; + } + shouldEncrypt = false; + return encryptedDoc.attributes; + } + }); + + // migrate the document + // then if `shouldEncrypt` is true, encrypt the attributes using the migration type definition + return mapAttributes(migration(documentToMigrate, context), (migratedAttributes) => { + return shouldEncrypt + ? migratedService.encryptAttributesSync(encryptDescriptor, migratedAttributes) + : migratedAttributes; + }); }; }; From 313ac7b8864c188dabddd40cd25393a23228080e Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Fri, 16 Jul 2021 10:46:47 -0400 Subject: [PATCH 02/17] Fixing types --- .../fixtures/api_consumer_plugin/server/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/test/encrypted_saved_objects_api_integration/fixtures/api_consumer_plugin/server/index.ts b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/api_consumer_plugin/server/index.ts index 96a0a3b2fa427..ceb5ef0dbbacc 100644 --- a/x-pack/test/encrypted_saved_objects_api_integration/fixtures/api_consumer_plugin/server/index.ts +++ b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/api_consumer_plugin/server/index.ts @@ -168,6 +168,7 @@ function defineTypeWithMigration(core: CoreSetup, deps: PluginsSet }, }; }, + false, // type hasn't changed as the field we're updating is not an encrypted one typePriorTo790, typePriorTo790 @@ -193,6 +194,7 @@ function defineTypeWithMigration(core: CoreSetup, deps: PluginsSet }, }; }, + false, typePriorTo790 ), }, From 95a0d7c9a142272cce421995e0948eed2a8ad0f5 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Fri, 16 Jul 2021 14:09:09 -0400 Subject: [PATCH 03/17] Updating readme and adding warning message --- x-pack/plugins/encrypted_saved_objects/README.md | 2 ++ .../encrypted_saved_objects/server/create_migration.ts | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/x-pack/plugins/encrypted_saved_objects/README.md b/x-pack/plugins/encrypted_saved_objects/README.md index 99ebf771126d5..4c96ad3d2616f 100644 --- a/x-pack/plugins/encrypted_saved_objects/README.md +++ b/x-pack/plugins/encrypted_saved_objects/README.md @@ -111,6 +111,8 @@ The `createMigration` function takes four arguments: |---|---|---| |isMigrationNeededPredicate|A predicate which is called for each document, prior to being decrypted, which confirms whether a document requires migration or not. This predicate is important as the decryption step is costly and we would rather not decrypt and re-encrypt a document if we can avoid it.|function| |migration|A migration function which will migrate each decrypted document from the old shape to the new one.|function| +|shouldMigrateIfDecryptionFails|Optional. A boolean flag which indicates whether to proceed with migration if a document fails to decrypt. If this is not set or +if it is set to `false`, decryption errors will be thrown. If set to `true`, a warning will be logged and the migration function will be applied to the un-decrypted document. Set this to `true` if you don't want decryption failures to block Kibana upgrades. |boolean| |inputType|Optional. An `EncryptedSavedObjectTypeRegistration` which describes the ESOType of the input (the document prior to migration). If this type isn't provided, we'll assume the input doc follows the registered type. |object| |migratedType| Optional. An `EncryptedSavedObjectTypeRegistration` which describes the ESOType of the output (the document after migration). If this type isn't provided, we'll assume the migrated doc follows the registered type.|object| diff --git a/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts b/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts index 54ab27c3d198a..1d0f023da0dfe 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts @@ -103,6 +103,10 @@ export const getCreateMigration = ( if (!shouldMigrateIfDecryptionFails) { throw err; } + + context.log.warning( + `decryption failed for encryptedSavedObject ${encryptedDoc.id} of type ${encryptedDoc.type} with error: ${err.message}. Migration will be applied to the un-decrypted document but this may cause decryption errors later on.` + ); shouldEncrypt = false; return encryptedDoc.attributes; } From c392ce6513494dc2406b4a0c616ce6e528de7f7a Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Tue, 20 Jul 2021 12:01:49 -0400 Subject: [PATCH 04/17] Updating README --- x-pack/plugins/encrypted_saved_objects/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugins/encrypted_saved_objects/README.md b/x-pack/plugins/encrypted_saved_objects/README.md index 4c96ad3d2616f..04bba1a4b5276 100644 --- a/x-pack/plugins/encrypted_saved_objects/README.md +++ b/x-pack/plugins/encrypted_saved_objects/README.md @@ -111,8 +111,7 @@ The `createMigration` function takes four arguments: |---|---|---| |isMigrationNeededPredicate|A predicate which is called for each document, prior to being decrypted, which confirms whether a document requires migration or not. This predicate is important as the decryption step is costly and we would rather not decrypt and re-encrypt a document if we can avoid it.|function| |migration|A migration function which will migrate each decrypted document from the old shape to the new one.|function| -|shouldMigrateIfDecryptionFails|Optional. A boolean flag which indicates whether to proceed with migration if a document fails to decrypt. If this is not set or -if it is set to `false`, decryption errors will be thrown. If set to `true`, a warning will be logged and the migration function will be applied to the un-decrypted document. Set this to `true` if you don't want decryption failures to block Kibana upgrades. |boolean| +|shouldMigrateIfDecryptionFails|Optional. A boolean flag which indicates whether to proceed with migration if a document fails to decrypt. If this is not set or if it is set to `false`, decryption errors will be thrown. If set to `true`, a warning will be logged and the migration function will be applied to the original encrypted document. Set this to `true` if you don't want decryption failures to block Kibana upgrades. |boolean| |inputType|Optional. An `EncryptedSavedObjectTypeRegistration` which describes the ESOType of the input (the document prior to migration). If this type isn't provided, we'll assume the input doc follows the registered type. |object| |migratedType| Optional. An `EncryptedSavedObjectTypeRegistration` which describes the ESOType of the output (the document after migration). If this type isn't provided, we'll assume the migrated doc follows the registered type.|object| From f2475b594beb95e0685ed0cccd957cc204c47cc7 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Tue, 20 Jul 2021 12:58:23 -0400 Subject: [PATCH 05/17] PR fixes --- .../server/create_migration.test.ts | 60 ++++++++++++++++++- .../server/create_migration.ts | 17 +++--- .../server/crypto/index.ts | 2 +- 3 files changed, 68 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts b/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts index 9163ab3f4af95..f7cf7f45c6c22 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts @@ -9,6 +9,7 @@ import type { SavedObjectUnsanitizedDoc } from 'src/core/server'; import { migrationMocks } from 'src/core/server/mocks'; import { getCreateMigration } from './create_migration'; +import { EncryptionError, EncryptionErrorOperation } from './crypto'; import { encryptedSavedObjectsServiceMock } from './crypto/index.mock'; afterEach(() => { @@ -164,7 +165,7 @@ describe('createMigration()', () => { expect(encryptionSavedObjectService.encryptAttributesSync).not.toHaveBeenCalled(); }); - it('runs migration function on decryption failure if shouldMigrateIfDecryptionFails is true', () => { + it('throws error on decryption failure if shouldMigrateIfDecryptionFails is true but error is not encryption error', () => { const instantiateServiceWithLegacyType = jest.fn(() => encryptedSavedObjectsServiceMock.create() ); @@ -190,6 +191,63 @@ describe('createMigration()', () => { throw new Error('decryption failed!'); }); + expect(() => { + noopMigration( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + attributes, + }, + migrationContext + ); + }).toThrowError(`decryption failed!`); + + expect(encryptionSavedObjectService.decryptAttributesSync).toHaveBeenCalledWith( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + }, + attributes, + { convertToMultiNamespaceType: false } + ); + + expect(migrationFunc).not.toHaveBeenCalled(); + expect(encryptionSavedObjectService.encryptAttributesSync).not.toHaveBeenCalled(); + }); + + it('runs migration function on decryption failure if shouldMigrateIfDecryptionFails is true and error is encryption error', () => { + const instantiateServiceWithLegacyType = jest.fn(() => + encryptedSavedObjectsServiceMock.create() + ); + const migrationFunc = jest.fn((doc) => doc); + + const migrationCreator = getCreateMigration( + encryptionSavedObjectService, + instantiateServiceWithLegacyType + ); + const noopMigration = migrationCreator( + function (doc): doc is SavedObjectUnsanitizedDoc { + return true; + }, + migrationFunc, + true + ); + + const attributes = { + firstAttr: 'first_attr', + }; + + encryptionSavedObjectService.decryptAttributesSync.mockImplementationOnce(() => { + throw new EncryptionError( + `Unable to decrypt attribute "'attribute'"`, + 'attribute', + EncryptionErrorOperation.Decryption, + new Error('decryption failed') + ); + }); + noopMigration( { id: '123', diff --git a/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts b/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts index 1d0f023da0dfe..54dcf5c21704e 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts @@ -11,6 +11,7 @@ import type { SavedObjectUnsanitizedDoc, } from 'src/core/server'; +import { EncryptionError } from './crypto'; import type { EncryptedSavedObjectsService, EncryptedSavedObjectTypeRegistration } from './crypto'; import { normalizeNamespace } from './saved_objects'; @@ -90,25 +91,23 @@ export const getCreateMigration = ( // an un-decrypted document const documentToMigrate = mapAttributes(encryptedDoc, (inputAttributes) => { try { - const decryptedDoc = inputService.decryptAttributesSync( + const decryptedAttributes = inputService.decryptAttributesSync( decryptDescriptor, inputAttributes, - { - convertToMultiNamespaceType, - } + { convertToMultiNamespaceType } ); shouldEncrypt = true; - return decryptedDoc; + return decryptedAttributes; } catch (err) { - if (!shouldMigrateIfDecryptionFails) { + if (!shouldMigrateIfDecryptionFails || !(err instanceof EncryptionError)) { throw err; } - context.log.warning( - `decryption failed for encryptedSavedObject ${encryptedDoc.id} of type ${encryptedDoc.type} with error: ${err.message}. Migration will be applied to the un-decrypted document but this may cause decryption errors later on.` + context.log.warn( + `Decryption failed for encrypted Saved Object "${encryptedDoc.id}" of type "${encryptedDoc.type}" with error: ${err.message}. Migration will be applied to the original encrypted document but this may cause decryption errors later on.` ); shouldEncrypt = false; - return encryptedDoc.attributes; + return inputAttributes; } }); diff --git a/x-pack/plugins/encrypted_saved_objects/server/crypto/index.ts b/x-pack/plugins/encrypted_saved_objects/server/crypto/index.ts index 3d838d7cba69a..31086f56c3b86 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/crypto/index.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/crypto/index.ts @@ -11,6 +11,6 @@ export { descriptorToArray, SavedObjectDescriptor, } from './encrypted_saved_objects_service'; -export { EncryptionError } from './encryption_error'; +export { EncryptionError, EncryptionErrorOperation } from './encryption_error'; export { EncryptedSavedObjectAttributesDefinition } from './encrypted_saved_object_type_definition'; export { EncryptionKeyRotationService } from './encryption_key_rotation_service'; From 716e7dcf06bb70bb2848fba63534afc697819bc5 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Tue, 20 Jul 2021 13:43:29 -0400 Subject: [PATCH 06/17] collapsing args to create migration fn --- .../server/saved_objects/migrations.test.ts | 2 +- .../server/saved_objects/migrations.ts | 15 +- .../server/saved_objects/migrations.test.ts | 2 +- .../server/saved_objects/migrations.ts | 15 +- .../plugins/encrypted_saved_objects/README.md | 30 ++-- .../server/create_migration.test.ts | 129 +++++++++--------- .../server/create_migration.ts | 35 +++-- .../encrypted_saved_objects/server/index.ts | 1 + .../saved_objects/migrations/to_v7_10_0.ts | 12 +- .../api_consumer_plugin/server/index.ts | 28 ++-- 10 files changed, 137 insertions(+), 132 deletions(-) diff --git a/x-pack/plugins/actions/server/saved_objects/migrations.test.ts b/x-pack/plugins/actions/server/saved_objects/migrations.test.ts index 17ba57e602723..db7d7bb6505b4 100644 --- a/x-pack/plugins/actions/server/saved_objects/migrations.test.ts +++ b/x-pack/plugins/actions/server/saved_objects/migrations.test.ts @@ -18,7 +18,7 @@ const encryptedSavedObjectsSetup = encryptedSavedObjectsMock.createSetup(); describe('successful migrations', () => { beforeEach(() => { jest.resetAllMocks(); - encryptedSavedObjectsSetup.createMigration.mockImplementation((_, migration) => migration); + encryptedSavedObjectsSetup.createMigration.mockImplementation(({ migration }) => migration); }); describe('7.10.0', () => { diff --git a/x-pack/plugins/actions/server/saved_objects/migrations.ts b/x-pack/plugins/actions/server/saved_objects/migrations.ts index e672dd4921e5d..3006f4aa293a7 100644 --- a/x-pack/plugins/actions/server/saved_objects/migrations.ts +++ b/x-pack/plugins/actions/server/saved_objects/migrations.ts @@ -14,6 +14,7 @@ import { } from '../../../../../src/core/server'; import { RawAction } from '../types'; import { EncryptedSavedObjectsPluginSetup } from '../../../encrypted_saved_objects/server'; +import type { IsMigrationNeededPredicate } from '../../../encrypted_saved_objects/server'; interface ActionsLogMeta extends LogMeta { migrations: { actionDocument: SavedObjectUnsanitizedDoc }; @@ -23,20 +24,16 @@ type ActionMigration = ( doc: SavedObjectUnsanitizedDoc ) => SavedObjectUnsanitizedDoc; -type IsMigrationNeededPredicate = ( - doc: SavedObjectUnsanitizedDoc -) => doc is SavedObjectUnsanitizedDoc; - function createEsoMigration( encryptedSavedObjects: EncryptedSavedObjectsPluginSetup, - isMigrationNeededPredicate: IsMigrationNeededPredicate, + isMigrationNeededPredicate: IsMigrationNeededPredicate, migrationFunc: ActionMigration ) { - return encryptedSavedObjects.createMigration( + return encryptedSavedObjects.createMigration({ isMigrationNeededPredicate, - migrationFunc, - true // shouldMigrateIfDecryptionFails flag that applies the migration to undecrypted document if decryption fails - ); + migration: migrationFunc, + shouldMigrateIfDecryptionFails: true, // shouldMigrateIfDecryptionFails flag that applies the migration to undecrypted document if decryption fails + }); } export function getMigrations( diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts b/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts index 5c4af3dfcfd16..9403c0c28c153 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts @@ -18,7 +18,7 @@ const encryptedSavedObjectsSetup = encryptedSavedObjectsMock.createSetup(); describe('successful migrations', () => { beforeEach(() => { jest.resetAllMocks(); - encryptedSavedObjectsSetup.createMigration.mockImplementation((_, migration) => migration); + encryptedSavedObjectsSetup.createMigration.mockImplementation(({ migration }) => migration); }); describe('7.10.0', () => { test('marks alerts as legacy', () => { diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations.ts b/x-pack/plugins/alerting/server/saved_objects/migrations.ts index ae1e9828336cd..9f6adeb27083a 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations.ts @@ -16,6 +16,7 @@ import { } from '../../../../../src/core/server'; import { RawAlert, RawAlertAction } from '../types'; import { EncryptedSavedObjectsPluginSetup } from '../../../encrypted_saved_objects/server'; +import type { IsMigrationNeededPredicate } from '../../../encrypted_saved_objects/server'; const SIEM_APP_ID = 'securitySolution'; const SIEM_SERVER_APP_ID = 'siem'; @@ -29,20 +30,16 @@ type AlertMigration = ( doc: SavedObjectUnsanitizedDoc ) => SavedObjectUnsanitizedDoc; -type IsMigrationNeededPredicate = ( - doc: SavedObjectUnsanitizedDoc -) => doc is SavedObjectUnsanitizedDoc; - function createEsoMigration( encryptedSavedObjects: EncryptedSavedObjectsPluginSetup, - isMigrationNeededPredicate: IsMigrationNeededPredicate, + isMigrationNeededPredicate: IsMigrationNeededPredicate, migrationFunc: AlertMigration ) { - return encryptedSavedObjects.createMigration( + return encryptedSavedObjects.createMigration({ isMigrationNeededPredicate, - migrationFunc, - true // shouldMigrateIfDecryptionFails flag that applies the migration to undecrypted document if decryption fails - ); + migration: migrationFunc, + shouldMigrateIfDecryptionFails: true, // shouldMigrateIfDecryptionFails flag that applies the migration to undecrypted document if decryption fails + }); } const SUPPORT_INCIDENTS_ACTION_TYPES = ['.servicenow', '.jira', '.resilient']; diff --git a/x-pack/plugins/encrypted_saved_objects/README.md b/x-pack/plugins/encrypted_saved_objects/README.md index 04bba1a4b5276..97e1ea5b657b3 100644 --- a/x-pack/plugins/encrypted_saved_objects/README.md +++ b/x-pack/plugins/encrypted_saved_objects/README.md @@ -124,11 +124,11 @@ encryptedSavedObjects.registerType({ attributesToExcludeFromAAD: new Set(['mutedInstanceIds', 'updatedBy']), }); -const migration790 = encryptedSavedObjects.createMigration( - function shouldBeMigrated(doc): doc is SavedObjectUnsanitizedDoc { +const migration790 = encryptedSavedObjects.createMigration({ + isMigrationNeededPredicate: function shouldBeMigrated(doc): doc is SavedObjectUnsanitizedDoc { return doc.consumer === 'alerting' || doc.consumer === undefined; }, - (doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc => { + migration: (doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc => { const { attributes: { consumer }, } = doc; @@ -140,7 +140,7 @@ const migration790 = encryptedSavedObjects.createMigration( }, }; } -); +}); ``` In the above example you can see thwe following: @@ -175,11 +175,11 @@ encryptedSavedObjects.registerType({ attributesToExcludeFromAAD: new Set(['mutedInstanceIds', 'updatedBy']), }); -const migration790 = encryptedSavedObjects.createMigration( - function shouldBeMigrated(doc): doc is SavedObjectUnsanitizedDoc { +const migration790 = encryptedSavedObjects.createMigration({ + isMigrationNeededPredicate: function shouldBeMigrated(doc): doc is SavedObjectUnsanitizedDoc { return doc.consumer === 'alerting' || doc.consumer === undefined; }, - (doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc => { + migration: (doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc => { const { attributes: { legacyEncryptedField, ...attributes }, } = doc; @@ -190,12 +190,12 @@ const migration790 = encryptedSavedObjects.createMigration( }, }; }, - { + inputType: { type: 'alert', attributesToEncrypt: new Set(['apiKey', 'legacyEncryptedField']), attributesToExcludeFromAAD: new Set(['mutedInstanceIds', 'updatedBy']), } -); +}); ``` As you can see in this example we provide a legacy type which describes the _input_ which needs to be decrypted. @@ -210,26 +210,26 @@ encryptedSavedObjects.registerType({ attributesToExcludeFromAAD: new Set(['mutedInstanceIds', 'updatedBy']), }); -const migration780 = encryptedSavedObjects.createMigration( - function shouldBeMigrated(doc): doc is SavedObjectUnsanitizedDoc { +const migration780 = encryptedSavedObjects.createMigration({ + isMigrationNeededPredicate: function shouldBeMigrated(doc): doc is SavedObjectUnsanitizedDoc { // ... }, - (doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc => { + migration: (doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc => { // ... }, // legacy input type - { + inputType: { type: 'alert', attributesToEncrypt: new Set(['apiKey', 'legacyEncryptedField']), attributesToExcludeFromAAD: new Set(['mutedInstanceIds', 'updatedBy']), }, // legacy migration type - { + migratedType: { type: 'alert', attributesToEncrypt: new Set(['apiKey', 'legacyEncryptedField']), attributesToExcludeFromAAD: new Set(['mutedInstanceIds', 'updatedBy', 'legacyEncryptedField']), } -); +}); ``` ## Testing diff --git a/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts b/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts index f7cf7f45c6c22..2f476ddac7634 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts @@ -40,21 +40,20 @@ describe('createMigration()', () => { encryptedSavedObjectsServiceMock.create() ); expect(() => - migrationCreator( - function (doc): doc is SavedObjectUnsanitizedDoc { + migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { return true; }, - (doc) => doc, - false, - { + migration: (doc) => doc, + inputType: { type: 'known-type-1', attributesToEncrypt: new Set(), }, - { + migratedType: { type: 'known-type-2', attributesToEncrypt: new Set(), - } - ) + }, + }) ).toThrowErrorMatchingInlineSnapshot( `"An Invalid Encrypted Saved Objects migration is trying to migrate across types (\\"known-type-1\\" => \\"known-type-2\\"), which isn't permitted"` ); @@ -70,12 +69,12 @@ describe('createMigration()', () => { encryptionSavedObjectService, instantiateServiceWithLegacyType ); - const noopMigration = migrationCreator( - function (doc): doc is SavedObjectUnsanitizedDoc { + const noopMigration = migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { return true; }, - (doc) => doc - ); + migration: (doc) => doc, + }); const attributes = { firstAttr: 'first_attr', @@ -124,12 +123,12 @@ describe('createMigration()', () => { encryptionSavedObjectService, instantiateServiceWithLegacyType ); - const noopMigration = migrationCreator(function ( - doc - ): doc is SavedObjectUnsanitizedDoc { - return true; - }, - migrationFunc); + const noopMigration = migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { + return true; + }, + migration: migrationFunc, + }); const attributes = { firstAttr: 'first_attr', @@ -175,13 +174,13 @@ describe('createMigration()', () => { encryptionSavedObjectService, instantiateServiceWithLegacyType ); - const noopMigration = migrationCreator( - function (doc): doc is SavedObjectUnsanitizedDoc { + const noopMigration = migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { return true; }, - migrationFunc, - true - ); + migration: migrationFunc, + shouldMigrateIfDecryptionFails: true, + }); const attributes = { firstAttr: 'first_attr', @@ -227,13 +226,13 @@ describe('createMigration()', () => { encryptionSavedObjectService, instantiateServiceWithLegacyType ); - const noopMigration = migrationCreator( - function (doc): doc is SavedObjectUnsanitizedDoc { + const noopMigration = migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { return true; }, - migrationFunc, - true - ); + migration: migrationFunc, + shouldMigrateIfDecryptionFails: true, + }); const attributes = { firstAttr: 'first_attr', @@ -284,12 +283,12 @@ describe('createMigration()', () => { encryptionSavedObjectService, instantiateServiceWithLegacyType ); - const noopMigration = migrationCreator(function ( - doc - ): doc is SavedObjectUnsanitizedDoc { - return true; - }, - migrationFunc); + const noopMigration = migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { + return true; + }, + migration: migrationFunc, + }); const attributes = { firstAttr: 'first_attr', @@ -335,13 +334,13 @@ describe('createMigration()', () => { encryptionSavedObjectService, instantiateServiceWithLegacyType ); - const noopMigration = migrationCreator( - function (doc): doc is SavedObjectUnsanitizedDoc { + const noopMigration = migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { return true; }, - migrationFunc, - true - ); + migration: migrationFunc, + shouldMigrateIfDecryptionFails: true, + }); const attributes = { firstAttr: 'first_attr', @@ -385,12 +384,12 @@ describe('createMigration()', () => { encryptionSavedObjectService, instantiateServiceWithLegacyType ); - const noopMigration = migrationCreator(function ( - doc - ): doc is SavedObjectUnsanitizedDoc { - return true; - }, - migrationFunc); + const noopMigration = migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { + return true; + }, + migration: migrationFunc, + }); const attributes = { firstAttr: 'first_attr', @@ -444,13 +443,13 @@ describe('createMigration()', () => { encryptionSavedObjectService, instantiateServiceWithLegacyType ); - const noopMigration = migrationCreator( - function (doc): doc is SavedObjectUnsanitizedDoc { + const noopMigration = migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { return true; }, - migrationFunc, - true - ); + migration: migrationFunc, + shouldMigrateIfDecryptionFails: true, + }); const attributes = { firstAttr: 'first_attr', @@ -504,14 +503,13 @@ describe('createMigration()', () => { encryptionSavedObjectService, instantiateServiceWithLegacyType ); - const noopMigration = migrationCreator( - function (doc): doc is SavedObjectUnsanitizedDoc { + const noopMigration = migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { return true; }, - (doc) => doc, - false, - inputType - ); + migration: (doc) => doc, + inputType, + }); const attributes = { firstAttr: 'first_attr', @@ -566,12 +564,12 @@ describe('createMigration()', () => { encryptionSavedObjectService, instantiateServiceWithLegacyType ); - const noopMigration = migrationCreator( - function (doc): doc is SavedObjectUnsanitizedDoc { + const noopMigration = migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { return true; }, - (doc) => doc - ); + migration: (doc) => doc, + }); const attributes = { firstAttr: 'first_attr', @@ -640,15 +638,15 @@ describe('createMigration()', () => { encryptionSavedObjectService, instantiateServiceWithLegacyType ); - return migrationCreator( - function (doc): doc is SavedObjectUnsanitizedDoc { + return migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { // migrate doc that have the second field return ( typeof (doc as SavedObjectUnsanitizedDoc).attributes.nonEncryptedAttr === 'string' ); }, - ({ attributes: { firstAttr, nonEncryptedAttr }, ...doc }) => ({ + migration: ({ attributes: { firstAttr, nonEncryptedAttr }, ...doc }) => ({ attributes: { // modify an encrypted field firstAttr: `~~${firstAttr}~~`, @@ -657,10 +655,9 @@ describe('createMigration()', () => { }, ...doc, }), - false, inputType, - migrationType - ); + migratedType: migrationType, + }); } it('doesnt decrypt saved objects that dont need to be migrated', async () => { diff --git a/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts b/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts index 54dcf5c21704e..0f13a49810c60 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts @@ -20,21 +20,28 @@ type SavedObjectOptionalMigrationFn = ( context: SavedObjectMigrationContext ) => SavedObjectUnsanitizedDoc; -type IsMigrationNeededPredicate = ( +export type IsMigrationNeededPredicate = ( encryptedDoc: | SavedObjectUnsanitizedDoc | SavedObjectUnsanitizedDoc ) => encryptedDoc is SavedObjectUnsanitizedDoc; +export interface CreateEncryptedSavedObjectsMigrationFnOpts< + InputAttributes = unknown, + MigratedAttributes = InputAttributes +> { + isMigrationNeededPredicate: IsMigrationNeededPredicate; + migration: SavedObjectMigrationFn; + shouldMigrateIfDecryptionFails?: boolean; + inputType?: EncryptedSavedObjectTypeRegistration; + migratedType?: EncryptedSavedObjectTypeRegistration; +} + export type CreateEncryptedSavedObjectsMigrationFn = < InputAttributes = unknown, MigratedAttributes = InputAttributes >( - isMigrationNeededPredicate: IsMigrationNeededPredicate, - migration: SavedObjectMigrationFn, - shouldMigrateIfDecryptionFails?: boolean, - inputType?: EncryptedSavedObjectTypeRegistration, - migratedType?: EncryptedSavedObjectTypeRegistration + opts: CreateEncryptedSavedObjectsMigrationFnOpts ) => SavedObjectOptionalMigrationFn; export const getCreateMigration = ( @@ -42,13 +49,15 @@ export const getCreateMigration = ( instantiateServiceWithLegacyType: ( typeRegistration: EncryptedSavedObjectTypeRegistration ) => EncryptedSavedObjectsService -): CreateEncryptedSavedObjectsMigrationFn => ( - isMigrationNeededPredicate, - migration, - shouldMigrateIfDecryptionFails, - inputType, - migratedType -) => { +): CreateEncryptedSavedObjectsMigrationFn => (opts) => { + const { + isMigrationNeededPredicate, + migration, + shouldMigrateIfDecryptionFails, + inputType, + migratedType, + } = opts; + if (inputType && migratedType && inputType.type !== migratedType.type) { throw new Error( `An Invalid Encrypted Saved Objects migration is trying to migrate across types ("${inputType.type}" => "${migratedType.type}"), which isn't permitted` diff --git a/x-pack/plugins/encrypted_saved_objects/server/index.ts b/x-pack/plugins/encrypted_saved_objects/server/index.ts index 95337b8c92913..2706da22d108b 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/index.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/index.ts @@ -13,6 +13,7 @@ import { EncryptedSavedObjectsPlugin } from './plugin'; export { EncryptedSavedObjectTypeRegistration, EncryptionError } from './crypto'; export { EncryptedSavedObjectsPluginSetup, EncryptedSavedObjectsPluginStart } from './plugin'; export { EncryptedSavedObjectsClient } from './saved_objects'; +export type { IsMigrationNeededPredicate } from './create_migration'; export const config = { schema: ConfigSchema }; export const plugin = (initializerContext: PluginInitializerContext) => diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_10_0.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_10_0.ts index 50780f168c459..39e65efcf2ab1 100644 --- a/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_10_0.ts +++ b/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_10_0.ts @@ -89,12 +89,14 @@ export const migrateSettingsToV7100: SavedObjectMigrationFn< export const migrateAgentActionToV7100 = ( encryptedSavedObjects: EncryptedSavedObjectsPluginSetup ): SavedObjectMigrationFn => { - return encryptedSavedObjects.createMigration( - (agentActionDoc): agentActionDoc is SavedObjectUnsanitizedDoc => { + return encryptedSavedObjects.createMigration({ + isMigrationNeededPredicate: ( + agentActionDoc + ): agentActionDoc is SavedObjectUnsanitizedDoc => { // @ts-expect-error return agentActionDoc.attributes.type === 'CONFIG_CHANGE'; }, - (agentActionDoc) => { + migration: (agentActionDoc) => { let agentActionData; try { agentActionData = agentActionDoc.attributes.data @@ -122,8 +124,8 @@ export const migrateAgentActionToV7100 = ( } else { return agentActionDoc; } - } - ); + }, + }); }; export const migrateInstallationToV7100: SavedObjectMigrationFn< diff --git a/x-pack/test/encrypted_saved_objects_api_integration/fixtures/api_consumer_plugin/server/index.ts b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/api_consumer_plugin/server/index.ts index ceb5ef0dbbacc..10846442d1c84 100644 --- a/x-pack/test/encrypted_saved_objects_api_integration/fixtures/api_consumer_plugin/server/index.ts +++ b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/api_consumer_plugin/server/index.ts @@ -150,11 +150,13 @@ function defineTypeWithMigration(core: CoreSetup, deps: PluginsSet }, migrations: { // in this version we migrated a non encrypted field and type didnt change - '7.8.0': deps.encryptedSavedObjects.createMigration( - function shouldBeMigrated(doc): doc is SavedObjectUnsanitizedDoc { + '7.8.0': deps.encryptedSavedObjects.createMigration({ + isMigrationNeededPredicate: function shouldBeMigrated( + doc + ): doc is SavedObjectUnsanitizedDoc { return true; }, - ( + migration: ( doc: SavedObjectUnsanitizedDoc ): SavedObjectUnsanitizedDoc => { const { @@ -168,17 +170,18 @@ function defineTypeWithMigration(core: CoreSetup, deps: PluginsSet }, }; }, - false, // type hasn't changed as the field we're updating is not an encrypted one - typePriorTo790, - typePriorTo790 - ), + inputType: typePriorTo790, + migratedType: typePriorTo790, + }), // in this version we encrypted an existing non encrypted field - '7.9.0': deps.encryptedSavedObjects.createMigration( - function shouldBeMigrated(doc): doc is SavedObjectUnsanitizedDoc { + '7.9.0': deps.encryptedSavedObjects.createMigration({ + isMigrationNeededPredicate: function shouldBeMigrated( + doc + ): doc is SavedObjectUnsanitizedDoc { return true; }, - ( + migration: ( doc: SavedObjectUnsanitizedDoc ): SavedObjectUnsanitizedDoc => { const { @@ -194,9 +197,8 @@ function defineTypeWithMigration(core: CoreSetup, deps: PluginsSet }, }; }, - false, - typePriorTo790 - ), + inputType: typePriorTo790, + }), }, }); } From 49f110cb92e3e2b2c444e33f9fa341f8a10c5a7b Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Tue, 20 Jul 2021 14:55:20 -0400 Subject: [PATCH 07/17] Adding functional tests --- .../data.json | 87 + .../mappings.json | 2440 +++++++++++++++++ .../encrypted_saved_objects_decryption.ts | 57 + .../tests/index.ts | 1 + 4 files changed, 2585 insertions(+) create mode 100644 x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/data.json create mode 100644 x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/mappings.json create mode 100644 x-pack/test/encrypted_saved_objects_api_integration/tests/encrypted_saved_objects_decryption.ts diff --git a/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/data.json b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/data.json new file mode 100644 index 0000000000000..30e1176b4a81c --- /dev/null +++ b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/data.json @@ -0,0 +1,87 @@ +{ + "type": "doc", + "value": { + "id": "alert:a0d18560-e985-11eb-b1e3-5b27f0de1e72", + "index": ".kibana_1", + "source": { + "alert" : { + "params" : { + "aggType" : "count", + "termSize" : 5, + "thresholdComparator" : ">", + "timeWindowSize" : 5, + "timeWindowUnit" : "m", + "groupBy" : "all", + "threshold" : [ + 1000 + ], + "index" : [ + ".kibana-event-log-8.0.0" + ], + "timeField" : "@timestamp" + }, + "consumer" : "alerts", + "schedule" : { + "interval" : "1m" + }, + "tags" : [ ], + "name" : "test rule", + "actions" : [ ], + "enabled" : true, + "throttle" : null, + "alertTypeId" : ".index-threshold", + "apiKeyOwner" : "elastic", + "apiKey" : "XpJzeSrg3p/zB+Xz8CcrylW7q5/NVlgC1de0xbbm5E0FbhcT9DskOoUH8jmmyCMOEn7SiPKm62LXbuuLknUmd0EKBDCecK0Mf8NMcqZbTusgWLmQDu5DDx+xAXcI3wsI2KD/wqLhE+RAbiBVuNtFcfm+gBwAKNikei7qtcGL5TzfmC1Cqhn2RjrnTfzCQ0csZwHrKpbBlMhUDA==", + "createdBy" : "elastic", + "updatedBy" : "elastic", + "createdAt" : "2021-07-20T18:09:31.067Z", + "muteAll" : false, + "mutedInstanceIds" : [ ], + "executionStatus" : { + "status" : "ok", + "lastExecutionDate" : "2021-07-20T18:09:34.970Z", + "error" : null + }, + "meta" : { + "versionApiKeyLastmodified" : "8.0.0" + }, + "scheduledTaskId" : "a1b4b970-e985-11eb-b1e3-5b27f0de1e72" + }, + "type" : "alert", + "references" : [ ], + "migrationVersion" : { + "alert" : "7.10.0" + }, + "updated_at" : "2021-07-20T18:09:35.093Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "action:b9127990-e985-11eb-b1e3-5b27f0de1e72", + "index": ".kibana_1", + "source": { + "action" : { + "actionTypeId" : ".email", + "name" : "email connector", + "config" : { + "hasAuth" : true, + "from" : "admin@company.com", + "host" : "mail.company.com", + "port" : 465, + "service" : null, + "secure" : null + }, + "secrets" : "NGVraK7+8QRoKXp4++g8mtwf8WxqFQn+RXn35Pa1gZSq/M8E2/yCycUbTEfzmhRm35It4dn9C0AQeFmlL/3nzY8fZgen4HGRVmn+FGSIoFbVY65Rkiy7v3neigO9NcZlFZ7UQAHn+mubdYrVRjFGwEzN9YbG9zK5zsCVUmVZ8w==" + }, + "type" : "action", + "references" : [ ], + "migrationVersion" : { + "alert" : "7.13.0" + }, + "updated_at" : "2021-07-20T18:10:11.024Z" + } + } +} diff --git a/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/mappings.json b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/mappings.json new file mode 100644 index 0000000000000..6de44ddece61d --- /dev/null +++ b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/mappings.json @@ -0,0 +1,2440 @@ +{ + "type": "index", + "value": { + "aliases": { + ".kibana": { + } + }, + "index": ".kibana_1", + "mappings": { + "_meta": { + "migrationMappingPropertyHashes": { + "action": "6e96ac5e648f57523879661ea72525b7", + "action_task_params": "a9d49f184ee89641044be0ca2950fa3a", + "alert": "7b44fba6773e37c806ce290ea9b7024e", + "apm-indices": "9bb9b2bf1fa636ed8619cbab5ce6a1dd", + "apm-telemetry": "3525d7c22c42bc80f5e6e9cb3f2b26a2", + "application_usage_totals": "c897e4310c5f24b07caaff3db53ae2c1", + "application_usage_transactional": "965839e75f809fefe04f92dc4d99722a", + "canvas-element": "7390014e1091044523666d97247392fc", + "canvas-workpad": "b0a1706d356228dbdcb4a17e6b9eb231", + "cases": "32aa96a6d3855ddda53010ae2048ac22", + "cases-comments": "c2061fb929f585df57425102fa928b4b", + "cases-configure": "42711cbb311976c0687853f4c1354572", + "cases-user-actions": "32277330ec6b721abe3b846cfd939a71", + "config": "ae24d22d5986d04124cc6568f771066f", + "dashboard": "d00f614b29a80360e1190193fd333bab", + "file-upload-telemetry": "0ed4d3e1983d1217a30982630897092e", + "graph-workspace": "cd7ba1330e6682e9cc00b78850874be1", + "index-pattern": "66eccb05066c5a89924f48a9e9736499", + "infrastructure-ui-source": "ddc0ecb18383f6b26101a2fadb2dab0c", + "inventory-view": "88fc7e12fd1b45b6f0787323ce4f18d2", + "kql-telemetry": "d12a98a6f19a2d273696597547e064ee", + "lens": "d33c68a69ff1e78c9888dedd2164ac22", + "lens-ui-telemetry": "509bfa5978586998e05f9e303c07a327", + "maps-telemetry": "bfd39d88aadadb4be597ea984d433dbe", + "metrics-explorer-view": "428e319af3e822c80a84cf87123ca35c", + "migrationVersion": "4a1746014a75ade3a714e1db5763276f", + "ml-telemetry": "257fd1d4b4fdbb9cb4b8a3b27da201e9", + "namespace": "2f4316de49999235636386fe51dc06c1", + "namespaces": "2f4316de49999235636386fe51dc06c1", + "query": "11aaeb7f5f7fa5bb43f25e18ce26e7d9", + "references": "7997cf5a56cc02bdc9c93361bde732b0", + "sample-data-telemetry": "7d3cfeb915303c9641c59681967ffeb4", + "search": "181661168bbadd1eff5902361e2a0d5c", + "space": "c5ca8acafa0beaa4d08d014a97b6bc6b", + "telemetry": "36a616f7026dfa617d6655df850fe16d", + "timelion-sheet": "9a2a2748877c7a7b582fef201ab1d4cf", + "todo": "082a2cc96a590268344d5cd74c159ac4", + "tsvb-validation-telemetry": "3a37ef6c8700ae6fc97d5c7da00e9215", + "type": "2f4316de49999235636386fe51dc06c1", + "ui-metric": "0d409297dc5ebe1e3a1da691c6ee32e3", + "updated_at": "00da57df13e94e9d98437d13ace4bfe0", + "upgrade-assistant-reindex-operation": "296a89039fc4260292be36b1b005d8f2", + "upgrade-assistant-telemetry": "56702cec857e0a9dacfb696655b4ff7b", + "uptime-dynamic-settings": "fcdb453a30092f022f2642db29523d80", + "url": "b675c3be8d76ecf029294d51dc7ec65d", + "visualization": "52d7a13ad68a150c4525b292d23e12cc" + } + }, + "dynamic": "strict", + "properties": { + "action": { + "properties": { + "actionTypeId": { + "type": "keyword" + }, + "config": { + "enabled": false, + "type": "object" + }, + "name": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + }, + "secrets": { + "type": "binary" + } + } + }, + "action_task_params": { + "properties": { + "actionId": { + "type": "keyword" + }, + "apiKey": { + "type": "binary" + }, + "params": { + "enabled": false, + "type": "object" + } + } + }, + "alert": { + "properties": { + "actions": { + "properties": { + "actionRef": { + "type": "keyword" + }, + "actionTypeId": { + "type": "keyword" + }, + "group": { + "type": "keyword" + }, + "params": { + "enabled": false, + "type": "object" + } + }, + "type": "nested" + }, + "alertTypeId": { + "type": "keyword" + }, + "apiKey": { + "type": "binary" + }, + "apiKeyOwner": { + "type": "keyword" + }, + "consumer": { + "type": "keyword" + }, + "createdAt": { + "type": "date" + }, + "createdBy": { + "type": "keyword" + }, + "enabled": { + "type": "boolean" + }, + "executionStatus": { + "properties": { + "status": { + "type": "keyword" + }, + "lastExecutionDate": { + "type": "date" + }, + "error": { + "properties": { + "reason": { + "type": "keyword" + }, + "message": { + "type": "keyword" + } + } + } + } + }, + "meta": { + "properties": { + "versionApiKeyLastmodified": { + "type": "keyword" + } + } + }, + "muteAll": { + "type": "boolean" + }, + "mutedInstanceIds": { + "type": "keyword" + }, + "name": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + }, + "params": { + "enabled": false, + "type": "object" + }, + "schedule": { + "properties": { + "interval": { + "type": "keyword" + } + } + }, + "scheduledTaskId": { + "type": "keyword" + }, + "tags": { + "type": "keyword" + }, + "throttle": { + "type": "keyword" + }, + "updatedBy": { + "type": "keyword" + } + } + }, + "saved-object-with-migration": { + "properties": { + "encryptedAttribute": { + "type": "binary" + }, + "nonEncryptedAttribute": { + "type": "keyword" + }, + "additionalEncryptedAttribute": { + "type": "binary" + } + } + }, + "apm-indices": { + "properties": { + "apm_oss": { + "properties": { + "errorIndices": { + "type": "keyword" + }, + "metricsIndices": { + "type": "keyword" + }, + "onboardingIndices": { + "type": "keyword" + }, + "sourcemapIndices": { + "type": "keyword" + }, + "spanIndices": { + "type": "keyword" + }, + "transactionIndices": { + "type": "keyword" + } + } + } + } + }, + "apm-telemetry": { + "properties": { + "agents": { + "properties": { + "dotnet": { + "properties": { + "agent": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "language": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "runtime": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "go": { + "properties": { + "agent": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "language": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "runtime": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "java": { + "properties": { + "agent": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "language": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "runtime": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "js-base": { + "properties": { + "agent": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "language": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "runtime": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "nodejs": { + "properties": { + "agent": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "language": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "runtime": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "python": { + "properties": { + "agent": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "language": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "runtime": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "ruby": { + "properties": { + "agent": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "language": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "runtime": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "rum-js": { + "properties": { + "agent": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "language": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "runtime": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + } + } + }, + "cardinality": { + "properties": { + "transaction": { + "properties": { + "name": { + "properties": { + "all_agents": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "rum": { + "properties": { + "1d": { + "type": "long" + } + } + } + } + } + } + }, + "user_agent": { + "properties": { + "original": { + "properties": { + "all_agents": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "rum": { + "properties": { + "1d": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "counts": { + "properties": { + "agent_configuration": { + "properties": { + "all": { + "type": "long" + } + } + }, + "error": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "max_error_groups_per_service": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "max_transaction_groups_per_service": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "metric": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "onboarding": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "services": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "sourcemap": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "span": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "traces": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "transaction": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + } + } + }, + "has_any_services": { + "type": "boolean" + }, + "indices": { + "properties": { + "all": { + "properties": { + "total": { + "properties": { + "docs": { + "properties": { + "count": { + "type": "long" + } + } + }, + "store": { + "properties": { + "size_in_bytes": { + "type": "long" + } + } + } + } + } + } + }, + "shards": { + "properties": { + "total": { + "type": "long" + } + } + } + } + }, + "integrations": { + "properties": { + "ml": { + "properties": { + "all_jobs_count": { + "type": "long" + } + } + } + } + }, + "retainment": { + "properties": { + "error": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "metric": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "onboarding": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "span": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "transaction": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "services_per_agent": { + "properties": { + "dotnet": { + "null_value": 0, + "type": "long" + }, + "go": { + "null_value": 0, + "type": "long" + }, + "java": { + "null_value": 0, + "type": "long" + }, + "js-base": { + "null_value": 0, + "type": "long" + }, + "nodejs": { + "null_value": 0, + "type": "long" + }, + "python": { + "null_value": 0, + "type": "long" + }, + "ruby": { + "null_value": 0, + "type": "long" + }, + "rum-js": { + "null_value": 0, + "type": "long" + } + } + }, + "tasks": { + "properties": { + "agent_configuration": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "agents": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "cardinality": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "groupings": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "indices_stats": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "integrations": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "processor_events": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "services": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "versions": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + } + } + }, + "version": { + "properties": { + "apm_server": { + "properties": { + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "patch": { + "type": "long" + } + } + } + } + } + } + }, + "application_usage_totals": { + "properties": { + "appId": { + "type": "keyword" + }, + "minutesOnScreen": { + "type": "float" + }, + "numberOfClicks": { + "type": "long" + } + } + }, + "application_usage_transactional": { + "properties": { + "appId": { + "type": "keyword" + }, + "minutesOnScreen": { + "type": "float" + }, + "numberOfClicks": { + "type": "long" + }, + "timestamp": { + "type": "date" + } + } + }, + "canvas-element": { + "dynamic": "false", + "properties": { + "@created": { + "type": "date" + }, + "@timestamp": { + "type": "date" + }, + "content": { + "type": "text" + }, + "help": { + "type": "text" + }, + "image": { + "type": "text" + }, + "name": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "canvas-workpad": { + "dynamic": "false", + "properties": { + "@created": { + "type": "date" + }, + "@timestamp": { + "type": "date" + }, + "name": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "cases": { + "properties": { + "closed_at": { + "type": "date" + }, + "closed_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "connector_id": { + "type": "keyword" + }, + "created_at": { + "type": "date" + }, + "created_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "description": { + "type": "text" + }, + "external_service": { + "properties": { + "connector_id": { + "type": "keyword" + }, + "connector_name": { + "type": "keyword" + }, + "external_id": { + "type": "keyword" + }, + "external_title": { + "type": "text" + }, + "external_url": { + "type": "text" + }, + "pushed_at": { + "type": "date" + }, + "pushed_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + } + } + }, + "status": { + "type": "keyword" + }, + "tags": { + "type": "keyword" + }, + "title": { + "type": "keyword" + }, + "updated_at": { + "type": "date" + }, + "updated_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + } + } + }, + "cases-comments": { + "properties": { + "comment": { + "type": "text" + }, + "created_at": { + "type": "date" + }, + "created_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "pushed_at": { + "type": "date" + }, + "pushed_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "updated_at": { + "type": "date" + }, + "updated_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + } + } + }, + "cases-configure": { + "properties": { + "closure_type": { + "type": "keyword" + }, + "connector_id": { + "type": "keyword" + }, + "connector_name": { + "type": "keyword" + }, + "created_at": { + "type": "date" + }, + "created_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "updated_at": { + "type": "date" + }, + "updated_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + } + } + }, + "cases-user-actions": { + "properties": { + "action": { + "type": "keyword" + }, + "action_at": { + "type": "date" + }, + "action_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "action_field": { + "type": "keyword" + }, + "new_value": { + "type": "text" + }, + "old_value": { + "type": "text" + } + } + }, + "config": { + "dynamic": "true", + "properties": { + "buildNum": { + "type": "keyword" + } + } + }, + "dashboard": { + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "optionsJSON": { + "type": "text" + }, + "panelsJSON": { + "type": "text" + }, + "refreshInterval": { + "properties": { + "display": { + "type": "keyword" + }, + "pause": { + "type": "boolean" + }, + "section": { + "type": "integer" + }, + "value": { + "type": "integer" + } + } + }, + "timeFrom": { + "type": "keyword" + }, + "timeRestore": { + "type": "boolean" + }, + "timeTo": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "file-upload-telemetry": { + "properties": { + "filesUploadedTotalCount": { + "type": "long" + } + } + }, + "graph-workspace": { + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "numLinks": { + "type": "integer" + }, + "numVertices": { + "type": "integer" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + }, + "wsState": { + "type": "text" + } + } + }, + "index-pattern": { + "properties": { + "fieldFormatMap": { + "type": "text" + }, + "fields": { + "type": "text" + }, + "intervalName": { + "type": "keyword" + }, + "notExpandable": { + "type": "boolean" + }, + "sourceFilters": { + "type": "text" + }, + "timeFieldName": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "type": { + "type": "keyword" + }, + "typeMeta": { + "type": "keyword" + } + } + }, + "infrastructure-ui-source": { + "properties": { + "description": { + "type": "text" + }, + "fields": { + "properties": { + "container": { + "type": "keyword" + }, + "host": { + "type": "keyword" + }, + "pod": { + "type": "keyword" + }, + "tiebreaker": { + "type": "keyword" + }, + "timestamp": { + "type": "keyword" + } + } + }, + "logAlias": { + "type": "keyword" + }, + "logColumns": { + "properties": { + "fieldColumn": { + "properties": { + "field": { + "type": "keyword" + }, + "id": { + "type": "keyword" + } + } + }, + "messageColumn": { + "properties": { + "id": { + "type": "keyword" + } + } + }, + "timestampColumn": { + "properties": { + "id": { + "type": "keyword" + } + } + } + }, + "type": "nested" + }, + "metricAlias": { + "type": "keyword" + }, + "name": { + "type": "text" + } + } + }, + "inventory-view": { + "properties": { + "accountId": { + "type": "keyword" + }, + "autoBounds": { + "type": "boolean" + }, + "autoReload": { + "type": "boolean" + }, + "boundsOverride": { + "properties": { + "max": { + "type": "integer" + }, + "min": { + "type": "integer" + } + } + }, + "customMetrics": { + "properties": { + "aggregation": { + "type": "keyword" + }, + "field": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "label": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + }, + "type": "nested" + }, + "customOptions": { + "properties": { + "field": { + "type": "keyword" + }, + "text": { + "type": "keyword" + } + }, + "type": "nested" + }, + "filterQuery": { + "properties": { + "expression": { + "type": "keyword" + }, + "kind": { + "type": "keyword" + } + } + }, + "groupBy": { + "properties": { + "field": { + "type": "keyword" + }, + "label": { + "type": "keyword" + } + }, + "type": "nested" + }, + "legend": { + "properties": { + "palette": { + "type": "keyword" + }, + "reverseColors": { + "type": "boolean" + }, + "steps": { + "type": "long" + } + } + }, + "metric": { + "properties": { + "aggregation": { + "type": "keyword" + }, + "field": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "label": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "name": { + "type": "keyword" + }, + "nodeType": { + "type": "keyword" + }, + "region": { + "type": "keyword" + }, + "sort": { + "properties": { + "by": { + "type": "keyword" + }, + "direction": { + "type": "keyword" + } + } + }, + "time": { + "type": "long" + }, + "view": { + "type": "keyword" + } + } + }, + "kql-telemetry": { + "properties": { + "optInCount": { + "type": "long" + }, + "optOutCount": { + "type": "long" + } + } + }, + "lens": { + "properties": { + "description": { + "type": "text" + }, + "expression": { + "index": false, + "type": "keyword" + }, + "state": { + "type": "flattened" + }, + "title": { + "type": "text" + }, + "visualizationType": { + "type": "keyword" + } + } + }, + "lens-ui-telemetry": { + "properties": { + "count": { + "type": "integer" + }, + "date": { + "type": "date" + }, + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "map": { + "properties": { + "bounds": { + "dynamic": false, + "properties": {} + }, + "description": { + "type": "text" + }, + "layerListJSON": { + "type": "text" + }, + "mapStateJSON": { + "type": "text" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "maps-telemetry": { + "properties": { + "attributesPerMap": { + "properties": { + "dataSourcesCount": { + "properties": { + "avg": { + "type": "long" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + }, + "emsVectorLayersCount": { + "dynamic": "true", + "type": "object" + }, + "layerTypesCount": { + "dynamic": "true", + "type": "object" + }, + "layersCount": { + "properties": { + "avg": { + "type": "long" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + } + } + }, + "indexPatternsWithGeoFieldCount": { + "type": "long" + }, + "indexPatternsWithGeoPointFieldCount": { + "type": "long" + }, + "indexPatternsWithGeoShapeFieldCount": { + "type": "long" + }, + "mapsTotalCount": { + "type": "long" + }, + "settings": { + "properties": { + "showMapVisualizationTypes": { + "type": "boolean" + } + } + }, + "timeCaptured": { + "type": "date" + } + } + }, + "metrics-explorer-view": { + "properties": { + "chartOptions": { + "properties": { + "stack": { + "type": "boolean" + }, + "type": { + "type": "keyword" + }, + "yAxisMode": { + "type": "keyword" + } + } + }, + "currentTimerange": { + "properties": { + "from": { + "type": "keyword" + }, + "interval": { + "type": "keyword" + }, + "to": { + "type": "keyword" + } + } + }, + "name": { + "type": "keyword" + }, + "options": { + "properties": { + "aggregation": { + "type": "keyword" + }, + "filterQuery": { + "type": "keyword" + }, + "forceInterval": { + "type": "boolean" + }, + "groupBy": { + "type": "keyword" + }, + "limit": { + "type": "integer" + }, + "metrics": { + "properties": { + "aggregation": { + "type": "keyword" + }, + "color": { + "type": "keyword" + }, + "field": { + "type": "keyword" + }, + "label": { + "type": "keyword" + } + }, + "type": "nested" + } + } + } + } + }, + "migrationVersion": { + "dynamic": "true", + "properties": { + "alert": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "saved-object-with-migration": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "config": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "space": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "ml-telemetry": { + "properties": { + "file_data_visualizer": { + "properties": { + "index_creation_count": { + "type": "long" + } + } + } + } + }, + "namespace": { + "type": "keyword" + }, + "namespaces": { + "type": "keyword" + }, + "query": { + "properties": { + "description": { + "type": "text" + }, + "filters": { + "enabled": false, + "type": "object" + }, + "query": { + "properties": { + "language": { + "type": "keyword" + }, + "query": { + "index": false, + "type": "keyword" + } + } + }, + "timefilter": { + "enabled": false, + "type": "object" + }, + "title": { + "type": "text" + } + } + }, + "references": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + }, + "type": "nested" + }, + "sample-data-telemetry": { + "properties": { + "installCount": { + "type": "long" + }, + "unInstallCount": { + "type": "long" + } + } + }, + "search": { + "properties": { + "columns": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "sort": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "space": { + "properties": { + "_reserved": { + "type": "boolean" + }, + "color": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "disabledFeatures": { + "type": "keyword" + }, + "imageUrl": { + "index": false, + "type": "text" + }, + "initials": { + "type": "keyword" + }, + "name": { + "fields": { + "keyword": { + "ignore_above": 2048, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "telemetry": { + "properties": { + "allowChangingOptInStatus": { + "type": "boolean" + }, + "enabled": { + "type": "boolean" + }, + "lastReported": { + "type": "date" + }, + "lastVersionChecked": { + "type": "keyword" + }, + "reportFailureCount": { + "type": "integer" + }, + "reportFailureVersion": { + "type": "keyword" + }, + "sendUsageFrom": { + "type": "keyword" + }, + "userHasSeenNotice": { + "type": "boolean" + } + } + }, + "timelion-sheet": { + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "timelion_chart_height": { + "type": "integer" + }, + "timelion_columns": { + "type": "integer" + }, + "timelion_interval": { + "type": "keyword" + }, + "timelion_other_interval": { + "type": "keyword" + }, + "timelion_rows": { + "type": "integer" + }, + "timelion_sheet": { + "type": "text" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "todo": { + "properties": { + "icon": { + "type": "keyword" + }, + "task": { + "type": "text" + }, + "title": { + "type": "keyword" + } + } + }, + "tsvb-validation-telemetry": { + "properties": { + "failedRequests": { + "type": "long" + } + } + }, + "type": { + "type": "keyword" + }, + "ui-metric": { + "properties": { + "count": { + "type": "integer" + } + } + }, + "updated_at": { + "type": "date" + }, + "upgrade-assistant-reindex-operation": { + "properties": { + "errorMessage": { + "type": "keyword" + }, + "indexName": { + "type": "keyword" + }, + "lastCompletedStep": { + "type": "integer" + }, + "locked": { + "type": "date" + }, + "newIndexName": { + "type": "keyword" + }, + "reindexOptions": { + "properties": { + "openAndClose": { + "type": "boolean" + }, + "queueSettings": { + "properties": { + "queuedAt": { + "type": "long" + }, + "startedAt": { + "type": "long" + } + } + } + } + }, + "reindexTaskId": { + "type": "keyword" + }, + "reindexTaskPercComplete": { + "type": "float" + }, + "runningReindexCount": { + "type": "integer" + }, + "status": { + "type": "integer" + } + } + }, + "upgrade-assistant-telemetry": { + "properties": { + "features": { + "properties": { + "deprecation_logging": { + "properties": { + "enabled": { + "null_value": true, + "type": "boolean" + } + } + } + } + }, + "ui_open": { + "properties": { + "cluster": { + "null_value": 0, + "type": "long" + }, + "indices": { + "null_value": 0, + "type": "long" + }, + "overview": { + "null_value": 0, + "type": "long" + } + } + }, + "ui_reindex": { + "properties": { + "close": { + "null_value": 0, + "type": "long" + }, + "open": { + "null_value": 0, + "type": "long" + }, + "start": { + "null_value": 0, + "type": "long" + }, + "stop": { + "null_value": 0, + "type": "long" + } + } + } + } + }, + "uptime-dynamic-settings": { + "properties": { + "certAgeThreshold": { + "type": "long" + }, + "certExpirationThreshold": { + "type": "long" + }, + "heartbeatIndices": { + "type": "keyword" + } + } + }, + "url": { + "properties": { + "accessCount": { + "type": "long" + }, + "accessDate": { + "type": "date" + }, + "createDate": { + "type": "date" + }, + "url": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "visualization": { + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "savedSearchRefName": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + }, + "visState": { + "type": "text" + } + } + } + } + }, + "settings": { + "index": { + "auto_expand_replicas": "0-1", + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} diff --git a/x-pack/test/encrypted_saved_objects_api_integration/tests/encrypted_saved_objects_decryption.ts b/x-pack/test/encrypted_saved_objects_api_integration/tests/encrypted_saved_objects_decryption.ts new file mode 100644 index 0000000000000..0331f9469d8ed --- /dev/null +++ b/x-pack/test/encrypted_saved_objects_api_integration/tests/encrypted_saved_objects_decryption.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + + describe('encrypted saved objects decryption', () => { + describe('migrations', () => { + before(async () => { + await esArchiver.load( + 'x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key' + ); + }); + + after(async () => { + await esArchiver.unload( + 'x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key' + ); + }); + + it('migrates alert and actions saved objects even if decryption fails', async () => { + const { body: migratedRule } = await supertest + .get(`/api/alerting/rule/a0d18560-e985-11eb-b1e3-5b27f0de1e72`) + .expect(200); + + await supertest + .get( + `/api/saved_objects/get-decrypted-as-internal-user/alert/a0d18560-e985-11eb-b1e3-5b27f0de1e72` + ) + .expect(500); + + expect(migratedRule.notify_when).to.eql('onActiveAlert'); + expect(migratedRule.updated_at).to.eql('2021-07-20T18:09:35.093Z'); + + const { body: migratedConnector } = await supertest + .get(`/api/actions/connector/b9127990-e985-11eb-b1e3-5b27f0de1e72`) + .expect(200); + + await supertest + .get( + `/api/saved_objects/get-decrypted-as-internal-user/action/b9127990-e985-11eb-b1e3-5b27f0de1e72` + ) + .expect(500); + + expect(migratedConnector.is_missing_secrets).to.eql(false); + }); + }); + }); +} diff --git a/x-pack/test/encrypted_saved_objects_api_integration/tests/index.ts b/x-pack/test/encrypted_saved_objects_api_integration/tests/index.ts index 535342ae7416a..de87d627ac486 100644 --- a/x-pack/test/encrypted_saved_objects_api_integration/tests/index.ts +++ b/x-pack/test/encrypted_saved_objects_api_integration/tests/index.ts @@ -11,5 +11,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { describe('encryptedSavedObjects', function encryptedSavedObjectsSuite() { this.tags('ciGroup13'); loadTestFile(require.resolve('./encrypted_saved_objects_api')); + loadTestFile(require.resolve('./encrypted_saved_objects_decryption')); }); } From 4711bd2a98deaae054fb331c07abce74cf8de821 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 21 Jul 2021 08:06:51 -0400 Subject: [PATCH 08/17] Adding comment to functional test --- .../tests/encrypted_saved_objects_decryption.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/x-pack/test/encrypted_saved_objects_api_integration/tests/encrypted_saved_objects_decryption.ts b/x-pack/test/encrypted_saved_objects_api_integration/tests/encrypted_saved_objects_decryption.ts index 0331f9469d8ed..c81ac36d247b4 100644 --- a/x-pack/test/encrypted_saved_objects_api_integration/tests/encrypted_saved_objects_decryption.ts +++ b/x-pack/test/encrypted_saved_objects_api_integration/tests/encrypted_saved_objects_decryption.ts @@ -13,6 +13,14 @@ export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); describe('encrypted saved objects decryption', () => { + // This test uses esArchiver to load alert and action saved objects that have been created with a different encryption key + // than what is used in the test. The SOs are from an older Kibana version to ensure that some migrations will be applied, + // specifically the 7.11 migration on alert SO and the 7.14 migration on action SO. + + // When the test runs, you will see in the console logs both the decryption error and a warning that the migration will run anyway. + // The test asserts that the alert and action SOs have the new fields expected post-migration but retrieving them via + // getDecryptedAsInternalUser fails (as expected) because the decryption fails. + describe('migrations', () => { before(async () => { await esArchiver.load( From 7990aa333ae38dc4019fc9d4d21425d85be80096 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 21 Jul 2021 10:39:22 -0400 Subject: [PATCH 09/17] Adding stripOrDecryptAttributesSync --- .../encrypted_saved_objects_service.test.ts | 204 ++++++++++++++++++ .../crypto/encrypted_saved_objects_service.ts | 57 +++++ 2 files changed, 261 insertions(+) diff --git a/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.test.ts b/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.test.ts index 5ac6467e8d78b..0625199eeed63 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.test.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.test.ts @@ -295,6 +295,210 @@ describe('#stripOrDecryptAttributes', () => { }); }); +describe('#stripOrDecryptAttributesSync', () => { + it('does not strip attributes from unknown types', () => { + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + + expect( + service.stripOrDecryptAttributesSync({ id: 'unknown-id', type: 'unknown-type' }, attributes) + ).toEqual({ attributes: { attrOne: 'one', attrTwo: 'two', attrThree: 'three' } }); + }); + + it('does not strip any attributes if none of them are supposed to be encrypted', () => { + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + + service.registerType({ type: 'known-type-1', attributesToEncrypt: new Set(['attrFour']) }); + + expect( + service.stripOrDecryptAttributesSync({ id: 'known-id', type: 'known-type-1' }, attributes) + ).toEqual({ attributes: { attrOne: 'one', attrTwo: 'two', attrThree: 'three' } }); + }); + + it('strips only attributes that are supposed to be encrypted', () => { + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + + service.registerType({ + type: 'known-type-1', + attributesToEncrypt: new Set(['attrOne', 'attrThree']), + }); + + expect( + service.stripOrDecryptAttributesSync({ id: 'known-id', type: 'known-type-1' }, attributes) + ).toEqual({ attributes: { attrTwo: 'two' } }); + }); + + describe('with `dangerouslyExposeValue`', () => { + it('decrypts and exposes values with `dangerouslyExposeValue` set to `true`', () => { + service.registerType({ + type: 'known-type-1', + attributesToEncrypt: new Set([ + 'attrOne', + { key: 'attrThree', dangerouslyExposeValue: true }, + ]), + }); + + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + const encryptedAttributes = service.encryptAttributesSync( + { type: 'known-type-1', id: 'object-id' }, + attributes + ); + + const mockUser = mockAuthenticatedUser(); + expect( + service.stripOrDecryptAttributesSync( + { type: 'known-type-1', id: 'object-id' }, + encryptedAttributes, + undefined, + { user: mockUser } + ) + ).toEqual({ attributes: { attrTwo: 'two', attrThree: 'three' } }); + + expect(mockAuditLogger.decryptAttributesSuccess).toHaveBeenCalledTimes(1); + expect(mockAuditLogger.decryptAttributesSuccess).toHaveBeenCalledWith( + ['attrThree'], + { type: 'known-type-1', id: 'object-id' }, + mockUser + ); + }); + + it('exposes values with `dangerouslyExposeValue` set to `true` using original attributes if provided', () => { + service.registerType({ + type: 'known-type-1', + attributesToEncrypt: new Set([ + 'attrOne', + { key: 'attrThree', dangerouslyExposeValue: true }, + ]), + }); + + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + const encryptedAttributes = { + attrOne: 'fake-enc-one', + attrTwo: 'two', + attrThree: 'fake-enc-three', + }; + + expect( + service.stripOrDecryptAttributesSync( + { type: 'known-type-1', id: 'object-id' }, + encryptedAttributes, + attributes + ) + ).toEqual({ attributes: { attrTwo: 'two', attrThree: 'three' } }); + + expect(mockAuditLogger.decryptAttributesSuccess).not.toHaveBeenCalled(); + expect(mockAuditLogger.decryptAttributeFailure).not.toHaveBeenCalled(); + }); + + it('strips attributes with `dangerouslyExposeValue` set to `true` if failed to decrypt', () => { + service.registerType({ + type: 'known-type-1', + attributesToEncrypt: new Set([ + 'attrOne', + { key: 'attrThree', dangerouslyExposeValue: true }, + ]), + }); + + const attributes = { + attrZero: 'zero', + attrOne: 'one', + attrTwo: 'two', + attrThree: 'three', + attrFour: 'four', + }; + const encryptedAttributes = service.encryptAttributesSync( + { type: 'known-type-1', id: 'object-id' }, + attributes + ); + + encryptedAttributes.attrThree = 'some-undecryptable-value'; + + const mockUser = mockAuthenticatedUser(); + const { attributes: decryptedAttributes, error } = service.stripOrDecryptAttributesSync( + { type: 'known-type-1', id: 'object-id' }, + encryptedAttributes, + undefined, + { user: mockUser } + ); + + expect(decryptedAttributes).toEqual({ attrZero: 'zero', attrTwo: 'two', attrFour: 'four' }); + expect(error).toMatchInlineSnapshot(`[Error: Unable to decrypt attribute "attrThree"]`); + + expect(mockAuditLogger.decryptAttributesSuccess).not.toHaveBeenCalled(); + expect(mockAuditLogger.decryptAttributeFailure).toHaveBeenCalledWith( + 'attrThree', + { type: 'known-type-1', id: 'object-id' }, + mockUser + ); + }); + }); + + describe('without encryption key', () => { + beforeEach(() => { + service = new EncryptedSavedObjectsService({ + logger: loggingSystemMock.create().get(), + audit: mockAuditLogger, + }); + }); + + it('does not fail if none of attributes are supposed to be encrypted', () => { + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + + service.registerType({ type: 'known-type-1', attributesToEncrypt: new Set(['attrFour']) }); + + expect( + service.stripOrDecryptAttributesSync({ id: 'known-id', type: 'known-type-1' }, attributes) + ).toEqual({ attributes: { attrOne: 'one', attrTwo: 'two', attrThree: 'three' } }); + }); + + it('does not fail if there are attributes are supposed to be encrypted, but should be stripped', () => { + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + + service.registerType({ + type: 'known-type-1', + attributesToEncrypt: new Set(['attrOne', 'attrThree']), + }); + + expect( + service.stripOrDecryptAttributesSync({ id: 'known-id', type: 'known-type-1' }, attributes) + ).toEqual({ attributes: { attrTwo: 'two' } }); + }); + + it('fails if needs to decrypt any attribute', () => { + service.registerType({ + type: 'known-type-1', + attributesToEncrypt: new Set([ + 'attrOne', + { key: 'attrThree', dangerouslyExposeValue: true }, + ]), + }); + + const mockUser = mockAuthenticatedUser(); + const { attributes, error } = service.stripOrDecryptAttributesSync( + { type: 'known-type-1', id: 'object-id' }, + { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }, + undefined, + { user: mockUser } + ); + + expect(attributes).toEqual({ attrTwo: 'two' }); + + const encryptionError = error as EncryptionError; + expect(encryptionError.attributeName).toBe('attrThree'); + expect(encryptionError.message).toBe('Unable to decrypt attribute "attrThree"'); + expect(encryptionError.cause).toEqual( + new Error('Decryption is disabled because of missing decryption keys.') + ); + + expect(mockAuditLogger.decryptAttributeFailure).toHaveBeenCalledTimes(1); + expect(mockAuditLogger.decryptAttributeFailure).toHaveBeenCalledWith( + 'attrThree', + { type: 'known-type-1', id: 'object-id' }, + mockUser + ); + }); + }); +}); + describe('#encryptAttributes', () => { beforeEach(() => { mockNodeCrypto.encrypt.mockImplementation( diff --git a/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.ts b/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.ts index 652a2c8b6870e..c542f24ba939f 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.ts @@ -223,6 +223,63 @@ export class EncryptedSavedObjectsService { return { attributes: clonedAttributes as T, error: decryptionError }; } + public stripOrDecryptAttributesSync>( + descriptor: SavedObjectDescriptor, + attributes: T, + originalAttributes?: T, + params?: DecryptParameters + ) { + const typeDefinition = this.typeDefinitions.get(descriptor.type); + if (typeDefinition === undefined) { + return { attributes }; + } + + let decryptedAttributes: T | null = null; + let decryptionError: Error | undefined; + const clonedAttributes: Record = {}; + for (const [attributeName, attributeValue] of Object.entries(attributes)) { + // We should strip encrypted attribute if definition explicitly mandates that or decryption + // failed. + if ( + typeDefinition.shouldBeStripped(attributeName) || + (!!decryptionError && typeDefinition.shouldBeEncrypted(attributeName)) + ) { + continue; + } + + // If attribute isn't supposed to be encrypted, just copy it to the resulting attribute set. + if (!typeDefinition.shouldBeEncrypted(attributeName)) { + clonedAttributes[attributeName] = attributeValue; + } else if (originalAttributes) { + // If attribute should be decrypted, but we have original attributes used to create object + // we should get raw unencrypted value from there to avoid performance penalty. + clonedAttributes[attributeName] = originalAttributes[attributeName]; + } else { + // Otherwise just try to decrypt attribute. We decrypt all attributes at once, cache it and + // reuse for any other attributes. + if (decryptedAttributes === null) { + try { + decryptedAttributes = this.decryptAttributesSync( + descriptor, + // Decrypt only attributes that are supposed to be exposed. + Object.fromEntries( + Object.entries(attributes).filter(([key]) => !typeDefinition.shouldBeStripped(key)) + ) as T, + params + ); + } catch (err) { + decryptionError = err; + continue; + } + } + + clonedAttributes[attributeName] = decryptedAttributes[attributeName]; + } + } + + return { attributes: clonedAttributes as T, error: decryptionError }; + } + private *attributesToEncryptIterator>( descriptor: SavedObjectDescriptor, attributes: T, From b9b08b838e617bbfe9d28a546f84ef3a0ac3b717 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 21 Jul 2021 14:57:13 -0400 Subject: [PATCH 10/17] Using stripOrDecryptAttributesSync --- .../server/saved_objects/migrations.ts | 2 +- .../server/task_runner/task_runner.ts | 10 +++++++- .../server/create_migration.test.ts | 20 +++++++++++++-- .../server/create_migration.ts | 25 +++++++++---------- .../encrypted_saved_objects_service.mocks.ts | 1 + .../data.json | 1 - .../encrypted_saved_objects_decryption.ts | 5 ++-- 7 files changed, 44 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/actions/server/saved_objects/migrations.ts b/x-pack/plugins/actions/server/saved_objects/migrations.ts index 3006f4aa293a7..de15de7b15e23 100644 --- a/x-pack/plugins/actions/server/saved_objects/migrations.ts +++ b/x-pack/plugins/actions/server/saved_objects/migrations.ts @@ -136,7 +136,7 @@ const addHasAuthConfigurationObject = ( if (doc.attributes.actionTypeId !== '.email' && doc.attributes.actionTypeId !== '.webhook') { return doc; } - const hasAuth = !!doc.attributes.secrets.user || !!doc.attributes.secrets.password; + const hasAuth = !!doc.attributes.secrets?.user || !!doc.attributes.secrets?.password; return { ...doc, attributes: { diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index 605588cbf321f..2cde01b8a315f 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -456,12 +456,20 @@ export class TaskRunner< const { params: { alertId, spaceId }, } = this.taskInstance; - let apiKey: string | null; + let apiKey: string | null | undefined; try { apiKey = await this.getApiKeyForAlertPermissions(alertId, spaceId); } catch (err) { throw new ErrorWithReason(AlertExecutionStatusErrorReasons.Decrypt, err); } + + if (apiKey === undefined) { + throw new ErrorWithReason( + AlertExecutionStatusErrorReasons.Decrypt, + new Error('Unable to decrypt attribute "apiKey" because "apiKey" is undefined.') + ); + } + const [services, alertsClient] = this.getServicesWithSpaceLevelPermissions(spaceId, apiKey); let alert: SanitizedAlert; diff --git a/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts b/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts index 2f476ddac7634..4455b320cde65 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts @@ -212,6 +212,7 @@ describe('createMigration()', () => { { convertToMultiNamespaceType: false } ); + expect(encryptionSavedObjectService.stripOrDecryptAttributesSync).not.toHaveBeenCalled(); expect(migrationFunc).not.toHaveBeenCalled(); expect(encryptionSavedObjectService.encryptAttributesSync).not.toHaveBeenCalled(); }); @@ -236,6 +237,10 @@ describe('createMigration()', () => { const attributes = { firstAttr: 'first_attr', + attrToStrip: 'secret', + }; + const strippedAttributes = { + firstAttr: 'first_attr', }; encryptionSavedObjectService.decryptAttributesSync.mockImplementationOnce(() => { @@ -247,6 +252,10 @@ describe('createMigration()', () => { ); }); + encryptionSavedObjectService.stripOrDecryptAttributesSync.mockReturnValueOnce({ + attributes: strippedAttributes, + }); + noopMigration( { id: '123', @@ -257,7 +266,7 @@ describe('createMigration()', () => { migrationContext ); - expect(encryptionSavedObjectService.decryptAttributesSync).toHaveBeenCalledWith( + expect(encryptionSavedObjectService.stripOrDecryptAttributesSync).toHaveBeenCalledWith( { id: '123', type: 'known-type-1', @@ -268,7 +277,14 @@ describe('createMigration()', () => { ); expect(migrationFunc).toHaveBeenCalled(); - expect(encryptionSavedObjectService.encryptAttributesSync).not.toHaveBeenCalled(); + expect(encryptionSavedObjectService.encryptAttributesSync).toHaveBeenCalledWith( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + }, + strippedAttributes + ); }); it('throws error on migration failure', () => { diff --git a/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts b/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts index 0f13a49810c60..d316d598b6c28 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts @@ -91,13 +91,10 @@ export const getCreateMigration = ( const decryptDescriptor = { id, type, namespace: decryptDescriptorNamespace }; const encryptDescriptor = { id, type, namespace: encryptedDoc.namespace }; - let shouldEncrypt = true; - // decrypt the attributes using the input type definition // if an error occurs during decryption, use the shouldMigrateIfDecryptionFails flag - // to determine whether to throw the error or return the un-decrypted document for migration - // if we are migrating the un-decrypted document, set `shouldEncrypt` to false to avoid encrypting - // an un-decrypted document + // to determine whether to throw the error or continue the migration + // if we are continuing the migration, strip encrypted attributes from the document using stripOrDecryptAttributesSync const documentToMigrate = mapAttributes(encryptedDoc, (inputAttributes) => { try { const decryptedAttributes = inputService.decryptAttributesSync( @@ -105,7 +102,6 @@ export const getCreateMigration = ( inputAttributes, { convertToMultiNamespaceType } ); - shouldEncrypt = true; return decryptedAttributes; } catch (err) { if (!shouldMigrateIfDecryptionFails || !(err instanceof EncryptionError)) { @@ -115,17 +111,20 @@ export const getCreateMigration = ( context.log.warn( `Decryption failed for encrypted Saved Object "${encryptedDoc.id}" of type "${encryptedDoc.type}" with error: ${err.message}. Migration will be applied to the original encrypted document but this may cause decryption errors later on.` ); - shouldEncrypt = false; - return inputAttributes; + const { attributes: strippedAttributes } = inputService.stripOrDecryptAttributesSync( + decryptDescriptor, + inputAttributes, + { + convertToMultiNamespaceType, + } + ); + return strippedAttributes; } }); - // migrate the document - // then if `shouldEncrypt` is true, encrypt the attributes using the migration type definition + // migrate and encrypt the document return mapAttributes(migration(documentToMigrate, context), (migratedAttributes) => { - return shouldEncrypt - ? migratedService.encryptAttributesSync(encryptDescriptor, migratedAttributes) - : migratedAttributes; + return migratedService.encryptAttributesSync(encryptDescriptor, migratedAttributes); }); }; }; diff --git a/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.mocks.ts b/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.mocks.ts index d7ff27ced38c6..2e8e37e61deed 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.mocks.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.mocks.ts @@ -19,6 +19,7 @@ function createEncryptedSavedObjectsServiceMock() { decryptAttributes: jest.fn(), encryptAttributesSync: jest.fn(), decryptAttributesSync: jest.fn(), + stripOrDecryptAttributesSync: jest.fn(), } as unknown) as jest.Mocked; } diff --git a/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/data.json b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/data.json index 30e1176b4a81c..6c3071fd311ab 100644 --- a/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/data.json +++ b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/data.json @@ -67,7 +67,6 @@ "actionTypeId" : ".email", "name" : "email connector", "config" : { - "hasAuth" : true, "from" : "admin@company.com", "host" : "mail.company.com", "port" : 465, diff --git a/x-pack/test/encrypted_saved_objects_api_integration/tests/encrypted_saved_objects_decryption.ts b/x-pack/test/encrypted_saved_objects_api_integration/tests/encrypted_saved_objects_decryption.ts index c81ac36d247b4..7a3b3a1e837e0 100644 --- a/x-pack/test/encrypted_saved_objects_api_integration/tests/encrypted_saved_objects_decryption.ts +++ b/x-pack/test/encrypted_saved_objects_api_integration/tests/encrypted_saved_objects_decryption.ts @@ -14,8 +14,7 @@ export default function ({ getService }: FtrProviderContext) { describe('encrypted saved objects decryption', () => { // This test uses esArchiver to load alert and action saved objects that have been created with a different encryption key - // than what is used in the test. The SOs are from an older Kibana version to ensure that some migrations will be applied, - // specifically the 7.11 migration on alert SO and the 7.14 migration on action SO. + // than what is used in the test. The SOs are from an older Kibana version to ensure that migrations will be applied, // When the test runs, you will see in the console logs both the decryption error and a warning that the migration will run anyway. // The test asserts that the alert and action SOs have the new fields expected post-migration but retrieving them via @@ -45,6 +44,7 @@ export default function ({ getService }: FtrProviderContext) { ) .expect(500); + expect(migratedRule.apiKey).to.be(undefined); expect(migratedRule.notify_when).to.eql('onActiveAlert'); expect(migratedRule.updated_at).to.eql('2021-07-20T18:09:35.093Z'); @@ -58,6 +58,7 @@ export default function ({ getService }: FtrProviderContext) { ) .expect(500); + expect(migratedRule.secrets).to.be(undefined); expect(migratedConnector.is_missing_secrets).to.eql(false); }); }); From c1e8afd411292bf220ac8b47f5c2c24817fc4b5b Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 21 Jul 2021 22:39:53 -0400 Subject: [PATCH 11/17] Fixing unit tests --- .../server/task_runner/task_runner.test.ts | 105 +++++++++++++++++- 1 file changed, 103 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts index 62ca000bc8365..c4194664dd5b9 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts @@ -2315,7 +2315,7 @@ describe('Task Runner', () => { ); }); - test(`doesn't use API key when not provided`, async () => { + test(`doesn't use API key when API key is null`, async () => { const taskRunner = new TaskRunner( alertType, mockedTaskInstance, @@ -2325,7 +2325,9 @@ describe('Task Runner', () => { encryptedSavedObjectsClient.getDecryptedAsInternalUser.mockResolvedValueOnce({ id: '1', type: 'alert', - attributes: {}, + attributes: { + apiKey: null, + }, references: [], }); @@ -2345,6 +2347,105 @@ describe('Task Runner', () => { ); }); + test(`doesn't use API key when API key doesn't exist`, async () => { + const taskRunner = new TaskRunner( + alertType, + mockedTaskInstance, + taskRunnerFactoryInitializerParams + ); + alertsClient.get.mockResolvedValue(mockedAlertTypeSavedObject); + encryptedSavedObjectsClient.getDecryptedAsInternalUser.mockResolvedValueOnce({ + id: '1', + type: 'alert', + attributes: {}, + references: [], + }); + + await taskRunner.run(); + + const eventLogger = taskRunnerFactoryInitializerParams.eventLogger; + expect(eventLogger.logEvent.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + Object { + "@timestamp": "1970-01-01T00:00:00.000Z", + "event": Object { + "action": "execute-start", + "category": Array [ + "alerts", + ], + "kind": "alert", + }, + "kibana": Object { + "saved_objects": Array [ + Object { + "id": "1", + "namespace": undefined, + "rel": "primary", + "type": "alert", + "type_id": "test", + }, + ], + "task": Object { + "schedule_delay": 0, + "scheduled": "1970-01-01T00:00:00.000Z", + }, + }, + "message": "alert execution start: \\"1\\"", + "rule": Object { + "category": "test", + "id": "1", + "license": "basic", + "ruleset": "alerts", + }, + }, + ], + Array [ + Object { + "@timestamp": "1970-01-01T00:00:00.000Z", + "error": Object { + "message": "Unable to decrypt attribute \\"apiKey\\" because \\"apiKey\\" is undefined.", + }, + "event": Object { + "action": "execute", + "category": Array [ + "alerts", + ], + "kind": "alert", + "outcome": "failure", + "reason": "decrypt", + }, + "kibana": Object { + "alerting": Object { + "status": "error", + }, + "saved_objects": Array [ + Object { + "id": "1", + "namespace": undefined, + "rel": "primary", + "type": "alert", + "type_id": "test", + }, + ], + "task": Object { + "schedule_delay": 0, + "scheduled": "1970-01-01T00:00:00.000Z", + }, + }, + "message": "test:1: execution failed", + "rule": Object { + "category": "test", + "id": "1", + "license": "basic", + "ruleset": "alerts", + }, + }, + ], + ] + `); + }); + test('rescheduled the Alert if the schedule has update during a task run', async () => { const taskRunner = new TaskRunner( alertType, From 1365d05c60aeae8c4bd64962cfa6f93293dc4e0d Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Thu, 22 Jul 2021 07:50:07 -0400 Subject: [PATCH 12/17] PR fixes --- .../server/task_runner/task_runner.test.ts | 2 +- .../server/create_migration.test.ts | 4 +- .../server/create_migration.ts | 22 +-- .../crypto/encrypted_saved_objects_service.ts | 147 ++++++++---------- 4 files changed, 77 insertions(+), 98 deletions(-) diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts index a6573fe642305..9ea224853a9b5 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts @@ -2353,7 +2353,7 @@ describe('Task Runner', () => { mockedTaskInstance, taskRunnerFactoryInitializerParams ); - alertsClient.get.mockResolvedValue(mockedAlertTypeSavedObject); + rulesClient.get.mockResolvedValue(mockedAlertTypeSavedObject); encryptedSavedObjectsClient.getDecryptedAsInternalUser.mockResolvedValueOnce({ id: '1', type: 'alert', diff --git a/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts b/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts index 4455b320cde65..32aac4a37e4c2 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts @@ -291,7 +291,7 @@ describe('createMigration()', () => { const instantiateServiceWithLegacyType = jest.fn(() => encryptedSavedObjectsServiceMock.create() ); - const migrationFunc = jest.fn((doc) => { + const migrationFunc = jest.fn(() => { throw new Error('migration failed!'); }); @@ -342,7 +342,7 @@ describe('createMigration()', () => { const instantiateServiceWithLegacyType = jest.fn(() => encryptedSavedObjectsServiceMock.create() ); - const migrationFunc = jest.fn((doc) => { + const migrationFunc = jest.fn(() => { throw new Error('migration failed!'); }); diff --git a/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts b/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts index d316d598b6c28..b35c988ed9a24 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts @@ -97,28 +97,20 @@ export const getCreateMigration = ( // if we are continuing the migration, strip encrypted attributes from the document using stripOrDecryptAttributesSync const documentToMigrate = mapAttributes(encryptedDoc, (inputAttributes) => { try { - const decryptedAttributes = inputService.decryptAttributesSync( - decryptDescriptor, - inputAttributes, - { convertToMultiNamespaceType } - ); - return decryptedAttributes; + return inputService.decryptAttributesSync(decryptDescriptor, inputAttributes, { + convertToMultiNamespaceType, + }); } catch (err) { if (!shouldMigrateIfDecryptionFails || !(err instanceof EncryptionError)) { throw err; } context.log.warn( - `Decryption failed for encrypted Saved Object "${encryptedDoc.id}" of type "${encryptedDoc.type}" with error: ${err.message}. Migration will be applied to the original encrypted document but this may cause decryption errors later on.` - ); - const { attributes: strippedAttributes } = inputService.stripOrDecryptAttributesSync( - decryptDescriptor, - inputAttributes, - { - convertToMultiNamespaceType, - } + `Decryption failed for encrypted Saved Object "${encryptedDoc.id}" of type "${encryptedDoc.type}" with error: ${err.message}. Encrypted attributes have been stripped from the original document and migration will be applied but this may cause decryption errors later on.` ); - return strippedAttributes; + return inputService.stripOrDecryptAttributesSync(decryptDescriptor, inputAttributes, { + convertToMultiNamespaceType, + }).attributes; } }); diff --git a/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.ts b/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.ts index c542f24ba939f..cc1a8414924c3 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.ts @@ -168,82 +168,78 @@ export class EncryptedSavedObjectsService { */ public async stripOrDecryptAttributes>( descriptor: SavedObjectDescriptor, - attributes: T, + attributesToStripOrDecrypt: T, originalAttributes?: T, params?: DecryptParameters ) { - const typeDefinition = this.typeDefinitions.get(descriptor.type); - if (typeDefinition === undefined) { - return { attributes }; - } - - let decryptedAttributes: T | null = null; - let decryptionError: Error | undefined; - const clonedAttributes: Record = {}; - for (const [attributeName, attributeValue] of Object.entries(attributes)) { - // We should strip encrypted attribute if definition explicitly mandates that or decryption - // failed. - if ( - typeDefinition.shouldBeStripped(attributeName) || - (!!decryptionError && typeDefinition.shouldBeEncrypted(attributeName)) - ) { - continue; - } - - // If attribute isn't supposed to be encrypted, just copy it to the resulting attribute set. - if (!typeDefinition.shouldBeEncrypted(attributeName)) { - clonedAttributes[attributeName] = attributeValue; - } else if (originalAttributes) { - // If attribute should be decrypted, but we have original attributes used to create object - // we should get raw unencrypted value from there to avoid performance penalty. - clonedAttributes[attributeName] = originalAttributes[attributeName]; - } else { - // Otherwise just try to decrypt attribute. We decrypt all attributes at once, cache it and - // reuse for any other attributes. - if (decryptedAttributes === null) { - try { - decryptedAttributes = await this.decryptAttributes( - descriptor, - // Decrypt only attributes that are supposed to be exposed. - Object.fromEntries( - Object.entries(attributes).filter(([key]) => !typeDefinition.shouldBeStripped(key)) - ) as T, - params - ); - } catch (err) { - decryptionError = err; - continue; - } - } - - clonedAttributes[attributeName] = decryptedAttributes[attributeName]; - } + const { attributes, attributesToDecrypt } = this.prepareAttributesForStripOrDecrypt( + descriptor, + attributesToStripOrDecrypt, + originalAttributes + ); + try { + const decryptedAttributes = attributesToDecrypt + ? await this.decryptAttributes(descriptor, attributesToDecrypt, params) + : {}; + return { attributes: { ...attributes, ...decryptedAttributes } }; + } catch (error) { + return { attributes, error }; } - - return { attributes: clonedAttributes as T, error: decryptionError }; } + /** + * Takes saved object attributes for the specified type and, depending on the type definition, + * either decrypts or strips encrypted attributes (e.g. in case AAD or encryption key has changed + * and decryption is no longer possible). + * @param descriptor Saved object descriptor (ID, type and optional namespace) + * @param attributesToStripOrDecrypt Object that includes a dictionary of __ALL__ saved object attributes stored + * in Elasticsearch. + * @param [originalAttributes] An optional dictionary of __ALL__ saved object original attributes + * that were used to create that saved object (i.e. values are NOT encrypted). + * @param [params] Parameters that control the way encrypted attributes are handled. + */ public stripOrDecryptAttributesSync>( descriptor: SavedObjectDescriptor, - attributes: T, + attributesToStripOrDecrypt: T, originalAttributes?: T, params?: DecryptParameters + ) { + const { attributes, attributesToDecrypt } = this.prepareAttributesForStripOrDecrypt( + descriptor, + attributesToStripOrDecrypt, + originalAttributes + ); + try { + const decryptedAttributes = attributesToDecrypt + ? this.decryptAttributesSync(descriptor, attributesToDecrypt, params) + : {}; + return { attributes: { ...attributes, ...decryptedAttributes } }; + } catch (error) { + return { attributes, error }; + } + } + + /** + * Takes saved object attributes for the specified type and, depending on the type definition, + * either strips encrypted attributes, replaces with original decrypted value if available, or + * prepares them for decryption. + * @private + */ + private prepareAttributesForStripOrDecrypt>( + descriptor: SavedObjectDescriptor, + attributes: T, + originalAttributes?: T ) { const typeDefinition = this.typeDefinitions.get(descriptor.type); if (typeDefinition === undefined) { - return { attributes }; + return { attributes, attributesToDecrypt: null }; } - let decryptedAttributes: T | null = null; - let decryptionError: Error | undefined; + let attributesToDecrypt: T | undefined; const clonedAttributes: Record = {}; for (const [attributeName, attributeValue] of Object.entries(attributes)) { - // We should strip encrypted attribute if definition explicitly mandates that or decryption - // failed. - if ( - typeDefinition.shouldBeStripped(attributeName) || - (!!decryptionError && typeDefinition.shouldBeEncrypted(attributeName)) - ) { + // We should strip encrypted attribute if definition explicitly mandates that. + if (typeDefinition.shouldBeStripped(attributeName)) { continue; } @@ -254,30 +250,21 @@ export class EncryptedSavedObjectsService { // If attribute should be decrypted, but we have original attributes used to create object // we should get raw unencrypted value from there to avoid performance penalty. clonedAttributes[attributeName] = originalAttributes[attributeName]; - } else { - // Otherwise just try to decrypt attribute. We decrypt all attributes at once, cache it and - // reuse for any other attributes. - if (decryptedAttributes === null) { - try { - decryptedAttributes = this.decryptAttributesSync( - descriptor, - // Decrypt only attributes that are supposed to be exposed. - Object.fromEntries( - Object.entries(attributes).filter(([key]) => !typeDefinition.shouldBeStripped(key)) - ) as T, - params - ); - } catch (err) { - decryptionError = err; - continue; - } - } - - clonedAttributes[attributeName] = decryptedAttributes[attributeName]; + } else if (!attributesToDecrypt) { + // Decrypt only attributes that are supposed to be exposed. + attributesToDecrypt = Object.fromEntries( + Object.entries(attributes).filter(([key]) => !typeDefinition.shouldBeStripped(key)) + ) as T; } } - return { attributes: clonedAttributes as T, error: decryptionError }; + return { + attributes: clonedAttributes as T, + attributesToDecrypt: + attributesToDecrypt && Object.keys(attributesToDecrypt).length > 0 + ? attributesToDecrypt + : null, + }; } private *attributesToEncryptIterator>( From 7c55cf95afd841a0fb2e6d798002c7a42dbcaab7 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Thu, 22 Jul 2021 09:25:13 -0400 Subject: [PATCH 13/17] PR fixes --- .../server/create_migration.ts | 2 +- .../tests/encrypted_saved_objects_decryption.ts | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts b/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts index b35c988ed9a24..b9e6dcf710924 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts @@ -106,7 +106,7 @@ export const getCreateMigration = ( } context.log.warn( - `Decryption failed for encrypted Saved Object "${encryptedDoc.id}" of type "${encryptedDoc.type}" with error: ${err.message}. Encrypted attributes have been stripped from the original document and migration will be applied but this may cause decryption errors later on.` + `Decryption failed for encrypted Saved Object "${encryptedDoc.id}" of type "${encryptedDoc.type}" with error: ${err.message}. Encrypted attributes have been stripped from the original document and migration will be applied but this may cause errors later on.` ); return inputService.stripOrDecryptAttributesSync(decryptDescriptor, inputAttributes, { convertToMultiNamespaceType, diff --git a/x-pack/test/encrypted_saved_objects_api_integration/tests/encrypted_saved_objects_decryption.ts b/x-pack/test/encrypted_saved_objects_api_integration/tests/encrypted_saved_objects_decryption.ts index 7a3b3a1e837e0..5045f912fc26f 100644 --- a/x-pack/test/encrypted_saved_objects_api_integration/tests/encrypted_saved_objects_decryption.ts +++ b/x-pack/test/encrypted_saved_objects_api_integration/tests/encrypted_saved_objects_decryption.ts @@ -17,8 +17,7 @@ export default function ({ getService }: FtrProviderContext) { // than what is used in the test. The SOs are from an older Kibana version to ensure that migrations will be applied, // When the test runs, you will see in the console logs both the decryption error and a warning that the migration will run anyway. - // The test asserts that the alert and action SOs have the new fields expected post-migration but retrieving them via - // getDecryptedAsInternalUser fails (as expected) because the decryption fails. + // The test asserts that the alert and action SOs have the new fields expected post-migration describe('migrations', () => { before(async () => { @@ -40,9 +39,9 @@ export default function ({ getService }: FtrProviderContext) { await supertest .get( - `/api/saved_objects/get-decrypted-as-internal-user/alert/a0d18560-e985-11eb-b1e3-5b27f0de1e72` + `/api/hidden_saved_objects/get-decrypted-as-internal-user/alert/a0d18560-e985-11eb-b1e3-5b27f0de1e72` ) - .expect(500); + .expect(200); expect(migratedRule.apiKey).to.be(undefined); expect(migratedRule.notify_when).to.eql('onActiveAlert'); @@ -54,9 +53,9 @@ export default function ({ getService }: FtrProviderContext) { await supertest .get( - `/api/saved_objects/get-decrypted-as-internal-user/action/b9127990-e985-11eb-b1e3-5b27f0de1e72` + `/api/hidden_saved_objects/get-decrypted-as-internal-user/action/b9127990-e985-11eb-b1e3-5b27f0de1e72` ) - .expect(500); + .expect(200); expect(migratedRule.secrets).to.be(undefined); expect(migratedConnector.is_missing_secrets).to.eql(false); From e93edc27b8f7685ce5db2feee2422f79295c4019 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Thu, 22 Jul 2021 13:58:01 -0400 Subject: [PATCH 14/17] Moving validation of apikey existence in alerting task runner --- .../alerting/server/task_runner/task_runner.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index de571864809a1..86cbe1553f9dc 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -116,14 +116,18 @@ export class TaskRunner< // Only fetch encrypted attributes here, we'll create a saved objects client // scoped with the API key to fetch the remaining data. const { - attributes: { apiKey }, + attributes, } = await this.context.encryptedSavedObjectsClient.getDecryptedAsInternalUser( 'alert', alertId, { namespace } ); - return apiKey; + if (!attributes.hasOwnProperty('apiKey')) { + throw new Error('apiKey does not exist'); + } + + return attributes.apiKey; } private getFakeKibanaRequest(spaceId: string, apiKey: RawAlert['apiKey']) { @@ -463,13 +467,6 @@ export class TaskRunner< throw new ErrorWithReason(AlertExecutionStatusErrorReasons.Decrypt, err); } - if (apiKey === undefined) { - throw new ErrorWithReason( - AlertExecutionStatusErrorReasons.Decrypt, - new Error('Unable to decrypt attribute "apiKey" because "apiKey" is undefined.') - ); - } - const [services, rulesClient] = this.getServicesWithSpaceLevelPermissions(spaceId, apiKey); let alert: SanitizedAlert; From bf26a9be693bffe8fa8516669e9c02caa67cdc36 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Thu, 22 Jul 2021 14:47:03 -0400 Subject: [PATCH 15/17] Cleanup --- x-pack/plugins/alerting/server/task_runner/task_runner.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index 86cbe1553f9dc..fce4e058764b4 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -460,13 +460,12 @@ export class TaskRunner< const { params: { alertId, spaceId }, } = this.taskInstance; - let apiKey: string | null | undefined; + let apiKey: string | null; try { apiKey = await this.getApiKeyForAlertPermissions(alertId, spaceId); } catch (err) { throw new ErrorWithReason(AlertExecutionStatusErrorReasons.Decrypt, err); } - const [services, rulesClient] = this.getServicesWithSpaceLevelPermissions(spaceId, apiKey); let alert: SanitizedAlert; From 03c34ad68198af5ad07fe747bb5bfc158a22b15a Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 28 Jul 2021 08:22:02 -0400 Subject: [PATCH 16/17] Reverting changes to alerting task runner --- .../server/task_runner/task_runner.test.ts | 105 +----------------- .../server/task_runner/task_runner.ts | 8 +- 2 files changed, 4 insertions(+), 109 deletions(-) diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts index e1d3d2e74264d..512663ee4e60e 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts @@ -2315,7 +2315,7 @@ describe('Task Runner', () => { ); }); - test(`doesn't use API key when API key is null`, async () => { + test(`doesn't use API key when not provided`, async () => { const taskRunner = new TaskRunner( alertType, mockedTaskInstance, @@ -2325,9 +2325,7 @@ describe('Task Runner', () => { encryptedSavedObjectsClient.getDecryptedAsInternalUser.mockResolvedValueOnce({ id: '1', type: 'alert', - attributes: { - apiKey: null, - }, + attributes: {}, references: [], }); @@ -2347,105 +2345,6 @@ describe('Task Runner', () => { ); }); - test(`doesn't use API key when API key doesn't exist`, async () => { - const taskRunner = new TaskRunner( - alertType, - mockedTaskInstance, - taskRunnerFactoryInitializerParams - ); - rulesClient.get.mockResolvedValue(mockedAlertTypeSavedObject); - encryptedSavedObjectsClient.getDecryptedAsInternalUser.mockResolvedValueOnce({ - id: '1', - type: 'alert', - attributes: {}, - references: [], - }); - - await taskRunner.run(); - - const eventLogger = taskRunnerFactoryInitializerParams.eventLogger; - expect(eventLogger.logEvent.mock.calls).toMatchInlineSnapshot(` - Array [ - Array [ - Object { - "@timestamp": "1970-01-01T00:00:00.000Z", - "event": Object { - "action": "execute-start", - "category": Array [ - "alerts", - ], - "kind": "alert", - }, - "kibana": Object { - "saved_objects": Array [ - Object { - "id": "1", - "namespace": undefined, - "rel": "primary", - "type": "alert", - "type_id": "test", - }, - ], - "task": Object { - "schedule_delay": 0, - "scheduled": "1970-01-01T00:00:00.000Z", - }, - }, - "message": "alert execution start: \\"1\\"", - "rule": Object { - "category": "test", - "id": "1", - "license": "basic", - "ruleset": "alerts", - }, - }, - ], - Array [ - Object { - "@timestamp": "1970-01-01T00:00:00.000Z", - "error": Object { - "message": "Unable to decrypt attribute \\"apiKey\\" because \\"apiKey\\" is undefined.", - }, - "event": Object { - "action": "execute", - "category": Array [ - "alerts", - ], - "kind": "alert", - "outcome": "failure", - "reason": "decrypt", - }, - "kibana": Object { - "alerting": Object { - "status": "error", - }, - "saved_objects": Array [ - Object { - "id": "1", - "namespace": undefined, - "rel": "primary", - "type": "alert", - "type_id": "test", - }, - ], - "task": Object { - "schedule_delay": 0, - "scheduled": "1970-01-01T00:00:00.000Z", - }, - }, - "message": "test:1: execution failed", - "rule": Object { - "category": "test", - "id": "1", - "license": "basic", - "ruleset": "alerts", - }, - }, - ], - ] - `); - }); - test('rescheduled the Alert if the schedule has update during a task run', async () => { const taskRunner = new TaskRunner( alertType, diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index ba5f677b94728..a326427a6e7fe 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -116,18 +116,14 @@ export class TaskRunner< // Only fetch encrypted attributes here, we'll create a saved objects client // scoped with the API key to fetch the remaining data. const { - attributes, + attributes: { apiKey }, } = await this.context.encryptedSavedObjectsClient.getDecryptedAsInternalUser( 'alert', alertId, { namespace } ); - if (!attributes.hasOwnProperty('apiKey')) { - throw new Error('apiKey does not exist'); - } - - return attributes.apiKey; + return apiKey; } private getFakeKibanaRequest(spaceId: string, apiKey: RawAlert['apiKey']) { From 3a2078c0dc111fb28da6d26407c532cfeb3de806 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 28 Jul 2021 08:24:43 -0400 Subject: [PATCH 17/17] PR fixes --- .../server/saved_objects/migrations.test.ts | 21 +++---------------- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/actions/server/saved_objects/migrations.test.ts b/x-pack/plugins/actions/server/saved_objects/migrations.test.ts index db7d7bb6505b4..beaea76756113 100644 --- a/x-pack/plugins/actions/server/saved_objects/migrations.test.ts +++ b/x-pack/plugins/actions/server/saved_objects/migrations.test.ts @@ -139,12 +139,7 @@ describe('handles errors during migrations', () => { `encryptedSavedObject 7.10.0 migration failed for action ${action.id} with error: Can't migrate!`, { migrations: { - actionDocument: { - ...action, - attributes: { - ...action.attributes, - }, - }, + actionDocument: action, }, } ); @@ -162,12 +157,7 @@ describe('handles errors during migrations', () => { `encryptedSavedObject 7.11.0 migration failed for action ${action.id} with error: Can't migrate!`, { migrations: { - actionDocument: { - ...action, - attributes: { - ...action.attributes, - }, - }, + actionDocument: action, }, } ); @@ -185,12 +175,7 @@ describe('handles errors during migrations', () => { `encryptedSavedObject 7.14.0 migration failed for action ${action.id} with error: Can't migrate!`, { migrations: { - actionDocument: { - ...action, - attributes: { - ...action.attributes, - }, - }, + actionDocument: action, }, } );