From 0e1bfb048f3a4dbf87c558b2309d38f431170941 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Mon, 9 Oct 2023 17:16:40 +0300 Subject: [PATCH 01/40] Register the case action --- .../cases_action/cases_connector.ts | 42 +++++++++++++++++++ .../connectors/cases_action/constants.ts | 13 ++++++ .../server/connectors/cases_action/index.ts | 29 +++++++++++++ .../server/connectors/cases_action/schema.ts | 24 +++++++++++ .../plugins/cases/server/connectors/index.ts | 7 ++++ x-pack/plugins/cases/server/plugin.ts | 6 +++ 6 files changed, 121 insertions(+) create mode 100644 x-pack/plugins/cases/server/connectors/cases_action/cases_connector.ts create mode 100644 x-pack/plugins/cases/server/connectors/cases_action/constants.ts create mode 100644 x-pack/plugins/cases/server/connectors/cases_action/schema.ts diff --git a/x-pack/plugins/cases/server/connectors/cases_action/cases_connector.ts b/x-pack/plugins/cases/server/connectors/cases_action/cases_connector.ts new file mode 100644 index 0000000000000..69bd019c61a29 --- /dev/null +++ b/x-pack/plugins/cases/server/connectors/cases_action/cases_connector.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ServiceParams } from '@kbn/actions-plugin/server'; +import { SubActionConnector } from '@kbn/actions-plugin/server'; +import { CASES_CONNECTOR_SUB_ACTION } from './constants'; +import type { CasesConnectorConfig, CasesConnectorSecrets } from './schema'; +import { CasesConnectorParamsSchema } from './schema'; + +export class CasesConnector extends SubActionConnector< + CasesConnectorConfig, + CasesConnectorSecrets +> { + constructor(params: ServiceParams) { + super(params); + + this.registerSubActions(); + } + + private registerSubActions() { + this.registerSubAction({ + name: CASES_CONNECTOR_SUB_ACTION.RUN, + method: 'run', + schema: CasesConnectorParamsSchema, + }); + } + + /** + * Method is not needed for the Case Connector. + * The function throws an error as a reminder to + * implement it if we need it in the future. + */ + protected getResponseErrorMessage(): string { + throw new Error('Method not implemented.'); + } + + public async run() {} +} diff --git a/x-pack/plugins/cases/server/connectors/cases_action/constants.ts b/x-pack/plugins/cases/server/connectors/cases_action/constants.ts new file mode 100644 index 0000000000000..bea96fcb4f387 --- /dev/null +++ b/x-pack/plugins/cases/server/connectors/cases_action/constants.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const CASES_CONNECTOR_ID = '.cases'; +export const CASES_CONNECTOR_TITLE = 'Cases'; + +export enum CASES_CONNECTOR_SUB_ACTION { + RUN = 'run', +} diff --git a/x-pack/plugins/cases/server/connectors/cases_action/index.ts b/x-pack/plugins/cases/server/connectors/cases_action/index.ts index 1fec1c76430eb..d537de5b58885 100644 --- a/x-pack/plugins/cases/server/connectors/cases_action/index.ts +++ b/x-pack/plugins/cases/server/connectors/cases_action/index.ts @@ -4,3 +4,32 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ + +import { SecurityConnectorFeatureId, UptimeConnectorFeatureId } from '@kbn/actions-plugin/common'; +import type { SubActionConnectorType } from '@kbn/actions-plugin/server/sub_action_framework/types'; +import { CasesConnector } from './cases_connector'; +import { CASES_CONNECTOR_ID, CASES_CONNECTOR_TITLE } from './constants'; +import type { CasesConnectorConfig, CasesConnectorSecrets } from './schema'; +import { CasesConnectorConfigSchema, CasesConnectorSecretsSchema } from './schema'; + +export const getCasesConnectorType = (): SubActionConnectorType< + CasesConnectorConfig, + CasesConnectorSecrets +> => ({ + id: CASES_CONNECTOR_ID, + name: CASES_CONNECTOR_TITLE, + Service: CasesConnector, + schema: { + config: CasesConnectorConfigSchema, + secrets: CasesConnectorSecretsSchema, + }, + /** + * TODO: Limit only to rule types that support + * alerts-as-data + */ + supportedFeatureIds: [SecurityConnectorFeatureId, UptimeConnectorFeatureId], + /** + * TODO: Verify license + */ + minimumLicenseRequired: 'platinum' as const, +}); diff --git a/x-pack/plugins/cases/server/connectors/cases_action/schema.ts b/x-pack/plugins/cases/server/connectors/cases_action/schema.ts new file mode 100644 index 0000000000000..8ce6f8ea1c8c3 --- /dev/null +++ b/x-pack/plugins/cases/server/connectors/cases_action/schema.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { TypeOf } from '@kbn/config-schema'; +import { schema } from '@kbn/config-schema'; + +/** + * The case connector does not have any configuration + * or secrets. + */ +export const CasesConnectorConfigSchema = schema.object({}); +export const CasesConnectorSecretsSchema = schema.object({}); +/** + * TODO: Add needed properties in the params schema. + */ +export const CasesConnectorParamsSchema = schema.object({}); + +export type CasesConnectorConfig = TypeOf; +export type CasesConnectorSecrets = TypeOf; +export type CasesConnectorParams = TypeOf; diff --git a/x-pack/plugins/cases/server/connectors/index.ts b/x-pack/plugins/cases/server/connectors/index.ts index 78b83223a3d66..2ebcddf1ee994 100644 --- a/x-pack/plugins/cases/server/connectors/index.ts +++ b/x-pack/plugins/cases/server/connectors/index.ts @@ -5,5 +5,12 @@ * 2.0. */ +import type { PluginSetupContract as ActionsPluginSetupContract } from '@kbn/actions-plugin/server'; +import { getCasesConnectorType } from './cases_action'; + export * from './types'; export { casesConnectors } from './factory'; + +export function registerConnectorTypes({ actions }: { actions: ActionsPluginSetupContract }) { + actions.registerSubActionConnectorType(getCasesConnectorType()); +} diff --git a/x-pack/plugins/cases/server/plugin.ts b/x-pack/plugins/cases/server/plugin.ts index 510686f1a98bd..1f9cec2a9d1f6 100644 --- a/x-pack/plugins/cases/server/plugin.ts +++ b/x-pack/plugins/cases/server/plugin.ts @@ -60,6 +60,7 @@ import { LICENSING_CASE_ASSIGNMENT_FEATURE } from './common/constants'; import { registerInternalAttachments } from './internal_attachments'; import { registerCaseFileKinds } from './files'; import type { ConfigType } from './config'; +import { registerConnectorTypes } from './connectors'; export interface PluginsSetup { actions: ActionsPluginSetup; @@ -116,6 +117,7 @@ export class CasePlugin { this.externalReferenceAttachmentTypeRegistry, this.persistableStateAttachmentTypeRegistry ); + registerCaseFileKinds(this.caseConfig.files, plugins.files); this.securityPluginSetup = plugins.security; @@ -133,6 +135,7 @@ export class CasePlugin { }, }) ); + core.savedObjects.registerType(caseConfigureSavedObjectType); core.savedObjects.registerType(caseConnectorMappingsSavedObjectType); core.savedObjects.registerType(createCaseSavedObjectType(core, this.logger)); @@ -141,6 +144,7 @@ export class CasePlugin { persistableStateAttachmentTypeRegistry: this.persistableStateAttachmentTypeRegistry, }) ); + core.savedObjects.registerType(casesTelemetrySavedObjectType); core.http.registerRouteHandlerContext( @@ -173,6 +177,8 @@ export class CasePlugin { plugins.licensing.featureUsage.register(LICENSING_CASE_ASSIGNMENT_FEATURE, 'platinum'); + registerConnectorTypes({ actions: plugins.actions }); + return { attachmentFramework: { registerExternalReference: (externalReferenceAttachmentType) => { From 4777f9c9c11c99024440bf85bfd87305402b9ee2 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Mon, 9 Oct 2023 17:41:57 +0300 Subject: [PATCH 02/40] Register the cases oracle --- .../plugins/cases/common/constants/index.ts | 1 + x-pack/plugins/cases/server/plugin.ts | 2 + .../server/saved_object_types/cases_oracle.ts | 45 +++++++++++++++++++ .../cases/server/saved_object_types/index.ts | 1 + 4 files changed, 49 insertions(+) create mode 100644 x-pack/plugins/cases/server/saved_object_types/cases_oracle.ts diff --git a/x-pack/plugins/cases/common/constants/index.ts b/x-pack/plugins/cases/common/constants/index.ts index 603fa70e7d0d4..7250f87c19ad4 100644 --- a/x-pack/plugins/cases/common/constants/index.ts +++ b/x-pack/plugins/cases/common/constants/index.ts @@ -24,6 +24,7 @@ export const CASE_CONNECTOR_MAPPINGS_SAVED_OBJECT = 'cases-connector-mappings' a export const CASE_USER_ACTION_SAVED_OBJECT = 'cases-user-actions' as const; export const CASE_COMMENT_SAVED_OBJECT = 'cases-comments' as const; export const CASE_CONFIGURE_SAVED_OBJECT = 'cases-configure' as const; +export const CASE_ORACLE_SAVED_OBJECT = 'cases-oracle' as const; /** * If more values are added here please also add them here: x-pack/test/cases_api_integration/common/plugins diff --git a/x-pack/plugins/cases/server/plugin.ts b/x-pack/plugins/cases/server/plugin.ts index 1f9cec2a9d1f6..66460404556b0 100644 --- a/x-pack/plugins/cases/server/plugin.ts +++ b/x-pack/plugins/cases/server/plugin.ts @@ -43,6 +43,7 @@ import { createCaseSavedObjectType, createCaseUserActionSavedObjectType, casesTelemetrySavedObjectType, + casesOracleSavedObjectType, } from './saved_object_types'; import type { CasesClient } from './client'; @@ -146,6 +147,7 @@ export class CasePlugin { ); core.savedObjects.registerType(casesTelemetrySavedObjectType); + core.savedObjects.registerType(casesOracleSavedObjectType); core.http.registerRouteHandlerContext( APP_ID, diff --git a/x-pack/plugins/cases/server/saved_object_types/cases_oracle.ts b/x-pack/plugins/cases/server/saved_object_types/cases_oracle.ts new file mode 100644 index 0000000000000..2c2bb6e23d274 --- /dev/null +++ b/x-pack/plugins/cases/server/saved_object_types/cases_oracle.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsType } from '@kbn/core/server'; +import { ALERTING_CASES_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; +import { CASE_ORACLE_SAVED_OBJECT } from '../../common/constants'; + +export const casesOracleSavedObjectType: SavedObjectsType = { + name: CASE_ORACLE_SAVED_OBJECT, + indexPattern: ALERTING_CASES_SAVED_OBJECT_INDEX, + hidden: true, + /** + * TODO: Verify + */ + namespaceType: 'agnostic', + mappings: { + dynamic: false, + properties: { + case_ids: { + type: 'keyword', + }, + counter: { + type: 'unsigned_long', + }, + created_at: { + type: 'date', + }, + /* + grouping_definition: { + type: 'keyword', + }, + */ + rule_id: { + type: 'keyword', + }, + updated_at: { + type: 'date', + }, + }, + }, +}; diff --git a/x-pack/plugins/cases/server/saved_object_types/index.ts b/x-pack/plugins/cases/server/saved_object_types/index.ts index a43e60c0a240b..47fb1f252d783 100644 --- a/x-pack/plugins/cases/server/saved_object_types/index.ts +++ b/x-pack/plugins/cases/server/saved_object_types/index.ts @@ -11,3 +11,4 @@ export { createCaseCommentSavedObjectType } from './comments'; export { createCaseUserActionSavedObjectType } from './user_actions'; export { caseConnectorMappingsSavedObjectType } from './connector_mappings'; export { casesTelemetrySavedObjectType } from './telemetry'; +export { casesOracleSavedObjectType } from './cases_oracle'; From 65a88faa6de8eec5a9a0d5dc6863fd28907f37b6 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 9 Oct 2023 15:13:59 +0000 Subject: [PATCH 03/40] [CI] Auto-commit changed files from 'node scripts/check_mappings_update --fix' --- .../current_mappings.json | 220 ++++++++++-------- 1 file changed, 120 insertions(+), 100 deletions(-) diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 2d77538b09572..d76629385dd4c 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -1468,6 +1468,26 @@ "dynamic": false, "properties": {} }, + "cases-oracle": { + "dynamic": false, + "properties": { + "case_ids": { + "type": "keyword" + }, + "counter": { + "type": "unsigned_long" + }, + "created_at": { + "type": "date" + }, + "rule_id": { + "type": "keyword" + }, + "updated_at": { + "type": "date" + } + } + }, "infrastructure-monitoring-log-view": { "dynamic": false, "properties": { @@ -2597,6 +2617,106 @@ "dynamic": false, "properties": {} }, + "infrastructure-ui-source": { + "dynamic": false, + "properties": {} + }, + "inventory-view": { + "dynamic": false, + "properties": {} + }, + "metrics-explorer-view": { + "dynamic": false, + "properties": {} + }, + "upgrade-assistant-reindex-operation": { + "dynamic": false, + "properties": { + "indexName": { + "type": "keyword" + }, + "status": { + "type": "integer" + } + } + }, + "upgrade-assistant-ml-upgrade-operation": { + "dynamic": false, + "properties": { + "snapshotId": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "monitoring-telemetry": { + "properties": { + "reportedClusterUuids": { + "type": "keyword" + } + } + }, + "apm-telemetry": { + "dynamic": false, + "properties": {} + }, + "apm-server-schema": { + "properties": { + "schemaJson": { + "type": "text", + "index": false + } + } + }, + "apm-service-group": { + "properties": { + "groupName": { + "type": "keyword" + }, + "kuery": { + "type": "text" + }, + "description": { + "type": "text" + }, + "color": { + "type": "text" + } + } + }, + "apm-custom-dashboards": { + "properties": { + "dashboardSavedObjectId": { + "type": "keyword" + }, + "kuery": { + "type": "text" + }, + "serviceEnvironmentFilterEnabled": { + "type": "boolean" + }, + "serviceNameFilterEnabled": { + "type": "boolean" + } + } + }, + "enterprise_search_telemetry": { + "dynamic": false, + "properties": {} + }, + "app_search_telemetry": { + "dynamic": false, + "properties": {} + }, + "workplace_search_telemetry": { + "dynamic": false, + "properties": {} + }, "siem-ui-timeline-note": { "properties": { "eventId": { @@ -3058,105 +3178,5 @@ "index": false } } - }, - "infrastructure-ui-source": { - "dynamic": false, - "properties": {} - }, - "inventory-view": { - "dynamic": false, - "properties": {} - }, - "metrics-explorer-view": { - "dynamic": false, - "properties": {} - }, - "upgrade-assistant-reindex-operation": { - "dynamic": false, - "properties": { - "indexName": { - "type": "keyword" - }, - "status": { - "type": "integer" - } - } - }, - "upgrade-assistant-ml-upgrade-operation": { - "dynamic": false, - "properties": { - "snapshotId": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - }, - "monitoring-telemetry": { - "properties": { - "reportedClusterUuids": { - "type": "keyword" - } - } - }, - "apm-telemetry": { - "dynamic": false, - "properties": {} - }, - "apm-server-schema": { - "properties": { - "schemaJson": { - "type": "text", - "index": false - } - } - }, - "apm-service-group": { - "properties": { - "groupName": { - "type": "keyword" - }, - "kuery": { - "type": "text" - }, - "description": { - "type": "text" - }, - "color": { - "type": "text" - } - } - }, - "apm-custom-dashboards": { - "properties": { - "dashboardSavedObjectId": { - "type": "keyword" - }, - "kuery": { - "type": "text" - }, - "serviceEnvironmentFilterEnabled": { - "type": "boolean" - }, - "serviceNameFilterEnabled": { - "type": "boolean" - } - } - }, - "enterprise_search_telemetry": { - "dynamic": false, - "properties": {} - }, - "app_search_telemetry": { - "dynamic": false, - "properties": {} - }, - "workplace_search_telemetry": { - "dynamic": false, - "properties": {} } } From eeb35a62700e443dae2b2f2e48a63074099e611c Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Mon, 9 Oct 2023 18:14:25 +0300 Subject: [PATCH 04/40] Calculate the hash of the record ID --- .../cases_action/cases_oracle_service.test.ts | 40 +++++++++++++++++ .../cases_action/cases_oracle_service.ts | 45 +++++++++++++++++++ .../cases_action/crypto_service.test.ts | 41 +++++++++++++++++ .../connectors/cases_action/crypto_service.ts | 17 +++++++ 4 files changed, 143 insertions(+) create mode 100644 x-pack/plugins/cases/server/connectors/cases_action/cases_oracle_service.test.ts create mode 100644 x-pack/plugins/cases/server/connectors/cases_action/cases_oracle_service.ts create mode 100644 x-pack/plugins/cases/server/connectors/cases_action/crypto_service.test.ts create mode 100644 x-pack/plugins/cases/server/connectors/cases_action/crypto_service.ts diff --git a/x-pack/plugins/cases/server/connectors/cases_action/cases_oracle_service.test.ts b/x-pack/plugins/cases/server/connectors/cases_action/cases_oracle_service.test.ts new file mode 100644 index 0000000000000..421f7627c36d2 --- /dev/null +++ b/x-pack/plugins/cases/server/connectors/cases_action/cases_oracle_service.test.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createHash } from 'node:crypto'; +import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks'; +import { loggerMock } from '@kbn/logging-mocks'; + +import { CasesOracleService } from './cases_oracle_service'; + +describe('CryptoService', () => { + const unsecuredSavedObjectsClient = savedObjectsClientMock.create(); + const mockLogger = loggerMock.create(); + + let service: CasesOracleService; + + beforeEach(() => { + jest.resetAllMocks(); + service = new CasesOracleService({ unsecuredSavedObjectsClient, log: mockLogger }); + }); + + it('return the record ID correctly', async () => { + const ruleId = 'test-rule-id'; + const spaceId = 'default'; + const owner = 'cases'; + const groupingDefinition = 'host.ip=0.0.0.1'; + + const payload = `${ruleId}:${spaceId}:${owner}:${groupingDefinition}`; + const hash = createHash('sha256'); + + hash.update(payload); + + const hex = hash.digest('hex'); + + expect(service.getRecordId({ ruleId, spaceId, owner, groupingDefinition })).toEqual(hex); + }); +}); diff --git a/x-pack/plugins/cases/server/connectors/cases_action/cases_oracle_service.ts b/x-pack/plugins/cases/server/connectors/cases_action/cases_oracle_service.ts new file mode 100644 index 0000000000000..7bf57150e2786 --- /dev/null +++ b/x-pack/plugins/cases/server/connectors/cases_action/cases_oracle_service.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger, SavedObjectsClientContract } from '@kbn/core/server'; +import { CryptoService } from './crypto_service'; + +interface CasesOracleGetRecordId { + ruleId: string; + spaceId: string; + owner: string; + groupingDefinition: string; +} + +export class CasesOracleService { + private readonly log: Logger; + private readonly unsecuredSavedObjectsClient: SavedObjectsClientContract; + private cryptoService: CryptoService; + + constructor({ + log, + unsecuredSavedObjectsClient, + }: { + log: Logger; + unsecuredSavedObjectsClient: SavedObjectsClientContract; + }) { + this.log = log; + this.unsecuredSavedObjectsClient = unsecuredSavedObjectsClient; + this.cryptoService = new CryptoService(); + } + + public getRecordId({ + ruleId, + spaceId, + owner, + groupingDefinition, + }: CasesOracleGetRecordId): string { + const payload = `${ruleId}:${spaceId}:${owner}:${groupingDefinition}`; + + return this.cryptoService.getHash(payload); + } +} diff --git a/x-pack/plugins/cases/server/connectors/cases_action/crypto_service.test.ts b/x-pack/plugins/cases/server/connectors/cases_action/crypto_service.test.ts new file mode 100644 index 0000000000000..86c7ac860e73a --- /dev/null +++ b/x-pack/plugins/cases/server/connectors/cases_action/crypto_service.test.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createHash } from 'node:crypto'; +import { CryptoService } from './crypto_service'; + +describe('CryptoService', () => { + let service: CryptoService; + + beforeEach(() => { + jest.resetAllMocks(); + service = new CryptoService(); + }); + + it('returns the sha256 of a payload correctly', async () => { + const payload = 'my payload'; + const hash = createHash('sha256'); + + hash.update(payload); + + const hex = hash.digest('hex'); + + expect(service.getHash(payload)).toEqual(hex); + }); + + it('creates a new instance of the hash function on each call', async () => { + const payload = 'my payload'; + const hash = createHash('sha256'); + + hash.update(payload); + + const hex = hash.digest('hex'); + + expect(service.getHash(payload)).toEqual(hex); + expect(service.getHash(payload)).toEqual(hex); + }); +}); diff --git a/x-pack/plugins/cases/server/connectors/cases_action/crypto_service.ts b/x-pack/plugins/cases/server/connectors/cases_action/crypto_service.ts new file mode 100644 index 0000000000000..ab56da14a9200 --- /dev/null +++ b/x-pack/plugins/cases/server/connectors/cases_action/crypto_service.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createHash } from 'node:crypto'; + +export class CryptoService { + public getHash(payload: string): string { + const hash = createHash('sha256'); + + hash.update(payload); + return hash.digest('hex'); + } +} From 4491476a1f595d5b80a9ce767e1e64a505e6c1bb Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Tue, 10 Oct 2023 13:47:33 +0300 Subject: [PATCH 05/40] Get oracle record --- .../cases_action/cases_oracle_service.test.ts | 51 ++++++++++++++----- .../cases_action/cases_oracle_service.ts | 34 ++++++++----- .../server/connectors/cases_action/index.ts | 2 +- .../server/connectors/cases_action/schema.ts | 5 -- .../server/connectors/cases_action/types.ts | 33 ++++++++++++ .../server/saved_object_types/cases_oracle.ts | 8 +-- 6 files changed, 99 insertions(+), 34 deletions(-) create mode 100644 x-pack/plugins/cases/server/connectors/cases_action/types.ts diff --git a/x-pack/plugins/cases/server/connectors/cases_action/cases_oracle_service.test.ts b/x-pack/plugins/cases/server/connectors/cases_action/cases_oracle_service.test.ts index 421f7627c36d2..0bb99253f23bc 100644 --- a/x-pack/plugins/cases/server/connectors/cases_action/cases_oracle_service.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases_action/cases_oracle_service.test.ts @@ -10,31 +10,58 @@ import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks import { loggerMock } from '@kbn/logging-mocks'; import { CasesOracleService } from './cases_oracle_service'; +import { CASE_ORACLE_SAVED_OBJECT } from '../../../common/constants'; -describe('CryptoService', () => { +describe('CasesOracleService', () => { const unsecuredSavedObjectsClient = savedObjectsClientMock.create(); const mockLogger = loggerMock.create(); let service: CasesOracleService; beforeEach(() => { - jest.resetAllMocks(); + jest.clearAllMocks(); service = new CasesOracleService({ unsecuredSavedObjectsClient, log: mockLogger }); }); - it('return the record ID correctly', async () => { - const ruleId = 'test-rule-id'; - const spaceId = 'default'; - const owner = 'cases'; - const groupingDefinition = 'host.ip=0.0.0.1'; + describe('getRecordId', () => { + it('return the record ID correctly', async () => { + const ruleId = 'test-rule-id'; + const spaceId = 'default'; + const owner = 'cases'; + const groupingDefinition = 'host.ip=0.0.0.1'; - const payload = `${ruleId}:${spaceId}:${owner}:${groupingDefinition}`; - const hash = createHash('sha256'); + const payload = `${ruleId}:${spaceId}:${owner}:${groupingDefinition}`; + const hash = createHash('sha256'); - hash.update(payload); + hash.update(payload); - const hex = hash.digest('hex'); + const hex = hash.digest('hex'); - expect(service.getRecordId({ ruleId, spaceId, owner, groupingDefinition })).toEqual(hex); + expect(service.getRecordId({ ruleId, spaceId, owner, groupingDefinition })).toEqual(hex); + }); + }); + + describe('getRecord', () => { + const oracleSO = { + id: 'so-id', + version: 'so-version', + attributes: { + counter: 1, + caseIds: ['test-case-id'], + ruleId: 'test-rule-id', + createdAt: '2023-10-10T10:23:42.769Z', + updatedAt: '2023-10-10T10:23:42.769Z', + }, + type: CASE_ORACLE_SAVED_OBJECT, + references: [], + }; + + unsecuredSavedObjectsClient.get.mockResolvedValue(oracleSO); + + it('gets a record correctly', async () => { + const record = await service.getRecord('so-id'); + + expect(record).toEqual({ ...oracleSO.attributes, id: 'so-id' }); + }); }); }); diff --git a/x-pack/plugins/cases/server/connectors/cases_action/cases_oracle_service.ts b/x-pack/plugins/cases/server/connectors/cases_action/cases_oracle_service.ts index 7bf57150e2786..3dd649574757a 100644 --- a/x-pack/plugins/cases/server/connectors/cases_action/cases_oracle_service.ts +++ b/x-pack/plugins/cases/server/connectors/cases_action/cases_oracle_service.ts @@ -6,14 +6,11 @@ */ import type { Logger, SavedObjectsClientContract } from '@kbn/core/server'; +import { CASE_ORACLE_SAVED_OBJECT } from '../../../common/constants'; import { CryptoService } from './crypto_service'; +import type { OracleKey, OracleRecord } from './types'; -interface CasesOracleGetRecordId { - ruleId: string; - spaceId: string; - owner: string; - groupingDefinition: string; -} +type OracleRecordWithoutId = Omit; export class CasesOracleService { private readonly log: Logger; @@ -32,14 +29,27 @@ export class CasesOracleService { this.cryptoService = new CryptoService(); } - public getRecordId({ - ruleId, - spaceId, - owner, - groupingDefinition, - }: CasesOracleGetRecordId): string { + public getRecordId({ ruleId, spaceId, owner, groupingDefinition }: OracleKey): string { const payload = `${ruleId}:${spaceId}:${owner}:${groupingDefinition}`; return this.cryptoService.getHash(payload); } + + public async getRecord(recordId: string): Promise { + this.log.debug(`Getting oracle record with ID: ${recordId}`); + + const oracleRecord = await this.unsecuredSavedObjectsClient.get( + CASE_ORACLE_SAVED_OBJECT, + recordId + ); + + return { + id: oracleRecord.id, + counter: oracleRecord.attributes.counter, + caseIds: oracleRecord.attributes.caseIds, + ruleId: oracleRecord.attributes.ruleId, + createdAt: oracleRecord.attributes.createdAt, + updatedAt: oracleRecord.attributes.updatedAt, + }; + } } diff --git a/x-pack/plugins/cases/server/connectors/cases_action/index.ts b/x-pack/plugins/cases/server/connectors/cases_action/index.ts index d537de5b58885..244474f286b8f 100644 --- a/x-pack/plugins/cases/server/connectors/cases_action/index.ts +++ b/x-pack/plugins/cases/server/connectors/cases_action/index.ts @@ -9,7 +9,7 @@ import { SecurityConnectorFeatureId, UptimeConnectorFeatureId } from '@kbn/actio import type { SubActionConnectorType } from '@kbn/actions-plugin/server/sub_action_framework/types'; import { CasesConnector } from './cases_connector'; import { CASES_CONNECTOR_ID, CASES_CONNECTOR_TITLE } from './constants'; -import type { CasesConnectorConfig, CasesConnectorSecrets } from './schema'; +import type { CasesConnectorConfig, CasesConnectorSecrets } from './types'; import { CasesConnectorConfigSchema, CasesConnectorSecretsSchema } from './schema'; export const getCasesConnectorType = (): SubActionConnectorType< diff --git a/x-pack/plugins/cases/server/connectors/cases_action/schema.ts b/x-pack/plugins/cases/server/connectors/cases_action/schema.ts index 8ce6f8ea1c8c3..4b6aedaee02d7 100644 --- a/x-pack/plugins/cases/server/connectors/cases_action/schema.ts +++ b/x-pack/plugins/cases/server/connectors/cases_action/schema.ts @@ -5,7 +5,6 @@ * 2.0. */ -import type { TypeOf } from '@kbn/config-schema'; import { schema } from '@kbn/config-schema'; /** @@ -18,7 +17,3 @@ export const CasesConnectorSecretsSchema = schema.object({}); * TODO: Add needed properties in the params schema. */ export const CasesConnectorParamsSchema = schema.object({}); - -export type CasesConnectorConfig = TypeOf; -export type CasesConnectorSecrets = TypeOf; -export type CasesConnectorParams = TypeOf; diff --git a/x-pack/plugins/cases/server/connectors/cases_action/types.ts b/x-pack/plugins/cases/server/connectors/cases_action/types.ts new file mode 100644 index 0000000000000..ab17cae70d19e --- /dev/null +++ b/x-pack/plugins/cases/server/connectors/cases_action/types.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { TypeOf } from '@kbn/config-schema'; +import type { + CasesConnectorConfigSchema, + CasesConnectorSecretsSchema, + CasesConnectorParamsSchema, +} from './schema'; + +export type CasesConnectorConfig = TypeOf; +export type CasesConnectorSecrets = TypeOf; +export type CasesConnectorParams = TypeOf; + +export interface OracleKey { + ruleId: string; + spaceId: string; + owner: string; + groupingDefinition: string; +} + +export interface OracleRecord { + id: string; + counter: number; + caseIds: string[]; + ruleId: string; + createdAt: string; + updatedAt: string; +} diff --git a/x-pack/plugins/cases/server/saved_object_types/cases_oracle.ts b/x-pack/plugins/cases/server/saved_object_types/cases_oracle.ts index 2c2bb6e23d274..2eea605a16726 100644 --- a/x-pack/plugins/cases/server/saved_object_types/cases_oracle.ts +++ b/x-pack/plugins/cases/server/saved_object_types/cases_oracle.ts @@ -20,13 +20,13 @@ export const casesOracleSavedObjectType: SavedObjectsType = { mappings: { dynamic: false, properties: { - case_ids: { + caseIds: { type: 'keyword', }, counter: { type: 'unsigned_long', }, - created_at: { + createdAt: { type: 'date', }, /* @@ -34,10 +34,10 @@ export const casesOracleSavedObjectType: SavedObjectsType = { type: 'keyword', }, */ - rule_id: { + ruleId: { type: 'keyword', }, - updated_at: { + updatedAt: { type: 'date', }, }, From f4a81a1d4fcd5c954225da6c8f5cd94c4abeee11 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Wed, 11 Oct 2023 15:21:52 +0300 Subject: [PATCH 06/40] Rename folder --- .../connectors/{cases_action => cases}/cases_connector.ts | 2 +- .../{cases_action => cases}/cases_oracle_service.test.ts | 0 .../connectors/{cases_action => cases}/cases_oracle_service.ts | 0 .../server/connectors/{cases_action => cases}/constants.ts | 0 .../connectors/{cases_action => cases}/crypto_service.test.ts | 0 .../server/connectors/{cases_action => cases}/crypto_service.ts | 0 .../cases/server/connectors/{cases_action => cases}/index.ts | 0 .../cases/server/connectors/{cases_action => cases}/schema.ts | 0 .../cases/server/connectors/{cases_action => cases}/types.ts | 0 x-pack/plugins/cases/server/connectors/index.ts | 2 +- 10 files changed, 2 insertions(+), 2 deletions(-) rename x-pack/plugins/cases/server/connectors/{cases_action => cases}/cases_connector.ts (98%) rename x-pack/plugins/cases/server/connectors/{cases_action => cases}/cases_oracle_service.test.ts (100%) rename x-pack/plugins/cases/server/connectors/{cases_action => cases}/cases_oracle_service.ts (100%) rename x-pack/plugins/cases/server/connectors/{cases_action => cases}/constants.ts (100%) rename x-pack/plugins/cases/server/connectors/{cases_action => cases}/crypto_service.test.ts (100%) rename x-pack/plugins/cases/server/connectors/{cases_action => cases}/crypto_service.ts (100%) rename x-pack/plugins/cases/server/connectors/{cases_action => cases}/index.ts (100%) rename x-pack/plugins/cases/server/connectors/{cases_action => cases}/schema.ts (100%) rename x-pack/plugins/cases/server/connectors/{cases_action => cases}/types.ts (100%) diff --git a/x-pack/plugins/cases/server/connectors/cases_action/cases_connector.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts similarity index 98% rename from x-pack/plugins/cases/server/connectors/cases_action/cases_connector.ts rename to x-pack/plugins/cases/server/connectors/cases/cases_connector.ts index 69bd019c61a29..d7cee94276c51 100644 --- a/x-pack/plugins/cases/server/connectors/cases_action/cases_connector.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts @@ -8,7 +8,7 @@ import type { ServiceParams } from '@kbn/actions-plugin/server'; import { SubActionConnector } from '@kbn/actions-plugin/server'; import { CASES_CONNECTOR_SUB_ACTION } from './constants'; -import type { CasesConnectorConfig, CasesConnectorSecrets } from './schema'; +import type { CasesConnectorConfig, CasesConnectorSecrets } from './types'; import { CasesConnectorParamsSchema } from './schema'; export class CasesConnector extends SubActionConnector< diff --git a/x-pack/plugins/cases/server/connectors/cases_action/cases_oracle_service.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts similarity index 100% rename from x-pack/plugins/cases/server/connectors/cases_action/cases_oracle_service.test.ts rename to x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts diff --git a/x-pack/plugins/cases/server/connectors/cases_action/cases_oracle_service.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts similarity index 100% rename from x-pack/plugins/cases/server/connectors/cases_action/cases_oracle_service.ts rename to x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts diff --git a/x-pack/plugins/cases/server/connectors/cases_action/constants.ts b/x-pack/plugins/cases/server/connectors/cases/constants.ts similarity index 100% rename from x-pack/plugins/cases/server/connectors/cases_action/constants.ts rename to x-pack/plugins/cases/server/connectors/cases/constants.ts diff --git a/x-pack/plugins/cases/server/connectors/cases_action/crypto_service.test.ts b/x-pack/plugins/cases/server/connectors/cases/crypto_service.test.ts similarity index 100% rename from x-pack/plugins/cases/server/connectors/cases_action/crypto_service.test.ts rename to x-pack/plugins/cases/server/connectors/cases/crypto_service.test.ts diff --git a/x-pack/plugins/cases/server/connectors/cases_action/crypto_service.ts b/x-pack/plugins/cases/server/connectors/cases/crypto_service.ts similarity index 100% rename from x-pack/plugins/cases/server/connectors/cases_action/crypto_service.ts rename to x-pack/plugins/cases/server/connectors/cases/crypto_service.ts diff --git a/x-pack/plugins/cases/server/connectors/cases_action/index.ts b/x-pack/plugins/cases/server/connectors/cases/index.ts similarity index 100% rename from x-pack/plugins/cases/server/connectors/cases_action/index.ts rename to x-pack/plugins/cases/server/connectors/cases/index.ts diff --git a/x-pack/plugins/cases/server/connectors/cases_action/schema.ts b/x-pack/plugins/cases/server/connectors/cases/schema.ts similarity index 100% rename from x-pack/plugins/cases/server/connectors/cases_action/schema.ts rename to x-pack/plugins/cases/server/connectors/cases/schema.ts diff --git a/x-pack/plugins/cases/server/connectors/cases_action/types.ts b/x-pack/plugins/cases/server/connectors/cases/types.ts similarity index 100% rename from x-pack/plugins/cases/server/connectors/cases_action/types.ts rename to x-pack/plugins/cases/server/connectors/cases/types.ts diff --git a/x-pack/plugins/cases/server/connectors/index.ts b/x-pack/plugins/cases/server/connectors/index.ts index 2ebcddf1ee994..329eb6a911133 100644 --- a/x-pack/plugins/cases/server/connectors/index.ts +++ b/x-pack/plugins/cases/server/connectors/index.ts @@ -6,7 +6,7 @@ */ import type { PluginSetupContract as ActionsPluginSetupContract } from '@kbn/actions-plugin/server'; -import { getCasesConnectorType } from './cases_action'; +import { getCasesConnectorType } from './cases'; export * from './types'; export { casesConnectors } from './factory'; From 5822d732b57fefdd8eb8b26476d9935cd5649450 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Wed, 11 Oct 2023 16:19:57 +0300 Subject: [PATCH 07/40] Sort grouping definition --- .../cases/cases_oracle_service.test.ts | 17 +++++++++ .../connectors/cases/cases_oracle_service.ts | 11 +++++- .../cases/server/connectors/cases/types.ts | 2 +- .../cases/server/connectors/cases/util.ts | 38 +++++++++++++++++++ .../server/connectors/cases/utils.test.ts | 33 ++++++++++++++++ 5 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugins/cases/server/connectors/cases/util.ts create mode 100644 x-pack/plugins/cases/server/connectors/cases/utils.test.ts diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts index 0bb99253f23bc..6324d41b62035 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts @@ -39,6 +39,23 @@ describe('CasesOracleService', () => { expect(service.getRecordId({ ruleId, spaceId, owner, groupingDefinition })).toEqual(hex); }); + + it('sorts the grouping definition correctly', async () => { + const ruleId = 'test-rule-id'; + const spaceId = 'default'; + const owner = 'cases'; + const groupingDefinition = 'host.ip=0.0.0.1&agent.id=8a4f500d'; + const sortedGroupingDefinition = 'agent.id=8a4f500d&host.ip=0.0.0.1'; + + const payload = `${ruleId}:${spaceId}:${owner}:${sortedGroupingDefinition}`; + const hash = createHash('sha256'); + + hash.update(payload); + + const hex = hash.digest('hex'); + + expect(service.getRecordId({ ruleId, spaceId, owner, groupingDefinition })).toEqual(hex); + }); }); describe('getRecord', () => { diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts index 3dd649574757a..eb9771f4b134b 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts @@ -5,10 +5,12 @@ * 2.0. */ +import { isEmpty } from 'lodash'; import type { Logger, SavedObjectsClientContract } from '@kbn/core/server'; import { CASE_ORACLE_SAVED_OBJECT } from '../../../common/constants'; import { CryptoService } from './crypto_service'; import type { OracleKey, OracleRecord } from './types'; +import { sortGroupDefinition } from './util'; type OracleRecordWithoutId = Omit; @@ -30,7 +32,14 @@ export class CasesOracleService { } public getRecordId({ ruleId, spaceId, owner, groupingDefinition }: OracleKey): string { - const payload = `${ruleId}:${spaceId}:${owner}:${groupingDefinition}`; + const initialPayload = `${ruleId}:${spaceId}:${owner}`; + + if (groupingDefinition == null || isEmpty(groupingDefinition)) { + return this.cryptoService.getHash(initialPayload); + } + + const sortedGroupingDefinition = sortGroupDefinition(groupingDefinition); + const payload = `${initialPayload}:${sortedGroupingDefinition}`; return this.cryptoService.getHash(payload); } diff --git a/x-pack/plugins/cases/server/connectors/cases/types.ts b/x-pack/plugins/cases/server/connectors/cases/types.ts index ab17cae70d19e..915226d2b5afe 100644 --- a/x-pack/plugins/cases/server/connectors/cases/types.ts +++ b/x-pack/plugins/cases/server/connectors/cases/types.ts @@ -20,7 +20,7 @@ export interface OracleKey { ruleId: string; spaceId: string; owner: string; - groupingDefinition: string; + groupingDefinition?: string; } export interface OracleRecord { diff --git a/x-pack/plugins/cases/server/connectors/cases/util.ts b/x-pack/plugins/cases/server/connectors/cases/util.ts new file mode 100644 index 0000000000000..ca9978f00fcf9 --- /dev/null +++ b/x-pack/plugins/cases/server/connectors/cases/util.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * groupingDefinition is of the form + * =&=. + * Example: host.ip=0.0.0.1&host.name=A + * + * The function assumes that no duplicated field keys + * exist. Also it assumes the following special characters + * ":", "=", "&" are escaped property. For example, + * host.ip=0.0.0.1&host.ip=0.0.0.2 or host.ip=2001:db8::8a2e:370:7334 are not valid + */ + +export const sortGroupDefinition = (groupingDefinition: string): string => { + const fieldMap = new Map(); + const fields = groupingDefinition.split('&'); + + if (fields.length <= 1) { + return groupingDefinition; + } + + for (const field of fields) { + const [key, value] = field.split('='); + fieldMap.set(key, value); + } + + const sortedKeys = Array.from(fieldMap.keys()).sort(sortStings); + const sortedFields = sortedKeys.map((key) => `${key}=${fieldMap.get(key)}`); + + return sortedFields.join('&'); +}; + +const sortStings = (a: string, b: string) => String(a).localeCompare(b); diff --git a/x-pack/plugins/cases/server/connectors/cases/utils.test.ts b/x-pack/plugins/cases/server/connectors/cases/utils.test.ts new file mode 100644 index 0000000000000..c5c03303d2f15 --- /dev/null +++ b/x-pack/plugins/cases/server/connectors/cases/utils.test.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { sortGroupDefinition } from './util'; + +describe('utils', () => { + describe('sortGroupDefinition', () => { + it('sorts a group definition correctly', async () => { + const groupingDefinition = + 'source.nat.ip=0.1.2.0&host.name=A&host.ip=0.0.0.1&agent.id=8a4f500d'; + + expect(sortGroupDefinition(groupingDefinition)).toBe( + 'agent.id=8a4f500d&host.ip=0.0.0.1&host.name=A&source.nat.ip=0.1.2.0' + ); + }); + + it('returns the grouping definition if there is only one field', async () => { + const groupingDefinition = 'host.name=A'; + + expect(sortGroupDefinition(groupingDefinition)).toBe('host.name=A'); + }); + + it('returns an empty string if the grouping definition is an empty string', async () => { + const groupingDefinition = ''; + + expect(sortGroupDefinition(groupingDefinition)).toBe(''); + }); + }); +}); From df77537cef68125e5a3d569f20ac703970f82828 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Wed, 11 Oct 2023 17:08:25 +0300 Subject: [PATCH 08/40] Increase counter --- .../cases/cases_oracle_service.test.ts | 112 +++++++++++++++++- .../connectors/cases/cases_oracle_service.ts | 65 ++++++++-- .../cases/server/connectors/cases/types.ts | 4 + 3 files changed, 169 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts index 6324d41b62035..356a1b6df7815 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts @@ -19,7 +19,7 @@ describe('CasesOracleService', () => { let service: CasesOracleService; beforeEach(() => { - jest.clearAllMocks(); + jest.resetAllMocks(); service = new CasesOracleService({ unsecuredSavedObjectsClient, log: mockLogger }); }); @@ -73,7 +73,9 @@ describe('CasesOracleService', () => { references: [], }; - unsecuredSavedObjectsClient.get.mockResolvedValue(oracleSO); + beforeEach(() => { + unsecuredSavedObjectsClient.get.mockResolvedValue(oracleSO); + }); it('gets a record correctly', async () => { const record = await service.getRecord('so-id'); @@ -81,4 +83,110 @@ describe('CasesOracleService', () => { expect(record).toEqual({ ...oracleSO.attributes, id: 'so-id' }); }); }); + + describe('createRecord', () => { + const ruleId = 'test-rule-id'; + const spaceId = 'default'; + const owner = 'cases'; + const groupingDefinition = 'host.ip=0.0.0.1'; + const caseIds = ['test-case-id']; + + const oracleSO = { + id: 'so-id', + version: 'so-version', + attributes: { + counter: 1, + caseIds, + ruleId, + createdAt: '2023-10-10T10:23:42.769Z', + updatedAt: '2023-10-10T10:23:42.769Z', + }, + type: CASE_ORACLE_SAVED_OBJECT, + references: [], + }; + + beforeEach(() => { + unsecuredSavedObjectsClient.create.mockResolvedValue(oracleSO); + }); + + it('creates a record correctly', async () => { + const record = await service.createRecord({ + ruleId, + spaceId, + owner, + groupingDefinition, + caseIds, + }); + + expect(record).toEqual({ ...oracleSO.attributes, id: 'so-id' }); + }); + + it('calls the unsecuredSavedObjectsClient.create method correctly', async () => { + const keyParams = { + ruleId, + spaceId, + owner, + groupingDefinition, + }; + + const id = service.getRecordId(keyParams); + + await service.createRecord({ + ...keyParams, + caseIds, + }); + + expect(unsecuredSavedObjectsClient.create).toHaveBeenCalledWith( + 'cases-oracle', + { + caseIds: ['test-case-id'], + counter: 1, + createdAt: expect.anything(), + ruleId: 'test-rule-id', + updatedAt: expect.anything(), + }, + { id } + ); + }); + }); + + describe('increaseCounter', () => { + const oracleSO = { + id: 'so-id', + version: 'so-version', + attributes: { + counter: 1, + caseIds: ['test-case-id'], + ruleId: 'test-rule-id', + createdAt: '2023-10-10T10:23:42.769Z', + updatedAt: '2023-10-10T10:23:42.769Z', + }, + type: CASE_ORACLE_SAVED_OBJECT, + references: [], + }; + + const oracleSOWithIncreasedCounter = { + ...oracleSO, + attributes: { ...oracleSO.attributes, counter: 2 }, + }; + + beforeEach(() => { + unsecuredSavedObjectsClient.get.mockResolvedValue(oracleSO); + unsecuredSavedObjectsClient.update.mockResolvedValue(oracleSOWithIncreasedCounter); + }); + + it('increases the counter correctly', async () => { + const record = await service.increaseCounter('so-id'); + + expect(record).toEqual({ ...oracleSO.attributes, id: 'so-id', counter: 2 }); + }); + + it('calls the unsecuredSavedObjectsClient.update method correctly', async () => { + await service.increaseCounter('so-id'); + + expect(unsecuredSavedObjectsClient.update).toHaveBeenCalledWith('cases-oracle', 'so-id', { + counter: 2, + }); + }); + }); }); diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts index eb9771f4b134b..85c5c309693f4 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts @@ -6,10 +6,10 @@ */ import { isEmpty } from 'lodash'; -import type { Logger, SavedObjectsClientContract } from '@kbn/core/server'; +import type { Logger, SavedObject, SavedObjectsClientContract } from '@kbn/core/server'; import { CASE_ORACLE_SAVED_OBJECT } from '../../../common/constants'; import { CryptoService } from './crypto_service'; -import type { OracleKey, OracleRecord } from './types'; +import type { OracleKey, OracleRecord, OracleRecordCreateRequest } from './types'; import { sortGroupDefinition } from './util'; type OracleRecordWithoutId = Omit; @@ -52,13 +52,58 @@ export class CasesOracleService { recordId ); - return { - id: oracleRecord.id, - counter: oracleRecord.attributes.counter, - caseIds: oracleRecord.attributes.caseIds, - ruleId: oracleRecord.attributes.ruleId, - createdAt: oracleRecord.attributes.createdAt, - updatedAt: oracleRecord.attributes.updatedAt, - }; + return this.getRecordResponse(oracleRecord); } + + public async createRecord(payload: OracleRecordCreateRequest): Promise { + const { ruleId, spaceId, owner, groupingDefinition } = payload; + const { caseIds } = payload; + const recordId = this.getRecordId({ ruleId, spaceId, owner, groupingDefinition }); + + this.log.debug(`Creating oracle record with ID: ${recordId}`); + + const oracleRecord = await this.unsecuredSavedObjectsClient.create( + CASE_ORACLE_SAVED_OBJECT, + { + counter: 1, + caseIds, + ruleId, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }, + { id: recordId } + ); + + return this.getRecordResponse(oracleRecord); + } + + public async increaseCounter(recordId: string): Promise { + const record = await this.getRecord(recordId); + const newCounter = record.counter + 1; + + this.log.debug( + `Increasing the counter of oracle record with ID: ${recordId} from ${record.counter} to ${newCounter}` + ); + + const oracleRecord = await this.unsecuredSavedObjectsClient.update( + CASE_ORACLE_SAVED_OBJECT, + recordId, + { counter: newCounter } + ); + + return this.getRecordResponse({ + ...oracleRecord, + attributes: { ...record, counter: newCounter }, + references: oracleRecord.references ?? [], + }); + } + + private getRecordResponse = (oracleRecord: SavedObject): OracleRecord => ({ + id: oracleRecord.id, + counter: oracleRecord.attributes.counter, + caseIds: oracleRecord.attributes.caseIds, + ruleId: oracleRecord.attributes.ruleId, + createdAt: oracleRecord.attributes.createdAt, + updatedAt: oracleRecord.attributes.updatedAt, + }); } diff --git a/x-pack/plugins/cases/server/connectors/cases/types.ts b/x-pack/plugins/cases/server/connectors/cases/types.ts index 915226d2b5afe..ceeea9d5831b3 100644 --- a/x-pack/plugins/cases/server/connectors/cases/types.ts +++ b/x-pack/plugins/cases/server/connectors/cases/types.ts @@ -31,3 +31,7 @@ export interface OracleRecord { createdAt: string; updatedAt: string; } + +export type OracleRecordCreateRequest = { + caseIds: string[]; +} & OracleKey; From 99552394106fd5902e00f0504afb6d8bd0078492 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Thu, 12 Oct 2023 11:52:35 +0300 Subject: [PATCH 09/40] Change grouping to record --- .../cases/cases_oracle_service.test.ts | 21 +++++----- .../connectors/cases/cases_oracle_service.ts | 13 +++---- .../connectors/cases/crypto_service.test.ts | 38 ++++++++++++------- .../server/connectors/cases/crypto_service.ts | 5 +++ .../cases/server/connectors/cases/types.ts | 2 +- .../cases/server/connectors/cases/util.ts | 38 ------------------- .../server/connectors/cases/utils.test.ts | 33 ---------------- 7 files changed, 47 insertions(+), 103 deletions(-) delete mode 100644 x-pack/plugins/cases/server/connectors/cases/util.ts delete mode 100644 x-pack/plugins/cases/server/connectors/cases/utils.test.ts diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts index 356a1b6df7815..705a942c0a27a 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts @@ -6,6 +6,7 @@ */ import { createHash } from 'node:crypto'; +import stringify from 'json-stable-stringify'; import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks'; import { loggerMock } from '@kbn/logging-mocks'; @@ -28,33 +29,33 @@ describe('CasesOracleService', () => { const ruleId = 'test-rule-id'; const spaceId = 'default'; const owner = 'cases'; - const groupingDefinition = 'host.ip=0.0.0.1'; + const grouping = { 'host.ip': '0.0.0.1' }; - const payload = `${ruleId}:${spaceId}:${owner}:${groupingDefinition}`; + const payload = `${ruleId}:${spaceId}:${owner}:${stringify(grouping)}`; const hash = createHash('sha256'); hash.update(payload); const hex = hash.digest('hex'); - expect(service.getRecordId({ ruleId, spaceId, owner, groupingDefinition })).toEqual(hex); + expect(service.getRecordId({ ruleId, spaceId, owner, grouping })).toEqual(hex); }); it('sorts the grouping definition correctly', async () => { const ruleId = 'test-rule-id'; const spaceId = 'default'; const owner = 'cases'; - const groupingDefinition = 'host.ip=0.0.0.1&agent.id=8a4f500d'; - const sortedGroupingDefinition = 'agent.id=8a4f500d&host.ip=0.0.0.1'; + const grouping = { 'host.ip': '0.0.0.1', 'agent.id': '8a4f500d' }; + const sortedGrouping = { 'agent.id': '8a4f500d', 'host.ip': '0.0.0.1' }; - const payload = `${ruleId}:${spaceId}:${owner}:${sortedGroupingDefinition}`; + const payload = `${ruleId}:${spaceId}:${owner}:${stringify(sortedGrouping)}`; const hash = createHash('sha256'); hash.update(payload); const hex = hash.digest('hex'); - expect(service.getRecordId({ ruleId, spaceId, owner, groupingDefinition })).toEqual(hex); + expect(service.getRecordId({ ruleId, spaceId, owner, grouping })).toEqual(hex); }); }); @@ -88,7 +89,7 @@ describe('CasesOracleService', () => { const ruleId = 'test-rule-id'; const spaceId = 'default'; const owner = 'cases'; - const groupingDefinition = 'host.ip=0.0.0.1'; + const grouping = { 'host.ip': '0.0.0.1' }; const caseIds = ['test-case-id']; const oracleSO = { @@ -114,7 +115,7 @@ describe('CasesOracleService', () => { ruleId, spaceId, owner, - groupingDefinition, + grouping, caseIds, }); @@ -126,7 +127,7 @@ describe('CasesOracleService', () => { ruleId, spaceId, owner, - groupingDefinition, + grouping, }; const id = service.getRecordId(keyParams); diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts index 85c5c309693f4..9595442cb78ef 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts @@ -10,7 +10,6 @@ import type { Logger, SavedObject, SavedObjectsClientContract } from '@kbn/core/ import { CASE_ORACLE_SAVED_OBJECT } from '../../../common/constants'; import { CryptoService } from './crypto_service'; import type { OracleKey, OracleRecord, OracleRecordCreateRequest } from './types'; -import { sortGroupDefinition } from './util'; type OracleRecordWithoutId = Omit; @@ -31,15 +30,15 @@ export class CasesOracleService { this.cryptoService = new CryptoService(); } - public getRecordId({ ruleId, spaceId, owner, groupingDefinition }: OracleKey): string { + public getRecordId({ ruleId, spaceId, owner, grouping }: OracleKey): string { const initialPayload = `${ruleId}:${spaceId}:${owner}`; - if (groupingDefinition == null || isEmpty(groupingDefinition)) { + if (grouping == null || isEmpty(grouping)) { return this.cryptoService.getHash(initialPayload); } - const sortedGroupingDefinition = sortGroupDefinition(groupingDefinition); - const payload = `${initialPayload}:${sortedGroupingDefinition}`; + const stringifiedAndSortedGrouping = this.cryptoService.stringifyDeterministically(grouping); + const payload = `${initialPayload}:${stringifiedAndSortedGrouping}`; return this.cryptoService.getHash(payload); } @@ -56,9 +55,9 @@ export class CasesOracleService { } public async createRecord(payload: OracleRecordCreateRequest): Promise { - const { ruleId, spaceId, owner, groupingDefinition } = payload; + const { ruleId, spaceId, owner, grouping } = payload; const { caseIds } = payload; - const recordId = this.getRecordId({ ruleId, spaceId, owner, groupingDefinition }); + const recordId = this.getRecordId({ ruleId, spaceId, owner, grouping }); this.log.debug(`Creating oracle record with ID: ${recordId}`); diff --git a/x-pack/plugins/cases/server/connectors/cases/crypto_service.test.ts b/x-pack/plugins/cases/server/connectors/cases/crypto_service.test.ts index 86c7ac860e73a..bf8a9f946ab58 100644 --- a/x-pack/plugins/cases/server/connectors/cases/crypto_service.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/crypto_service.test.ts @@ -16,26 +16,36 @@ describe('CryptoService', () => { service = new CryptoService(); }); - it('returns the sha256 of a payload correctly', async () => { - const payload = 'my payload'; - const hash = createHash('sha256'); + describe('getHash', () => { + it('returns the sha256 of a payload correctly', async () => { + const payload = 'my payload'; + const hash = createHash('sha256'); - hash.update(payload); + hash.update(payload); - const hex = hash.digest('hex'); + const hex = hash.digest('hex'); - expect(service.getHash(payload)).toEqual(hex); - }); + expect(service.getHash(payload)).toEqual(hex); + }); + + it('creates a new instance of the hash function on each call', async () => { + const payload = 'my payload'; + const hash = createHash('sha256'); - it('creates a new instance of the hash function on each call', async () => { - const payload = 'my payload'; - const hash = createHash('sha256'); + hash.update(payload); - hash.update(payload); + const hex = hash.digest('hex'); - const hex = hash.digest('hex'); + expect(service.getHash(payload)).toEqual(hex); + expect(service.getHash(payload)).toEqual(hex); + }); + }); - expect(service.getHash(payload)).toEqual(hex); - expect(service.getHash(payload)).toEqual(hex); + describe('stringifyDeterministically', () => { + it('deterministically stringifies an object', async () => { + expect( + service.stringifyDeterministically({ 'host.ip': '0.0.0.1', 'agent.id': '8a4f500d' }) + ).toEqual('{"agent.id":"8a4f500d","host.ip":"0.0.0.1"}'); + }); }); }); diff --git a/x-pack/plugins/cases/server/connectors/cases/crypto_service.ts b/x-pack/plugins/cases/server/connectors/cases/crypto_service.ts index ab56da14a9200..d35d94d87ebb2 100644 --- a/x-pack/plugins/cases/server/connectors/cases/crypto_service.ts +++ b/x-pack/plugins/cases/server/connectors/cases/crypto_service.ts @@ -6,6 +6,7 @@ */ import { createHash } from 'node:crypto'; +import stringify from 'json-stable-stringify'; export class CryptoService { public getHash(payload: string): string { @@ -14,4 +15,8 @@ export class CryptoService { hash.update(payload); return hash.digest('hex'); } + + public stringifyDeterministically(obj: Record) { + return stringify(obj); + } } diff --git a/x-pack/plugins/cases/server/connectors/cases/types.ts b/x-pack/plugins/cases/server/connectors/cases/types.ts index ceeea9d5831b3..1ac5bfebcf9c9 100644 --- a/x-pack/plugins/cases/server/connectors/cases/types.ts +++ b/x-pack/plugins/cases/server/connectors/cases/types.ts @@ -20,7 +20,7 @@ export interface OracleKey { ruleId: string; spaceId: string; owner: string; - groupingDefinition?: string; + grouping?: Record; } export interface OracleRecord { diff --git a/x-pack/plugins/cases/server/connectors/cases/util.ts b/x-pack/plugins/cases/server/connectors/cases/util.ts deleted file mode 100644 index ca9978f00fcf9..0000000000000 --- a/x-pack/plugins/cases/server/connectors/cases/util.ts +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -/** - * groupingDefinition is of the form - * =&=. - * Example: host.ip=0.0.0.1&host.name=A - * - * The function assumes that no duplicated field keys - * exist. Also it assumes the following special characters - * ":", "=", "&" are escaped property. For example, - * host.ip=0.0.0.1&host.ip=0.0.0.2 or host.ip=2001:db8::8a2e:370:7334 are not valid - */ - -export const sortGroupDefinition = (groupingDefinition: string): string => { - const fieldMap = new Map(); - const fields = groupingDefinition.split('&'); - - if (fields.length <= 1) { - return groupingDefinition; - } - - for (const field of fields) { - const [key, value] = field.split('='); - fieldMap.set(key, value); - } - - const sortedKeys = Array.from(fieldMap.keys()).sort(sortStings); - const sortedFields = sortedKeys.map((key) => `${key}=${fieldMap.get(key)}`); - - return sortedFields.join('&'); -}; - -const sortStings = (a: string, b: string) => String(a).localeCompare(b); diff --git a/x-pack/plugins/cases/server/connectors/cases/utils.test.ts b/x-pack/plugins/cases/server/connectors/cases/utils.test.ts deleted file mode 100644 index c5c03303d2f15..0000000000000 --- a/x-pack/plugins/cases/server/connectors/cases/utils.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { sortGroupDefinition } from './util'; - -describe('utils', () => { - describe('sortGroupDefinition', () => { - it('sorts a group definition correctly', async () => { - const groupingDefinition = - 'source.nat.ip=0.1.2.0&host.name=A&host.ip=0.0.0.1&agent.id=8a4f500d'; - - expect(sortGroupDefinition(groupingDefinition)).toBe( - 'agent.id=8a4f500d&host.ip=0.0.0.1&host.name=A&source.nat.ip=0.1.2.0' - ); - }); - - it('returns the grouping definition if there is only one field', async () => { - const groupingDefinition = 'host.name=A'; - - expect(sortGroupDefinition(groupingDefinition)).toBe('host.name=A'); - }); - - it('returns an empty string if the grouping definition is an empty string', async () => { - const groupingDefinition = ''; - - expect(sortGroupDefinition(groupingDefinition)).toBe(''); - }); - }); -}); From 58bc3d6256f542e933a6aa7b17b2e3943519a4fb Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Thu, 12 Oct 2023 12:11:58 +0300 Subject: [PATCH 10/40] Make the rule ID optional in the key --- .../cases/cases_oracle_service.test.ts | 85 +++++++++++++++++++ .../connectors/cases/cases_oracle_service.ts | 17 ++-- .../server/connectors/cases/crypto_service.ts | 6 +- .../cases/server/connectors/cases/types.ts | 2 +- 4 files changed, 101 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts index 705a942c0a27a..738fb89c5b78c 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts @@ -12,6 +12,7 @@ import { loggerMock } from '@kbn/logging-mocks'; import { CasesOracleService } from './cases_oracle_service'; import { CASE_ORACLE_SAVED_OBJECT } from '../../../common/constants'; +import { isEmpty, set } from 'lodash'; describe('CasesOracleService', () => { const unsecuredSavedObjectsClient = savedObjectsClientMock.create(); @@ -57,6 +58,90 @@ describe('CasesOracleService', () => { expect(service.getRecordId({ ruleId, spaceId, owner, grouping })).toEqual(hex); }); + + it('return the record ID correctly without grouping', async () => { + const ruleId = 'test-rule-id'; + const spaceId = 'default'; + const owner = 'cases'; + + const payload = `${ruleId}:${spaceId}:${owner}`; + const hash = createHash('sha256'); + + hash.update(payload); + + const hex = hash.digest('hex'); + + expect(service.getRecordId({ ruleId, spaceId, owner })).toEqual(hex); + }); + + it('return the record ID correctly with empty grouping', async () => { + const ruleId = 'test-rule-id'; + const spaceId = 'default'; + const owner = 'cases'; + const grouping = {}; + + const payload = `${ruleId}:${spaceId}:${owner}:${stringify(grouping)}`; + const hash = createHash('sha256'); + + hash.update(payload); + + const hex = hash.digest('hex'); + + expect(service.getRecordId({ ruleId, spaceId, owner, grouping })).toEqual(hex); + }); + + it('return the record ID correctly without rule', async () => { + const spaceId = 'default'; + const owner = 'cases'; + const grouping = { 'host.ip': '0.0.0.1' }; + + const payload = `${spaceId}:${owner}:${stringify(grouping)}`; + const hash = createHash('sha256'); + + hash.update(payload); + + const hex = hash.digest('hex'); + + expect(service.getRecordId({ spaceId, owner, grouping })).toEqual(hex); + }); + + it('throws an error when the ruleId and the grouping is missing', async () => { + const spaceId = 'default'; + const owner = 'cases'; + + expect(() => service.getRecordId({ spaceId, owner })).toThrowErrorMatchingInlineSnapshot( + `"ruleID or grouping is required"` + ); + }); + + it.each(['ruleId', 'spaceId', 'owner'])( + 'return the record ID correctly with empty string for %s', + async (key) => { + const getPayloadValue = (value: string) => (isEmpty(value) ? '' : `${value}:`); + + const params = { + ruleId: 'test-rule-id', + spaceId: 'default', + owner: 'cases', + }; + + const grouping = { 'host.ip': '0.0.0.1' }; + + set(params, key, ''); + + const payload = `${getPayloadValue(params.ruleId)}${getPayloadValue( + params.spaceId + )}${getPayloadValue(params.owner)}${stringify(grouping)}`; + + const hash = createHash('sha256'); + + hash.update(payload); + + const hex = hash.digest('hex'); + + expect(service.getRecordId({ ...params, grouping })).toEqual(hex); + } + ); }); describe('getRecord', () => { diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts index 9595442cb78ef..78983d953db8a 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { isEmpty } from 'lodash'; import type { Logger, SavedObject, SavedObjectsClientContract } from '@kbn/core/server'; import { CASE_ORACLE_SAVED_OBJECT } from '../../../common/constants'; import { CryptoService } from './crypto_service'; @@ -31,14 +30,18 @@ export class CasesOracleService { } public getRecordId({ ruleId, spaceId, owner, grouping }: OracleKey): string { - const initialPayload = `${ruleId}:${spaceId}:${owner}`; - - if (grouping == null || isEmpty(grouping)) { - return this.cryptoService.getHash(initialPayload); + if (grouping == null && ruleId == null) { + throw new Error('ruleID or grouping is required'); } - const stringifiedAndSortedGrouping = this.cryptoService.stringifyDeterministically(grouping); - const payload = `${initialPayload}:${stringifiedAndSortedGrouping}`; + const payload = [ + ruleId, + spaceId, + owner, + this.cryptoService.stringifyDeterministically(grouping), + ] + .filter(Boolean) + .join(':'); return this.cryptoService.getHash(payload); } diff --git a/x-pack/plugins/cases/server/connectors/cases/crypto_service.ts b/x-pack/plugins/cases/server/connectors/cases/crypto_service.ts index d35d94d87ebb2..e35b4e51ed1b4 100644 --- a/x-pack/plugins/cases/server/connectors/cases/crypto_service.ts +++ b/x-pack/plugins/cases/server/connectors/cases/crypto_service.ts @@ -16,7 +16,11 @@ export class CryptoService { return hash.digest('hex'); } - public stringifyDeterministically(obj: Record) { + public stringifyDeterministically(obj?: Record): string | null { + if (obj == null) { + return null; + } + return stringify(obj); } } diff --git a/x-pack/plugins/cases/server/connectors/cases/types.ts b/x-pack/plugins/cases/server/connectors/cases/types.ts index 1ac5bfebcf9c9..bb0c9cc6b7160 100644 --- a/x-pack/plugins/cases/server/connectors/cases/types.ts +++ b/x-pack/plugins/cases/server/connectors/cases/types.ts @@ -17,7 +17,7 @@ export type CasesConnectorSecrets = TypeOf; export type CasesConnectorParams = TypeOf; export interface OracleKey { - ruleId: string; + ruleId?: string; spaceId: string; owner: string; grouping?: Record; From 581819a5ee4b43a3dec7f07a993fcc4c71c26bd2 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Fri, 13 Oct 2023 13:38:49 +0300 Subject: [PATCH 11/40] Better types --- .../cases/cases_oracle_service.test.ts | 54 ++++++++----------- .../connectors/cases/cases_oracle_service.ts | 17 +++--- .../cases/server/connectors/cases/types.ts | 25 ++++++--- 3 files changed, 47 insertions(+), 49 deletions(-) diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts index 738fb89c5b78c..d21a2d3e0fe6a 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts @@ -109,6 +109,7 @@ describe('CasesOracleService', () => { const spaceId = 'default'; const owner = 'cases'; + // @ts-expect-error: ruleId and grouping are omitted for testing expect(() => service.getRecordId({ spaceId, owner })).toThrowErrorMatchingInlineSnapshot( `"ruleID or grouping is required"` ); @@ -145,13 +146,16 @@ describe('CasesOracleService', () => { }); describe('getRecord', () => { + const cases = [{ id: 'test-case-id' }]; + const rules = [{ id: 'test-rule-id' }]; + const oracleSO = { id: 'so-id', version: 'so-version', attributes: { counter: 1, - caseIds: ['test-case-id'], - ruleId: 'test-rule-id', + cases, + rules, createdAt: '2023-10-10T10:23:42.769Z', updatedAt: '2023-10-10T10:23:42.769Z', }, @@ -171,19 +175,16 @@ describe('CasesOracleService', () => { }); describe('createRecord', () => { - const ruleId = 'test-rule-id'; - const spaceId = 'default'; - const owner = 'cases'; - const grouping = { 'host.ip': '0.0.0.1' }; - const caseIds = ['test-case-id']; + const cases = [{ id: 'test-case-id' }]; + const rules = [{ id: 'test-rule-id' }]; const oracleSO = { id: 'so-id', version: 'so-version', attributes: { counter: 1, - caseIds, - ruleId, + cases, + rules, createdAt: '2023-10-10T10:23:42.769Z', updatedAt: '2023-10-10T10:23:42.769Z', }, @@ -196,39 +197,23 @@ describe('CasesOracleService', () => { }); it('creates a record correctly', async () => { - const record = await service.createRecord({ - ruleId, - spaceId, - owner, - grouping, - caseIds, - }); + const record = await service.createRecord('so-id', { cases, rules }); expect(record).toEqual({ ...oracleSO.attributes, id: 'so-id' }); }); it('calls the unsecuredSavedObjectsClient.create method correctly', async () => { - const keyParams = { - ruleId, - spaceId, - owner, - grouping, - }; - - const id = service.getRecordId(keyParams); - - await service.createRecord({ - ...keyParams, - caseIds, - }); + const id = 'so-id'; + + await service.createRecord(id, { cases, rules }); expect(unsecuredSavedObjectsClient.create).toHaveBeenCalledWith( 'cases-oracle', { - caseIds: ['test-case-id'], + cases, counter: 1, createdAt: expect.anything(), - ruleId: 'test-rule-id', + rules, updatedAt: expect.anything(), }, { id } @@ -237,13 +222,16 @@ describe('CasesOracleService', () => { }); describe('increaseCounter', () => { + const cases = [{ id: 'test-case-id' }]; + const rules = [{ id: 'test-rule-id' }]; + const oracleSO = { id: 'so-id', version: 'so-version', attributes: { counter: 1, - caseIds: ['test-case-id'], - ruleId: 'test-rule-id', + cases, + rules, createdAt: '2023-10-10T10:23:42.769Z', updatedAt: '2023-10-10T10:23:42.769Z', }, diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts index 78983d953db8a..af4136fd4b88c 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts @@ -57,10 +57,11 @@ export class CasesOracleService { return this.getRecordResponse(oracleRecord); } - public async createRecord(payload: OracleRecordCreateRequest): Promise { - const { ruleId, spaceId, owner, grouping } = payload; - const { caseIds } = payload; - const recordId = this.getRecordId({ ruleId, spaceId, owner, grouping }); + public async createRecord( + recordId: string, + payload: OracleRecordCreateRequest + ): Promise { + const { cases, rules } = payload; this.log.debug(`Creating oracle record with ID: ${recordId}`); @@ -68,8 +69,8 @@ export class CasesOracleService { CASE_ORACLE_SAVED_OBJECT, { counter: 1, - caseIds, - ruleId, + cases, + rules, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), }, @@ -103,8 +104,8 @@ export class CasesOracleService { private getRecordResponse = (oracleRecord: SavedObject): OracleRecord => ({ id: oracleRecord.id, counter: oracleRecord.attributes.counter, - caseIds: oracleRecord.attributes.caseIds, - ruleId: oracleRecord.attributes.ruleId, + cases: oracleRecord.attributes.cases, + rules: oracleRecord.attributes.rules, createdAt: oracleRecord.attributes.createdAt, updatedAt: oracleRecord.attributes.updatedAt, }); diff --git a/x-pack/plugins/cases/server/connectors/cases/types.ts b/x-pack/plugins/cases/server/connectors/cases/types.ts index bb0c9cc6b7160..e47fa60ead6fb 100644 --- a/x-pack/plugins/cases/server/connectors/cases/types.ts +++ b/x-pack/plugins/cases/server/connectors/cases/types.ts @@ -5,6 +5,7 @@ * 2.0. */ +import type { ExclusiveUnion } from '@elastic/eui'; import type { TypeOf } from '@kbn/config-schema'; import type { CasesConnectorConfigSchema, @@ -16,22 +17,30 @@ export type CasesConnectorConfig = TypeOf; export type CasesConnectorSecrets = TypeOf; export type CasesConnectorParams = TypeOf; -export interface OracleKey { - ruleId?: string; +type Optional = Pick, K> & Omit; + +interface OracleKeyAllRequired { + ruleId: string; spaceId: string; owner: string; - grouping?: Record; + grouping: Record; } +type OracleKeyWithRequiredRule = Optional; +type OracleKeyWithRequiredGrouping = Optional; + +export type OracleKey = ExclusiveUnion; + export interface OracleRecord { id: string; counter: number; - caseIds: string[]; - ruleId: string; + cases: Array<{ id: string }>; + rules: Array<{ id: string }>; createdAt: string; updatedAt: string; } -export type OracleRecordCreateRequest = { - caseIds: string[]; -} & OracleKey; +export interface OracleRecordCreateRequest { + cases: Array<{ id: string }>; + rules: Array<{ id: string }>; +} From 449a1b7e8409897f27b92f7cd59949deb23d67c0 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Fri, 13 Oct 2023 13:44:12 +0300 Subject: [PATCH 12/40] Add version when updating --- .../cases/cases_oracle_service.test.ts | 24 +++++++++++++------ .../connectors/cases/cases_oracle_service.ts | 20 +++++++++------- .../cases/server/connectors/cases/types.ts | 3 ++- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts index d21a2d3e0fe6a..e297b438f047a 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts @@ -170,7 +170,7 @@ describe('CasesOracleService', () => { it('gets a record correctly', async () => { const record = await service.getRecord('so-id'); - expect(record).toEqual({ ...oracleSO.attributes, id: 'so-id' }); + expect(record).toEqual({ ...oracleSO.attributes, id: 'so-id', version: 'so-version' }); }); }); @@ -199,7 +199,7 @@ describe('CasesOracleService', () => { it('creates a record correctly', async () => { const record = await service.createRecord('so-id', { cases, rules }); - expect(record).toEqual({ ...oracleSO.attributes, id: 'so-id' }); + expect(record).toEqual({ ...oracleSO.attributes, id: 'so-id', version: 'so-version' }); }); it('calls the unsecuredSavedObjectsClient.create method correctly', async () => { @@ -214,7 +214,7 @@ describe('CasesOracleService', () => { counter: 1, createdAt: expect.anything(), rules, - updatedAt: expect.anything(), + updatedAt: null, }, { id } ); @@ -252,15 +252,25 @@ describe('CasesOracleService', () => { it('increases the counter correctly', async () => { const record = await service.increaseCounter('so-id'); - expect(record).toEqual({ ...oracleSO.attributes, id: 'so-id', counter: 2 }); + expect(record).toEqual({ + ...oracleSO.attributes, + id: 'so-id', + version: 'so-version', + counter: 2, + }); }); it('calls the unsecuredSavedObjectsClient.update method correctly', async () => { await service.increaseCounter('so-id'); - expect(unsecuredSavedObjectsClient.update).toHaveBeenCalledWith('cases-oracle', 'so-id', { - counter: 2, - }); + expect(unsecuredSavedObjectsClient.update).toHaveBeenCalledWith( + 'cases-oracle', + 'so-id', + { + counter: 2, + }, + { version: 'so-version' } + ); }); }); }); diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts index af4136fd4b88c..a6c1febfbcfe3 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts @@ -10,7 +10,7 @@ import { CASE_ORACLE_SAVED_OBJECT } from '../../../common/constants'; import { CryptoService } from './crypto_service'; import type { OracleKey, OracleRecord, OracleRecordCreateRequest } from './types'; -type OracleRecordWithoutId = Omit; +type OracleRecordAttributes = Omit; export class CasesOracleService { private readonly log: Logger; @@ -49,7 +49,7 @@ export class CasesOracleService { public async getRecord(recordId: string): Promise { this.log.debug(`Getting oracle record with ID: ${recordId}`); - const oracleRecord = await this.unsecuredSavedObjectsClient.get( + const oracleRecord = await this.unsecuredSavedObjectsClient.get( CASE_ORACLE_SAVED_OBJECT, recordId ); @@ -65,14 +65,14 @@ export class CasesOracleService { this.log.debug(`Creating oracle record with ID: ${recordId}`); - const oracleRecord = await this.unsecuredSavedObjectsClient.create( + const oracleRecord = await this.unsecuredSavedObjectsClient.create( CASE_ORACLE_SAVED_OBJECT, { counter: 1, cases, rules, createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), + updatedAt: null, }, { id: recordId } ); @@ -81,17 +81,18 @@ export class CasesOracleService { } public async increaseCounter(recordId: string): Promise { - const record = await this.getRecord(recordId); + const { id: _, version, ...record } = await this.getRecord(recordId); const newCounter = record.counter + 1; this.log.debug( `Increasing the counter of oracle record with ID: ${recordId} from ${record.counter} to ${newCounter}` ); - const oracleRecord = await this.unsecuredSavedObjectsClient.update( + const oracleRecord = await this.unsecuredSavedObjectsClient.update( CASE_ORACLE_SAVED_OBJECT, recordId, - { counter: newCounter } + { counter: newCounter }, + { version } ); return this.getRecordResponse({ @@ -101,8 +102,11 @@ export class CasesOracleService { }); } - private getRecordResponse = (oracleRecord: SavedObject): OracleRecord => ({ + private getRecordResponse = ( + oracleRecord: SavedObject + ): OracleRecord => ({ id: oracleRecord.id, + version: oracleRecord.version ?? '', counter: oracleRecord.attributes.counter, cases: oracleRecord.attributes.cases, rules: oracleRecord.attributes.rules, diff --git a/x-pack/plugins/cases/server/connectors/cases/types.ts b/x-pack/plugins/cases/server/connectors/cases/types.ts index e47fa60ead6fb..80fdb2734a123 100644 --- a/x-pack/plugins/cases/server/connectors/cases/types.ts +++ b/x-pack/plugins/cases/server/connectors/cases/types.ts @@ -37,7 +37,8 @@ export interface OracleRecord { cases: Array<{ id: string }>; rules: Array<{ id: string }>; createdAt: string; - updatedAt: string; + updatedAt: string | null; + version: string; } export interface OracleRecordCreateRequest { From c210a0c9cefc167ac1c1339757ec8494729abb4e Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Fri, 13 Oct 2023 17:29:16 +0300 Subject: [PATCH 13/40] Improve types --- x-pack/plugins/cases/server/connectors/cases/types.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/cases/server/connectors/cases/types.ts b/x-pack/plugins/cases/server/connectors/cases/types.ts index 80fdb2734a123..37616718bd694 100644 --- a/x-pack/plugins/cases/server/connectors/cases/types.ts +++ b/x-pack/plugins/cases/server/connectors/cases/types.ts @@ -26,10 +26,10 @@ interface OracleKeyAllRequired { grouping: Record; } -type OracleKeyWithRequiredRule = Optional; -type OracleKeyWithRequiredGrouping = Optional; +type OracleKeyWithOptionalKey = Optional; +type OracleKeyWithOptionalGrouping = Optional; -export type OracleKey = ExclusiveUnion; +export type OracleKey = ExclusiveUnion; export interface OracleRecord { id: string; From c802637eefd6482c9d8c246e1269520595384ddd Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Fri, 13 Oct 2023 17:33:25 +0300 Subject: [PATCH 14/40] Fix tests --- .../migrations/group2/check_registered_types.test.ts | 1 + .../tests/actions/check_registered_connector_types.ts | 1 + .../test_suites/task_manager/check_registered_task_types.ts | 1 + 3 files changed, 3 insertions(+) diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts index 26ab99d41633d..ccf082e545949 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts @@ -74,6 +74,7 @@ describe('checking migration metadata changes on all registered SO types', () => "cases-comments": "5cb0a421588831c2a950e50f486048b8aabbae25", "cases-configure": "44ed7b8e0f44df39516b8870589b89e32224d2bf", "cases-connector-mappings": "f9d1ac57e484e69506c36a8051e4d61f4a8cfd25", + "cases-oracle": "5c8eafc1cff28c72953d896f0bf9a41bc4d887d0", "cases-telemetry": "f219eb7e26772884342487fc9602cfea07b3cedc", "cases-user-actions": "483f10db9b3bd1617948d7032a98b7791bf87414", "config": "179b3e2bc672626aafce3cf92093a113f456af38", diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/actions/check_registered_connector_types.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/actions/check_registered_connector_types.ts index 13fcc069a4df3..243e12a92e938 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/actions/check_registered_connector_types.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/actions/check_registered_connector_types.ts @@ -50,6 +50,7 @@ export default function createRegisteredConnectorTypeTests({ getService }: FtrPr '.opsgenie', '.gen-ai', '.bedrock', + '.cases', ].sort() ); }); diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index 17154d9a254c4..f0b75d3abdfae 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -51,6 +51,7 @@ export default function ({ getService }: FtrProviderContext) { 'Synthetics:Clean-Up-Package-Policies', 'UPTIME:SyntheticsService:Sync-Saved-Monitor-Objects', 'actions:.bedrock', + 'actions:.cases', 'actions:.cases-webhook', 'actions:.d3security', 'actions:.email', From c120c97566f7ee11a29c13258546c1d0d2e8c766 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Mon, 16 Oct 2023 11:47:58 +0300 Subject: [PATCH 15/40] Add model version and improve mapping --- .../cases/cases_oracle_service.test.ts | 11 +++++- .../connectors/cases/cases_oracle_service.ts | 4 +- .../cases/server/connectors/cases/types.ts | 2 + .../server/saved_object_types/cases_oracle.ts | 39 ++++++++++++++++--- 4 files changed, 47 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts index e297b438f047a..5d1261f9d9539 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts @@ -148,6 +148,7 @@ describe('CasesOracleService', () => { describe('getRecord', () => { const cases = [{ id: 'test-case-id' }]; const rules = [{ id: 'test-rule-id' }]; + const grouping = { 'host.ip': '0.0.0.1', 'agent.id': '8a4f500d' }; const oracleSO = { id: 'so-id', @@ -156,6 +157,7 @@ describe('CasesOracleService', () => { counter: 1, cases, rules, + grouping, createdAt: '2023-10-10T10:23:42.769Z', updatedAt: '2023-10-10T10:23:42.769Z', }, @@ -177,6 +179,7 @@ describe('CasesOracleService', () => { describe('createRecord', () => { const cases = [{ id: 'test-case-id' }]; const rules = [{ id: 'test-rule-id' }]; + const grouping = { 'host.ip': '0.0.0.1', 'agent.id': '8a4f500d' }; const oracleSO = { id: 'so-id', @@ -185,6 +188,7 @@ describe('CasesOracleService', () => { counter: 1, cases, rules, + grouping, createdAt: '2023-10-10T10:23:42.769Z', updatedAt: '2023-10-10T10:23:42.769Z', }, @@ -197,7 +201,7 @@ describe('CasesOracleService', () => { }); it('creates a record correctly', async () => { - const record = await service.createRecord('so-id', { cases, rules }); + const record = await service.createRecord('so-id', { cases, rules, grouping }); expect(record).toEqual({ ...oracleSO.attributes, id: 'so-id', version: 'so-version' }); }); @@ -205,7 +209,7 @@ describe('CasesOracleService', () => { it('calls the unsecuredSavedObjectsClient.create method correctly', async () => { const id = 'so-id'; - await service.createRecord(id, { cases, rules }); + await service.createRecord(id, { cases, rules, grouping }); expect(unsecuredSavedObjectsClient.create).toHaveBeenCalledWith( 'cases-oracle', @@ -214,6 +218,7 @@ describe('CasesOracleService', () => { counter: 1, createdAt: expect.anything(), rules, + grouping, updatedAt: null, }, { id } @@ -224,6 +229,7 @@ describe('CasesOracleService', () => { describe('increaseCounter', () => { const cases = [{ id: 'test-case-id' }]; const rules = [{ id: 'test-rule-id' }]; + const grouping = { 'host.ip': '0.0.0.1', 'agent.id': '8a4f500d' }; const oracleSO = { id: 'so-id', @@ -232,6 +238,7 @@ describe('CasesOracleService', () => { counter: 1, cases, rules, + grouping, createdAt: '2023-10-10T10:23:42.769Z', updatedAt: '2023-10-10T10:23:42.769Z', }, diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts index a6c1febfbcfe3..816df7719f036 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts @@ -61,7 +61,7 @@ export class CasesOracleService { recordId: string, payload: OracleRecordCreateRequest ): Promise { - const { cases, rules } = payload; + const { cases, rules, grouping } = payload; this.log.debug(`Creating oracle record with ID: ${recordId}`); @@ -71,6 +71,7 @@ export class CasesOracleService { counter: 1, cases, rules, + grouping, createdAt: new Date().toISOString(), updatedAt: null, }, @@ -109,6 +110,7 @@ export class CasesOracleService { version: oracleRecord.version ?? '', counter: oracleRecord.attributes.counter, cases: oracleRecord.attributes.cases, + grouping: oracleRecord.attributes.grouping, rules: oracleRecord.attributes.rules, createdAt: oracleRecord.attributes.createdAt, updatedAt: oracleRecord.attributes.updatedAt, diff --git a/x-pack/plugins/cases/server/connectors/cases/types.ts b/x-pack/plugins/cases/server/connectors/cases/types.ts index 37616718bd694..373c24ed7e690 100644 --- a/x-pack/plugins/cases/server/connectors/cases/types.ts +++ b/x-pack/plugins/cases/server/connectors/cases/types.ts @@ -35,6 +35,7 @@ export interface OracleRecord { id: string; counter: number; cases: Array<{ id: string }>; + grouping: Record; rules: Array<{ id: string }>; createdAt: string; updatedAt: string | null; @@ -44,4 +45,5 @@ export interface OracleRecord { export interface OracleRecordCreateRequest { cases: Array<{ id: string }>; rules: Array<{ id: string }>; + grouping: Record; } diff --git a/x-pack/plugins/cases/server/saved_object_types/cases_oracle.ts b/x-pack/plugins/cases/server/saved_object_types/cases_oracle.ts index 2eea605a16726..3419b607d0276 100644 --- a/x-pack/plugins/cases/server/saved_object_types/cases_oracle.ts +++ b/x-pack/plugins/cases/server/saved_object_types/cases_oracle.ts @@ -7,6 +7,7 @@ import type { SavedObjectsType } from '@kbn/core/server'; import { ALERTING_CASES_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; +import { schema } from '@kbn/config-schema'; import { CASE_ORACLE_SAVED_OBJECT } from '../../common/constants'; export const casesOracleSavedObjectType: SavedObjectsType = { @@ -20,8 +21,12 @@ export const casesOracleSavedObjectType: SavedObjectsType = { mappings: { dynamic: false, properties: { - caseIds: { - type: 'keyword', + cases: { + properties: { + id: { + type: 'keyword', + }, + }, }, counter: { type: 'unsigned_long', @@ -30,16 +35,38 @@ export const casesOracleSavedObjectType: SavedObjectsType = { type: 'date', }, /* - grouping_definition: { - type: 'keyword', + grouping: { + type: 'flattened', }, */ - ruleId: { - type: 'keyword', + rules: { + properties: { + id: { + type: 'keyword', + }, + }, }, updatedAt: { type: 'date', }, }, }, + management: { + importableAndExportable: false, + }, + modelVersions: { + '1': { + changes: [], + schemas: { + create: schema.object({ + cases: schema.arrayOf(schema.object({ id: schema.string() })), + counter: schema.number(), + createdAt: schema.string(), + grouping: schema.recordOf(schema.string(), schema.any()), + rules: schema.arrayOf(schema.object({ id: schema.string() })), + updatedAt: schema.string(), + }), + }, + }, + }, }; From 043a9fe732eadff2e2c95abf37418d908ffab13c Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Mon, 16 Oct 2023 13:30:03 +0300 Subject: [PATCH 16/40] Fix tests --- .../migrations/group2/check_registered_types.test.ts | 2 +- .../saved_objects/migrations/group3/type_registrations.test.ts | 1 + .../saved_objects/migrations/group5/dot_kibana_split.test.ts | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts index 571cba0bf3f48..1653975b5a40f 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts @@ -74,7 +74,7 @@ describe('checking migration metadata changes on all registered SO types', () => "cases-comments": "5cb0a421588831c2a950e50f486048b8aabbae25", "cases-configure": "44ed7b8e0f44df39516b8870589b89e32224d2bf", "cases-connector-mappings": "f9d1ac57e484e69506c36a8051e4d61f4a8cfd25", - "cases-oracle": "5c8eafc1cff28c72953d896f0bf9a41bc4d887d0", + "cases-oracle": "afd99cd22b5551ac336b7c0f30f9ee31aa2b9f20", "cases-telemetry": "f219eb7e26772884342487fc9602cfea07b3cedc", "cases-user-actions": "483f10db9b3bd1617948d7032a98b7791bf87414", "config": "179b3e2bc672626aafce3cf92093a113f456af38", diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts index 2cef3801868bd..cf8461f07713b 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts @@ -33,6 +33,7 @@ const previouslyRegisteredTypes = [ 'cases-comments', 'cases-configure', 'cases-connector-mappings', + 'cases-oracle', 'cases-sub-case', 'cases-user-actions', 'cases-telemetry', diff --git a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts index c39ceaf30da69..19856c39f8f86 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts @@ -195,6 +195,7 @@ describe('split .kibana index into multiple system indices', () => { "cases-comments", "cases-configure", "cases-connector-mappings", + "cases-oracle", "cases-telemetry", "cases-user-actions", "config", From a9db13fe74e73360f33f22909c6ba45db98e492c Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Mon, 16 Oct 2023 14:48:17 +0300 Subject: [PATCH 17/40] Fix mapping test --- .../current_mappings.json | 220 ++++++++---------- 1 file changed, 100 insertions(+), 120 deletions(-) diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 3f696cb38e871..4ebb8065dbe1d 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -1465,26 +1465,6 @@ "dynamic": false, "properties": {} }, - "cases-oracle": { - "dynamic": false, - "properties": { - "case_ids": { - "type": "keyword" - }, - "counter": { - "type": "unsigned_long" - }, - "created_at": { - "type": "date" - }, - "rule_id": { - "type": "keyword" - }, - "updated_at": { - "type": "date" - } - } - }, "infrastructure-monitoring-log-view": { "dynamic": false, "properties": { @@ -2614,106 +2594,6 @@ "dynamic": false, "properties": {} }, - "infrastructure-ui-source": { - "dynamic": false, - "properties": {} - }, - "inventory-view": { - "dynamic": false, - "properties": {} - }, - "metrics-explorer-view": { - "dynamic": false, - "properties": {} - }, - "upgrade-assistant-reindex-operation": { - "dynamic": false, - "properties": { - "indexName": { - "type": "keyword" - }, - "status": { - "type": "integer" - } - } - }, - "upgrade-assistant-ml-upgrade-operation": { - "dynamic": false, - "properties": { - "snapshotId": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - }, - "monitoring-telemetry": { - "properties": { - "reportedClusterUuids": { - "type": "keyword" - } - } - }, - "apm-telemetry": { - "dynamic": false, - "properties": {} - }, - "apm-server-schema": { - "properties": { - "schemaJson": { - "type": "text", - "index": false - } - } - }, - "apm-service-group": { - "properties": { - "groupName": { - "type": "keyword" - }, - "kuery": { - "type": "text" - }, - "description": { - "type": "text" - }, - "color": { - "type": "text" - } - } - }, - "apm-custom-dashboards": { - "properties": { - "dashboardSavedObjectId": { - "type": "keyword" - }, - "kuery": { - "type": "text" - }, - "serviceEnvironmentFilterEnabled": { - "type": "boolean" - }, - "serviceNameFilterEnabled": { - "type": "boolean" - } - } - }, - "enterprise_search_telemetry": { - "dynamic": false, - "properties": {} - }, - "app_search_telemetry": { - "dynamic": false, - "properties": {} - }, - "workplace_search_telemetry": { - "dynamic": false, - "properties": {} - }, "siem-ui-timeline-note": { "properties": { "eventId": { @@ -3175,5 +3055,105 @@ "index": false } } + }, + "infrastructure-ui-source": { + "dynamic": false, + "properties": {} + }, + "inventory-view": { + "dynamic": false, + "properties": {} + }, + "metrics-explorer-view": { + "dynamic": false, + "properties": {} + }, + "upgrade-assistant-reindex-operation": { + "dynamic": false, + "properties": { + "indexName": { + "type": "keyword" + }, + "status": { + "type": "integer" + } + } + }, + "upgrade-assistant-ml-upgrade-operation": { + "dynamic": false, + "properties": { + "snapshotId": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "monitoring-telemetry": { + "properties": { + "reportedClusterUuids": { + "type": "keyword" + } + } + }, + "apm-telemetry": { + "dynamic": false, + "properties": {} + }, + "apm-server-schema": { + "properties": { + "schemaJson": { + "type": "text", + "index": false + } + } + }, + "apm-service-group": { + "properties": { + "groupName": { + "type": "keyword" + }, + "kuery": { + "type": "text" + }, + "description": { + "type": "text" + }, + "color": { + "type": "text" + } + } + }, + "apm-custom-dashboards": { + "properties": { + "dashboardSavedObjectId": { + "type": "keyword" + }, + "kuery": { + "type": "text" + }, + "serviceEnvironmentFilterEnabled": { + "type": "boolean" + }, + "serviceNameFilterEnabled": { + "type": "boolean" + } + } + }, + "enterprise_search_telemetry": { + "dynamic": false, + "properties": {} + }, + "app_search_telemetry": { + "dynamic": false, + "properties": {} + }, + "workplace_search_telemetry": { + "dynamic": false, + "properties": {} } } From 32af9e37d508890abf779df1ad4731cff222c711 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 16 Oct 2023 12:11:13 +0000 Subject: [PATCH 18/40] [CI] Auto-commit changed files from 'node scripts/check_mappings_update --fix' --- .../current_mappings.json | 228 ++++++++++-------- 1 file changed, 128 insertions(+), 100 deletions(-) diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 4ebb8065dbe1d..146b67837d563 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -1465,6 +1465,34 @@ "dynamic": false, "properties": {} }, + "cases-oracle": { + "dynamic": false, + "properties": { + "cases": { + "properties": { + "id": { + "type": "keyword" + } + } + }, + "counter": { + "type": "unsigned_long" + }, + "createdAt": { + "type": "date" + }, + "rules": { + "properties": { + "id": { + "type": "keyword" + } + } + }, + "updatedAt": { + "type": "date" + } + } + }, "infrastructure-monitoring-log-view": { "dynamic": false, "properties": { @@ -2594,6 +2622,106 @@ "dynamic": false, "properties": {} }, + "infrastructure-ui-source": { + "dynamic": false, + "properties": {} + }, + "inventory-view": { + "dynamic": false, + "properties": {} + }, + "metrics-explorer-view": { + "dynamic": false, + "properties": {} + }, + "upgrade-assistant-reindex-operation": { + "dynamic": false, + "properties": { + "indexName": { + "type": "keyword" + }, + "status": { + "type": "integer" + } + } + }, + "upgrade-assistant-ml-upgrade-operation": { + "dynamic": false, + "properties": { + "snapshotId": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "monitoring-telemetry": { + "properties": { + "reportedClusterUuids": { + "type": "keyword" + } + } + }, + "apm-telemetry": { + "dynamic": false, + "properties": {} + }, + "apm-server-schema": { + "properties": { + "schemaJson": { + "type": "text", + "index": false + } + } + }, + "apm-service-group": { + "properties": { + "groupName": { + "type": "keyword" + }, + "kuery": { + "type": "text" + }, + "description": { + "type": "text" + }, + "color": { + "type": "text" + } + } + }, + "apm-custom-dashboards": { + "properties": { + "dashboardSavedObjectId": { + "type": "keyword" + }, + "kuery": { + "type": "text" + }, + "serviceEnvironmentFilterEnabled": { + "type": "boolean" + }, + "serviceNameFilterEnabled": { + "type": "boolean" + } + } + }, + "enterprise_search_telemetry": { + "dynamic": false, + "properties": {} + }, + "app_search_telemetry": { + "dynamic": false, + "properties": {} + }, + "workplace_search_telemetry": { + "dynamic": false, + "properties": {} + }, "siem-ui-timeline-note": { "properties": { "eventId": { @@ -3055,105 +3183,5 @@ "index": false } } - }, - "infrastructure-ui-source": { - "dynamic": false, - "properties": {} - }, - "inventory-view": { - "dynamic": false, - "properties": {} - }, - "metrics-explorer-view": { - "dynamic": false, - "properties": {} - }, - "upgrade-assistant-reindex-operation": { - "dynamic": false, - "properties": { - "indexName": { - "type": "keyword" - }, - "status": { - "type": "integer" - } - } - }, - "upgrade-assistant-ml-upgrade-operation": { - "dynamic": false, - "properties": { - "snapshotId": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - }, - "monitoring-telemetry": { - "properties": { - "reportedClusterUuids": { - "type": "keyword" - } - } - }, - "apm-telemetry": { - "dynamic": false, - "properties": {} - }, - "apm-server-schema": { - "properties": { - "schemaJson": { - "type": "text", - "index": false - } - } - }, - "apm-service-group": { - "properties": { - "groupName": { - "type": "keyword" - }, - "kuery": { - "type": "text" - }, - "description": { - "type": "text" - }, - "color": { - "type": "text" - } - } - }, - "apm-custom-dashboards": { - "properties": { - "dashboardSavedObjectId": { - "type": "keyword" - }, - "kuery": { - "type": "text" - }, - "serviceEnvironmentFilterEnabled": { - "type": "boolean" - }, - "serviceNameFilterEnabled": { - "type": "boolean" - } - } - }, - "enterprise_search_telemetry": { - "dynamic": false, - "properties": {} - }, - "app_search_telemetry": { - "dynamic": false, - "properties": {} - }, - "workplace_search_telemetry": { - "dynamic": false, - "properties": {} } } From 7b2700841b8a9b0ef94b332f8515e35455e3f904 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Tue, 17 Oct 2023 14:47:16 +0300 Subject: [PATCH 19/40] Define connector params initial schema --- .../cases/server/connectors/cases/schema.ts | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/cases/server/connectors/cases/schema.ts b/x-pack/plugins/cases/server/connectors/cases/schema.ts index 4b6aedaee02d7..ecb7dd66571dc 100644 --- a/x-pack/plugins/cases/server/connectors/cases/schema.ts +++ b/x-pack/plugins/cases/server/connectors/cases/schema.ts @@ -7,13 +7,38 @@ import { schema } from '@kbn/config-schema'; +const AlertSchema = schema.recordOf(schema.string(), schema.string(), { + validate: (value) => { + if (!Object.hasOwn(value, 'id') || !Object.hasOwn(value, 'index')) { + return 'Alert ID and index must be defined'; + } + }, +}); + +/** + * At the moment only one field is supported for grouping + */ +const GroupingSchema = schema.arrayOf(schema.string(), { minSize: 0, maxSize: 1 }); + +const RuleSchema = schema.object({ + id: schema.string(), + name: schema.string(), + /** + * TODO: Verify limits + */ + tags: schema.arrayOf(schema.string({ minLength: 1, maxLength: 50 }), { minSize: 0, maxSize: 10 }), +}); + /** * The case connector does not have any configuration * or secrets. */ export const CasesConnectorConfigSchema = schema.object({}); export const CasesConnectorSecretsSchema = schema.object({}); -/** - * TODO: Add needed properties in the params schema. - */ -export const CasesConnectorParamsSchema = schema.object({}); + +export const CasesConnectorParamsSchema = schema.object({ + alerts: schema.arrayOf(AlertSchema), + groupingBy: GroupingSchema, + owner: schema.string(), + rule: RuleSchema, +}); From 13fd0133629b8eb2ef890c473abb792b8c57fc4b Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Tue, 17 Oct 2023 19:22:39 +0300 Subject: [PATCH 20/40] Bulk get records --- x-pack/plugins/cases/server/common/types.ts | 3 ++ .../connectors/cases/cases_oracle_service.ts | 35 ++++++++++++++++++- .../cases/server/connectors/cases/types.ts | 9 +++-- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/cases/server/common/types.ts b/x-pack/plugins/cases/server/common/types.ts index c79cb96d0d0b6..b44f4a996c8dc 100644 --- a/x-pack/plugins/cases/server/common/types.ts +++ b/x-pack/plugins/cases/server/common/types.ts @@ -52,3 +52,6 @@ export type FileAttachmentRequest = Omit< export type AttachmentSavedObject = SavedObject; export type SOWithErrors = Omit, 'attributes'> & { error: SavedObjectError }; +export interface SavedObjectsBulkResponseWithErrors { + saved_objects: Array | SOWithErrors>; +} diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts index 816df7719f036..5a69280690fc1 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts @@ -7,13 +7,24 @@ import type { Logger, SavedObject, SavedObjectsClientContract } from '@kbn/core/server'; import { CASE_ORACLE_SAVED_OBJECT } from '../../../common/constants'; +import type { SavedObjectsBulkResponseWithErrors } from '../../common/types'; +import { isSOError } from '../../common/utils'; import { CryptoService } from './crypto_service'; -import type { OracleKey, OracleRecord, OracleRecordCreateRequest } from './types'; +import type { + BulkGetRecordsResponse, + OracleKey, + OracleRecord, + OracleRecordCreateRequest, +} from './types'; type OracleRecordAttributes = Omit; export class CasesOracleService { private readonly log: Logger; + /** + * TODO: Think about permissions etc. + * Should we authorize based on the owner? + */ private readonly unsecuredSavedObjectsClient: SavedObjectsClientContract; private cryptoService: CryptoService; @@ -57,6 +68,16 @@ export class CasesOracleService { return this.getRecordResponse(oracleRecord); } + public async bulkGetRecord(ids: string[]): Promise { + this.log.debug(`Getting oracle records with ID: ${ids}`); + + const oracleRecords = (await this.unsecuredSavedObjectsClient.bulkGet( + ids.map((id) => ({ id, type: CASE_ORACLE_SAVED_OBJECT })) + )) as SavedObjectsBulkResponseWithErrors; + + return this.getBulkRecordsResponse(oracleRecords); + } + public async createRecord( recordId: string, payload: OracleRecordCreateRequest @@ -115,4 +136,16 @@ export class CasesOracleService { createdAt: oracleRecord.attributes.createdAt, updatedAt: oracleRecord.attributes.updatedAt, }); + + private getBulkRecordsResponse( + oracleRecords: SavedObjectsBulkResponseWithErrors + ): BulkGetRecordsResponse { + return oracleRecords.saved_objects.map((oracleRecord) => { + if (isSOError(oracleRecord)) { + return { ...oracleRecord.error }; + } + + return this.getRecordResponse(oracleRecord); + }); + } } diff --git a/x-pack/plugins/cases/server/connectors/cases/types.ts b/x-pack/plugins/cases/server/connectors/cases/types.ts index 373c24ed7e690..38436b2a86b9b 100644 --- a/x-pack/plugins/cases/server/connectors/cases/types.ts +++ b/x-pack/plugins/cases/server/connectors/cases/types.ts @@ -7,15 +7,16 @@ import type { ExclusiveUnion } from '@elastic/eui'; import type { TypeOf } from '@kbn/config-schema'; +import type { SavedObjectError } from '@kbn/core-saved-objects-common'; import type { CasesConnectorConfigSchema, CasesConnectorSecretsSchema, - CasesConnectorParamsSchema, + CasesConnectorRunParamsSchema, } from './schema'; export type CasesConnectorConfig = TypeOf; export type CasesConnectorSecrets = TypeOf; -export type CasesConnectorParams = TypeOf; +export type CasesConnectorRunParams = TypeOf; type Optional = Pick, K> & Omit; @@ -23,7 +24,7 @@ interface OracleKeyAllRequired { ruleId: string; spaceId: string; owner: string; - grouping: Record; + grouping: Record; } type OracleKeyWithOptionalKey = Optional; @@ -47,3 +48,5 @@ export interface OracleRecordCreateRequest { rules: Array<{ id: string }>; grouping: Record; } + +export type BulkGetRecordsResponse = Array; From 69a8778f81d4af894964a9c969055dfd8d45b047 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Tue, 17 Oct 2023 19:23:44 +0300 Subject: [PATCH 21/40] Group alerts and bulk get oracle records --- .../connectors/cases/cases_connector.test.ts | 46 ++++++++++ .../connectors/cases/cases_connector.ts | 89 +++++++++++++++++-- .../cases/server/connectors/cases/schema.ts | 4 +- 3 files changed, 132 insertions(+), 7 deletions(-) create mode 100644 x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts new file mode 100644 index 0000000000000..df7346891be5a --- /dev/null +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { actionsConfigMock } from '@kbn/actions-plugin/server/actions_config.mock'; +import { actionsMock } from '@kbn/actions-plugin/server/mocks'; +import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; +import { CasesConnector } from './cases_connector'; +import { CASES_CONNECTOR_ID } from './constants'; + +describe('CasesConnector', () => { + let connector: CasesConnector; + const alerts = [ + { 'host.name': 'A', 'dest.ip': '0.0.0.1', 'source.ip': '0.0.0.2' }, + { 'host.name': 'B', 'dest.ip': '0.0.0.1', 'file.hash': '12345' }, + { 'host.name': 'A', 'dest.ip': '0.0.0.1' }, + { 'host.name': 'B', 'dest.ip': '0.0.0.3' }, + { 'host.name': 'A', 'source.ip': '0.0.0.5' }, + ]; + + const groupingBy = ['host.name', 'dest.ip']; + const rule = { id: 'rule-test-id', name: 'Test rule', tags: ['rule', 'test'] }; + + beforeEach(() => { + jest.resetAllMocks(); + connector = new CasesConnector({ + configurationUtilities: actionsConfigMock.create(), + config: {}, + secrets: {}, + connector: { id: '1', type: CASES_CONNECTOR_ID }, + logger: loggingSystemMock.createLogger(), + services: actionsMock.createServices(), + }); + }); + + describe('run', () => { + describe('alerts grouping', () => { + it('groups the alerts correctly', async () => { + connector.run({ alerts, groupingBy, owner: 'cases', rule }); + }); + }); + }); +}); diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts index d7cee94276c51..62467ee5bb78d 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts @@ -5,19 +5,37 @@ * 2.0. */ +import stringify from 'json-stable-stringify'; import type { ServiceParams } from '@kbn/actions-plugin/server'; import { SubActionConnector } from '@kbn/actions-plugin/server'; +import { pick } from 'lodash'; import { CASES_CONNECTOR_SUB_ACTION } from './constants'; -import type { CasesConnectorConfig, CasesConnectorSecrets } from './types'; -import { CasesConnectorParamsSchema } from './schema'; +import type { CasesConnectorConfig, CasesConnectorRunParams, CasesConnectorSecrets } from './types'; +import { CasesConnectorRunParamsSchema } from './schema'; +import { CasesOracleService } from './cases_oracle_service'; + +interface GroupingMapValue { + alerts: CasesConnectorRunParams['alerts']; + grouping: Record; +} export class CasesConnector extends SubActionConnector< CasesConnectorConfig, CasesConnectorSecrets > { + private readonly casesOracleService; + constructor(params: ServiceParams) { super(params); - + this.casesOracleService = new CasesOracleService({ + log: this.logger, + /** + * TODO: Think about permissions etc. + * Should we use our own savedObjectsClient as we do + * in the cases client? + */ + unsecuredSavedObjectsClient: this.savedObjectsClient, + }); this.registerSubActions(); } @@ -25,7 +43,7 @@ export class CasesConnector extends SubActionConnector< this.registerSubAction({ name: CASES_CONNECTOR_SUB_ACTION.RUN, method: 'run', - schema: CasesConnectorParamsSchema, + schema: CasesConnectorRunParamsSchema, }); } @@ -38,5 +56,66 @@ export class CasesConnector extends SubActionConnector< throw new Error('Method not implemented.'); } - public async run() {} + private groupAlerts({ + alerts, + groupingBy, + }: Pick): Map { + const uniqueGroupingByFields = Array.from(new Set(groupingBy)); + const groupingMap = new Map(); + + const filteredAlerts = alerts.filter((alert) => + uniqueGroupingByFields.every((groupingByField) => Object.hasOwn(alert, groupingByField)) + ); + + for (const alert of filteredAlerts) { + const alertWithOnlyTheGroupingFields = pick(alert, uniqueGroupingByFields); + const groupingKey = stringify(alertWithOnlyTheGroupingFields); + + if (groupingMap.has(groupingKey)) { + groupingMap.get(groupingKey)?.alerts.push(alert); + } else { + groupingMap.set(groupingKey, { alerts: [alert], grouping: alertWithOnlyTheGroupingFields }); + } + } + + return groupingMap; + } + + private generateOracleKeys( + params: CasesConnectorRunParams, + groupingMap: Map + ): Map { + const { rule, owner } = params; + /** + * TODO: Take spaceId from the actions framework + */ + const spaceId = 'default'; + + const oracleMap = new Map(); + + for (const { grouping, alerts } of groupingMap.values()) { + const oracleKey = this.casesOracleService.getRecordId({ + ruleId: rule.id, + grouping, + owner, + spaceId, + }); + + oracleMap.set(oracleKey, { grouping, alerts }); + } + + return oracleMap; + } + + private getOracleRecord(groupingMap: Map): Promise {} + + public async run(params: CasesConnectorRunParams) { + const { alerts, groupingBy } = params; + const groupingMap = this.groupAlerts({ alerts, groupingBy }); + const oracleMap = this.generateOracleKeys(params, groupingMap); + + const oracleKeys = oracleMap.keys(); + + const oracleBulkGetRes = this.casesOracleService.bulkGetRecord(Array.from(oracleKeys)); + } } diff --git a/x-pack/plugins/cases/server/connectors/cases/schema.ts b/x-pack/plugins/cases/server/connectors/cases/schema.ts index ecb7dd66571dc..3d38ed15ccb35 100644 --- a/x-pack/plugins/cases/server/connectors/cases/schema.ts +++ b/x-pack/plugins/cases/server/connectors/cases/schema.ts @@ -7,7 +7,7 @@ import { schema } from '@kbn/config-schema'; -const AlertSchema = schema.recordOf(schema.string(), schema.string(), { +const AlertSchema = schema.recordOf(schema.string(), schema.any(), { validate: (value) => { if (!Object.hasOwn(value, 'id') || !Object.hasOwn(value, 'index')) { return 'Alert ID and index must be defined'; @@ -36,7 +36,7 @@ const RuleSchema = schema.object({ export const CasesConnectorConfigSchema = schema.object({}); export const CasesConnectorSecretsSchema = schema.object({}); -export const CasesConnectorParamsSchema = schema.object({ +export const CasesConnectorRunParamsSchema = schema.object({ alerts: schema.arrayOf(AlertSchema), groupingBy: GroupingSchema, owner: schema.string(), From e5c73a39c8d458d3d4381a3791eea540dea5f4e6 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 18 Oct 2023 12:25:24 +0000 Subject: [PATCH 22/40] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/plugins/cases/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/cases/tsconfig.json b/x-pack/plugins/cases/tsconfig.json index ccd6e228d6aff..1f7d76113b0ec 100644 --- a/x-pack/plugins/cases/tsconfig.json +++ b/x-pack/plugins/cases/tsconfig.json @@ -70,6 +70,7 @@ "@kbn/core-http-server", "@kbn/alerting-plugin", "@kbn/content-management-plugin", + "@kbn/core-logging-server-mocks", ], "exclude": [ "target/**/*", From 1ba61173cd810181a95f77db0d1cbb57b569579c Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Wed, 18 Oct 2023 19:16:01 +0300 Subject: [PATCH 23/40] Bulk create records --- .../connectors/cases/cases_connector.ts | 4 +- .../cases/cases_oracle_service.test.ts | 255 ++++++++++++++++++ .../connectors/cases/cases_oracle_service.ts | 67 ++++- .../server/connectors/cases/index.mock.ts | 26 ++ .../cases/server/connectors/cases/types.ts | 2 + .../server/connectors/cases/utils.test.ts | 37 +++ .../cases/server/connectors/cases/utils.ts | 23 ++ 7 files changed, 409 insertions(+), 5 deletions(-) create mode 100644 x-pack/plugins/cases/server/connectors/cases/index.mock.ts create mode 100644 x-pack/plugins/cases/server/connectors/cases/utils.test.ts create mode 100644 x-pack/plugins/cases/server/connectors/cases/utils.ts diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts index 62467ee5bb78d..aba9ad9d2560b 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts @@ -32,7 +32,7 @@ export class CasesConnector extends SubActionConnector< /** * TODO: Think about permissions etc. * Should we use our own savedObjectsClient as we do - * in the cases client? + * in the cases client? Should we so the createInternalRepository? */ unsecuredSavedObjectsClient: this.savedObjectsClient, }); @@ -116,6 +116,6 @@ export class CasesConnector extends SubActionConnector< const oracleKeys = oracleMap.keys(); - const oracleBulkGetRes = this.casesOracleService.bulkGetRecord(Array.from(oracleKeys)); + // const oracleBulkGetRes = this.casesOracleService.bulkGetRecords(Array.from(oracleKeys)); } } diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts index 5d1261f9d9539..3b555590a34b5 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts @@ -174,6 +174,67 @@ describe('CasesOracleService', () => { expect(record).toEqual({ ...oracleSO.attributes, id: 'so-id', version: 'so-version' }); }); + + it('calls the unsecuredSavedObjectsClient.get method correctly', async () => { + await service.getRecord('so-id'); + + expect(unsecuredSavedObjectsClient.get).toHaveBeenCalledWith('cases-oracle', 'so-id'); + }); + }); + + describe('bulkGetRecord', () => { + const cases = [{ id: 'test-case-id' }]; + const rules = [{ id: 'test-rule-id' }]; + const grouping = { 'host.ip': '0.0.0.1', 'agent.id': '8a4f500d' }; + + const bulkGetSOs = [ + { + id: 'so-id', + version: 'so-version', + attributes: { + counter: 1, + cases, + rules, + grouping, + createdAt: '2023-10-10T10:23:42.769Z', + updatedAt: '2023-10-10T10:23:42.769Z', + }, + type: CASE_ORACLE_SAVED_OBJECT, + references: [], + }, + { + id: 'so-id-2', + type: CASE_ORACLE_SAVED_OBJECT, + error: { + message: 'Not found', + statusCode: 404, + error: 'Not found', + }, + }, + ]; + + beforeEach(() => { + // @ts-expect-error: types of the SO client are wrong and they do not accept errors + unsecuredSavedObjectsClient.bulkGet.mockResolvedValue({ saved_objects: bulkGetSOs }); + }); + + it('formats the response correctly', async () => { + const res = await service.bulkGetRecords(['so-id', 'so-id-2']); + + expect(res).toEqual([ + { ...bulkGetSOs[0].attributes, id: 'so-id', version: 'so-version' }, + { ...bulkGetSOs[1].error, id: 'so-id-2' }, + ]); + }); + + it('calls the unsecuredSavedObjectsClient.bulkGet method correctly', async () => { + await service.bulkGetRecords(['so-id', 'so-id-2']); + + expect(unsecuredSavedObjectsClient.bulkGet).toHaveBeenCalledWith([ + { id: 'so-id', type: 'cases-oracle' }, + { id: 'so-id-2', type: 'cases-oracle' }, + ]); + }); }); describe('createRecord', () => { @@ -226,6 +287,89 @@ describe('CasesOracleService', () => { }); }); + describe('bulkCreateRecord', () => { + const cases = [{ id: 'test-case-id' }]; + const rules = [{ id: 'test-rule-id' }]; + const grouping = { 'host.ip': '0.0.0.1', 'agent.id': '8a4f500d' }; + + const bulkCreateSOs = [ + { + id: 'so-id', + version: 'so-version', + attributes: { + counter: 1, + cases, + rules, + grouping, + createdAt: '2023-10-10T10:23:42.769Z', + updatedAt: '2023-10-10T10:23:42.769Z', + }, + type: CASE_ORACLE_SAVED_OBJECT, + references: [], + }, + { + id: 'so-id-2', + type: CASE_ORACLE_SAVED_OBJECT, + error: { + message: 'Not found', + statusCode: 404, + error: 'Not found', + }, + }, + ]; + + beforeEach(() => { + // @ts-expect-error: types of the SO client are wrong and they do not accept errors + unsecuredSavedObjectsClient.bulkCreate.mockResolvedValue({ saved_objects: bulkCreateSOs }); + }); + + it('formats the response correctly', async () => { + const res = await service.bulkCreateRecord([ + { recordId: 'so-id', payload: { cases, rules, grouping } }, + { recordId: 'so-id-2', payload: { cases, rules, grouping } }, + ]); + + expect(res).toEqual([ + { ...bulkCreateSOs[0].attributes, id: 'so-id', version: 'so-version' }, + { ...bulkCreateSOs[1].error, id: 'so-id-2' }, + ]); + }); + + it('calls the bulkCreate correctly', async () => { + await service.bulkCreateRecord([ + { recordId: 'so-id', payload: { cases, rules, grouping } }, + { recordId: 'so-id-2', payload: { cases, rules, grouping } }, + ]); + + expect(unsecuredSavedObjectsClient.bulkCreate).toHaveBeenCalledWith([ + { + attributes: { + cases, + rules, + grouping, + counter: 1, + createdAt: expect.anything(), + updatedAt: null, + }, + id: 'so-id', + type: 'cases-oracle', + }, + { + attributes: { + cases, + rules, + grouping, + counter: 1, + createdAt: expect.anything(), + updatedAt: null, + }, + id: 'so-id-2', + type: 'cases-oracle', + }, + ]); + }); + }); + describe('increaseCounter', () => { const cases = [{ id: 'test-case-id' }]; const rules = [{ id: 'test-rule-id' }]; @@ -280,4 +424,115 @@ describe('CasesOracleService', () => { ); }); }); + + describe('bulkGetOrCreateRecords', () => { + const cases = [{ id: 'test-case-id' }]; + const rules = [{ id: 'test-rule-id' }]; + const grouping = { 'host.ip': '0.0.0.1', 'agent.id': '8a4f500d' }; + + const bulkSOs = [ + { + id: 'so-id', + version: 'so-version', + attributes: { + counter: 1, + cases, + rules, + grouping, + createdAt: '2023-10-10T10:23:42.769Z', + updatedAt: '2023-10-10T10:23:42.769Z', + }, + type: CASE_ORACLE_SAVED_OBJECT, + references: [], + }, + { + id: 'so-id-2', + type: CASE_ORACLE_SAVED_OBJECT, + error: { + message: 'Not found', + statusCode: 404, + error: 'Not found', + }, + }, + ]; + + beforeEach(() => { + // @ts-expect-error: types of the SO client are wrong and they do not accept errors + unsecuredSavedObjectsClient.bulkGet.mockResolvedValue({ saved_objects: bulkSOs }); + unsecuredSavedObjectsClient.bulkCreate.mockResolvedValue({ + saved_objects: [ + // @ts-expect-error: types of the SO client are wrong and they do not accept errors + { + ...bulkSOs[0], + id: 'so-id-2', + attributes: { ...bulkSOs[0].attributes, cases: [{ id: 'test-case-id-2' }] }, + }, + ], + }); + }); + + it('creates the new records if they do not exist', async () => { + const res = await service.bulkGetOrCreateRecords([ + { recordId: 'so-id', payload: { cases, rules, grouping } }, + { recordId: 'so-id-2', payload: { cases: [{ id: 'test-case-id-2' }], rules, grouping } }, + ]); + + expect(res).toEqual([ + { + id: 'so-id', + cases: [{ id: 'test-case-id' }], + counter: 1, + createdAt: '2023-10-10T10:23:42.769Z', + grouping: { 'agent.id': '8a4f500d', 'host.ip': '0.0.0.1' }, + rules: [{ id: 'test-rule-id' }], + updatedAt: '2023-10-10T10:23:42.769Z', + version: 'so-version', + }, + { + grouping: { 'agent.id': '8a4f500d', 'host.ip': '0.0.0.1' }, + id: 'so-id-2', + cases: [{ id: 'test-case-id-2' }], + counter: 1, + createdAt: '2023-10-10T10:23:42.769Z', + rules: [{ id: 'test-rule-id' }], + updatedAt: '2023-10-10T10:23:42.769Z', + version: 'so-version', + }, + ]); + }); + + it('calls the unsecuredSavedObjectsClient.bulkGet correctly', async () => { + await service.bulkGetOrCreateRecords([ + { recordId: 'so-id', payload: { cases, rules, grouping } }, + { recordId: 'so-id-2', payload: { cases: [{ id: 'test-case-id-2' }], rules, grouping } }, + ]); + + expect(unsecuredSavedObjectsClient.bulkGet).toHaveBeenCalledWith([ + { id: 'so-id', type: 'cases-oracle' }, + { id: 'so-id-2', type: 'cases-oracle' }, + ]); + }); + + it('calls the unsecuredSavedObjectsClient.bulkCreate correctly', async () => { + await service.bulkGetOrCreateRecords([ + { recordId: 'so-id', payload: { cases, rules, grouping } }, + { recordId: 'so-id-2', payload: { cases: [{ id: 'test-case-id-2' }], rules, grouping } }, + ]); + + expect(unsecuredSavedObjectsClient.bulkCreate).toHaveBeenCalledWith([ + { + attributes: { + cases: [{ id: 'test-case-id-2' }], + counter: 1, + createdAt: expect.anything(), + grouping: { 'agent.id': '8a4f500d', 'host.ip': '0.0.0.1' }, + rules: [{ id: 'test-rule-id' }], + updatedAt: null, + }, + id: 'so-id-2', + type: 'cases-oracle', + }, + ]); + }); + }); }); diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts index 5a69280690fc1..e5a613426bc2f 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts @@ -16,8 +16,13 @@ import type { OracleRecord, OracleRecordCreateRequest, } from './types'; +import { partitionRecords } from './utils'; type OracleRecordAttributes = Omit; +type BulkCreateRequest = Array<{ + recordId: string; + payload: OracleRecordCreateRequest; +}>; export class CasesOracleService { private readonly log: Logger; @@ -68,8 +73,8 @@ export class CasesOracleService { return this.getRecordResponse(oracleRecord); } - public async bulkGetRecord(ids: string[]): Promise { - this.log.debug(`Getting oracle records with ID: ${ids}`); + public async bulkGetRecords(ids: string[]): Promise { + this.log.debug(`Getting oracle records with IDs: ${ids}`); const oracleRecords = (await this.unsecuredSavedObjectsClient.bulkGet( ids.map((id) => ({ id, type: CASE_ORACLE_SAVED_OBJECT })) @@ -102,6 +107,62 @@ export class CasesOracleService { return this.getRecordResponse(oracleRecord); } + public async bulkCreateRecord(records: BulkCreateRequest): Promise { + const recordIds = records.map((record) => record.recordId); + + this.log.debug(`Creating oracle record with ID: ${recordIds}`); + + const req = records.map((record) => ({ + id: record.recordId, + type: CASE_ORACLE_SAVED_OBJECT, + attributes: { + counter: 1, + cases: record.payload.cases, + rules: record.payload.rules, + grouping: record.payload.grouping, + createdAt: new Date().toISOString(), + updatedAt: null, + }, + })); + + const oracleRecords = + (await this.unsecuredSavedObjectsClient.bulkCreate( + req + )) as SavedObjectsBulkResponseWithErrors; + + return this.getBulkRecordsResponse(oracleRecords); + } + + public async bulkGetOrCreateRecords(records: BulkCreateRequest): Promise { + const recordsMap = new Map( + records.map(({ recordId, payload }) => [recordId, payload]) + ); + const bulkCreateReq: BulkCreateRequest = []; + + const ids = records.map(({ recordId }) => recordId); + + const bulkGetRes = await this.bulkGetRecords(ids); + const [bulkGetValidRecords, bulkGetRecordsErrors] = partitionRecords(bulkGetRes); + + for (const error of bulkGetRecordsErrors) { + if (error.id && recordsMap.has(error.id)) { + bulkCreateReq.push({ + recordId: error.id, + payload: recordsMap.get(error.id) ?? { cases: [], rules: [], grouping: {} }, + }); + } + } + + const bulkCreateRes = await this.bulkCreateRecord(bulkCreateReq); + + /** + * TODO: Retry on errors + */ + const [bulkCreateValidRecords, _] = partitionRecords(bulkCreateRes); + + return [...bulkGetValidRecords, ...bulkCreateValidRecords]; + } + public async increaseCounter(recordId: string): Promise { const { id: _, version, ...record } = await this.getRecord(recordId); const newCounter = record.counter + 1; @@ -142,7 +203,7 @@ export class CasesOracleService { ): BulkGetRecordsResponse { return oracleRecords.saved_objects.map((oracleRecord) => { if (isSOError(oracleRecord)) { - return { ...oracleRecord.error }; + return { ...oracleRecord.error, id: oracleRecord.id }; } return this.getRecordResponse(oracleRecord); diff --git a/x-pack/plugins/cases/server/connectors/cases/index.mock.ts b/x-pack/plugins/cases/server/connectors/cases/index.mock.ts new file mode 100644 index 0000000000000..de9d3f5f045ad --- /dev/null +++ b/x-pack/plugins/cases/server/connectors/cases/index.mock.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { OracleRecord, OracleRecordError } from './types'; + +export const oracleRecord: OracleRecord = { + id: 'so-id', + version: 'so-version', + cases: [{ id: 'test-case-id' }], + rules: [{ id: 'test-rule-id' }], + grouping: { 'host.ip': '0.0.0.1', 'agent.id': '8a4f500d' }, + counter: 1, + createdAt: '2023-10-10T10:23:42.769Z', + updatedAt: '2023-10-10T10:23:42.769Z', +}; + +export const oracleRecordError: OracleRecordError = { + id: 'so-id', + error: 'An error', + statusCode: 404, + message: 'An error', +}; diff --git a/x-pack/plugins/cases/server/connectors/cases/types.ts b/x-pack/plugins/cases/server/connectors/cases/types.ts index 38436b2a86b9b..c208346b46cca 100644 --- a/x-pack/plugins/cases/server/connectors/cases/types.ts +++ b/x-pack/plugins/cases/server/connectors/cases/types.ts @@ -43,6 +43,8 @@ export interface OracleRecord { version: string; } +export type OracleRecordError = { id?: string } & SavedObjectError; + export interface OracleRecordCreateRequest { cases: Array<{ id: string }>; rules: Array<{ id: string }>; diff --git a/x-pack/plugins/cases/server/connectors/cases/utils.test.ts b/x-pack/plugins/cases/server/connectors/cases/utils.test.ts new file mode 100644 index 0000000000000..444dd8da6b27e --- /dev/null +++ b/x-pack/plugins/cases/server/connectors/cases/utils.test.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { oracleRecordError, oracleRecord } from './index.mock'; +import { isRecordError, partitionRecords } from './utils'; + +describe('utils', () => { + describe('isRecordError', () => { + it('returns true if the record contains an error', () => { + expect(isRecordError(oracleRecordError)).toBe(true); + }); + + it('returns false if the record is an oracle record', () => { + expect(isRecordError(oracleRecord)).toBe(false); + }); + + it('returns false if the record is an empty object', () => { + // @ts-expect-error: need to test for empty objects + expect(isRecordError({})).toBe(false); + }); + }); + + describe('partitionRecords', () => { + it('partition records correctly', () => { + expect( + partitionRecords([oracleRecordError, oracleRecord, oracleRecordError, oracleRecord]) + ).toEqual([ + [oracleRecord, oracleRecord], + [oracleRecordError, oracleRecordError], + ]); + }); + }); +}); diff --git a/x-pack/plugins/cases/server/connectors/cases/utils.ts b/x-pack/plugins/cases/server/connectors/cases/utils.ts new file mode 100644 index 0000000000000..24064efab1439 --- /dev/null +++ b/x-pack/plugins/cases/server/connectors/cases/utils.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { partition } from 'lodash'; +import type { BulkGetRecordsResponse, OracleRecord, OracleRecordError } from './types'; + +export const isRecordError = (so: OracleRecord | OracleRecordError): so is OracleRecordError => + (so as OracleRecordError).error != null; + +export const partitionRecords = ( + res: BulkGetRecordsResponse +): [OracleRecord[], OracleRecordError[]] => { + const [errors, validRecords] = partition(res, isRecordError) as [ + OracleRecordError[], + OracleRecord[] + ]; + + return [validRecords, errors]; +}; From c6982a5f18d080ef54311edf198d894eef037d50 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Thu, 19 Oct 2023 11:11:53 +0300 Subject: [PATCH 24/40] Add TODOs --- .../cases/server/connectors/cases/cases_connector.ts | 6 +++++- .../cases/server/connectors/cases/cases_oracle_service.ts | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts index aba9ad9d2560b..6b25aa7703d65 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts @@ -111,11 +111,15 @@ export class CasesConnector extends SubActionConnector< public async run(params: CasesConnectorRunParams) { const { alerts, groupingBy } = params; + /** + * TODO: Handle when grouping is not defined + * One case should be created per rule + */ const groupingMap = this.groupAlerts({ alerts, groupingBy }); const oracleMap = this.generateOracleKeys(params, groupingMap); const oracleKeys = oracleMap.keys(); - // const oracleBulkGetRes = this.casesOracleService.bulkGetRecords(Array.from(oracleKeys)); + const oracleBulkGetRes = this.casesOracleService.bulkGetOrCreateRecords(Array.from(oracleKeys)); } } diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts index e5a613426bc2f..8de7541f91552 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts @@ -153,6 +153,10 @@ export class CasesOracleService { } } + /** + * TODO: Create records with only 404 errors + * All others should throw an error and retry again + */ const bulkCreateRes = await this.bulkCreateRecord(bulkCreateReq); /** From f184a08c67ddbdfeac26089aef6c0feea1233cd2 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Fri, 20 Oct 2023 14:24:39 +0300 Subject: [PATCH 25/40] Move bulkGetOrCreateOracleRecords logic to the connector --- .../connectors/cases/cases_connector.test.ts | 132 +++++++++++++++++- .../connectors/cases/cases_connector.ts | 94 ++++++++++--- .../cases/cases_oracle_service.test.ts | 111 --------------- .../connectors/cases/cases_oracle_service.ts | 50 +------ .../cases/server/connectors/cases/types.ts | 7 +- .../cases/server/connectors/cases/utils.ts | 4 +- 6 files changed, 217 insertions(+), 181 deletions(-) diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts index df7346891be5a..1b064934d101b 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts @@ -10,9 +10,16 @@ import { actionsMock } from '@kbn/actions-plugin/server/mocks'; import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; import { CasesConnector } from './cases_connector'; import { CASES_CONNECTOR_ID } from './constants'; +import { CASE_ORACLE_SAVED_OBJECT } from '../../../common/constants'; +import { CasesOracleService } from './cases_oracle_service'; + +jest.mock('./cases_oracle_service'); + +const CasesOracleServiceMock = CasesOracleService as jest.Mock; describe('CasesConnector', () => { - let connector: CasesConnector; + const services = actionsMock.createServices(); + const alerts = [ { 'host.name': 'A', 'dest.ip': '0.0.0.1', 'source.ip': '0.0.0.2' }, { 'host.name': 'B', 'dest.ip': '0.0.0.1', 'file.hash': '12345' }, @@ -23,23 +30,136 @@ describe('CasesConnector', () => { const groupingBy = ['host.name', 'dest.ip']; const rule = { id: 'rule-test-id', name: 'Test rule', tags: ['rule', 'test'] }; + const owner = 'cases'; + + const groupedAlertsWithOracleKey = [ + { + alerts: [alerts[0], alerts[2]], + grouping: { 'host.name': 'A', 'dest.ip': '0.0.0.1' }, + oracleKey: 'so-oracle-record-0', + }, + { + alerts: [alerts[1]], + grouping: { 'host.name': 'B', 'dest.ip': '0.0.0.1' }, + oracleKey: 'so-oracle-record-1', + }, + { + alerts: [alerts[3]], + grouping: { 'host.name': 'B', 'dest.ip': '0.0.0.3' }, + oracleKey: 'so-oracle-record-2', + }, + ]; + + const oracleRecords = [ + { + id: groupedAlertsWithOracleKey[0].oracleKey, + version: 'so-version-0', + counter: 1, + cases: [], + rules: [], + grouping: groupedAlertsWithOracleKey[0].grouping, + createdAt: '2023-10-10T10:23:42.769Z', + updatedAt: '2023-10-10T10:23:42.769Z', + }, + { + id: groupedAlertsWithOracleKey[1].oracleKey, + version: 'so-version-1', + counter: 1, + cases: [], + rules: [], + grouping: groupedAlertsWithOracleKey[1].grouping, + createdAt: '2023-10-10T10:23:42.769Z', + updatedAt: '2023-10-10T10:23:42.769Z', + }, + { + id: groupedAlertsWithOracleKey[2].oracleKey, + type: CASE_ORACLE_SAVED_OBJECT, + message: 'Not found', + statusCode: 404, + error: 'Not found', + }, + ]; + + const mockGetRecordId = jest.fn(); + const bulkGetRecords = jest.fn(); + const bulkCreateRecord = jest.fn(); + + let connector: CasesConnector; beforeEach(() => { - jest.resetAllMocks(); + jest.clearAllMocks(); + + CasesOracleServiceMock.mockImplementation(() => { + let idCounter = 0; + + return { + getRecordId: mockGetRecordId.mockImplementation(() => `so-oracle-record-${idCounter++}`), + bulkGetRecords: bulkGetRecords.mockResolvedValue(oracleRecords), + bulkCreateRecord: bulkCreateRecord.mockResolvedValue({ + ...oracleRecords[0], + id: groupedAlertsWithOracleKey[2].oracleKey, + grouping: groupedAlertsWithOracleKey[2].grouping, + version: 'so-version-2', + }), + }; + }); + connector = new CasesConnector({ configurationUtilities: actionsConfigMock.create(), config: {}, secrets: {}, connector: { id: '1', type: CASES_CONNECTOR_ID }, logger: loggingSystemMock.createLogger(), - services: actionsMock.createServices(), + services, }); }); describe('run', () => { - describe('alerts grouping', () => { - it('groups the alerts correctly', async () => { - connector.run({ alerts, groupingBy, owner: 'cases', rule }); + describe('Oracle records', () => { + it('generates the oracle keys correctly', async () => { + await connector.run({ alerts, groupingBy, owner, rule }); + + for (const [index, { grouping }] of groupedAlertsWithOracleKey.entries()) { + expect(mockGetRecordId).nthCalledWith(index + 1, { + ruleId: rule.id, + grouping, + owner, + spaceId: 'default', + }); + } + }); + + it('gets the oracle records correctly', async () => { + await connector.run({ alerts, groupingBy, owner, rule }); + + expect(bulkGetRecords).toHaveBeenCalledWith([ + groupedAlertsWithOracleKey[0].oracleKey, + groupedAlertsWithOracleKey[1].oracleKey, + groupedAlertsWithOracleKey[2].oracleKey, + ]); + }); + + it('created the no found oracle records correctly', async () => { + await connector.run({ alerts, groupingBy, owner, rule }); + + expect(bulkCreateRecord).toHaveBeenCalledWith([ + { + recordId: groupedAlertsWithOracleKey[2].oracleKey, + payload: { + cases: [], + grouping: groupedAlertsWithOracleKey[2].grouping, + rules: [], + }, + }, + ]); + }); + + it('does not create oracle records if there are no 404 errors', async () => { + bulkGetRecords.mockResolvedValue([oracleRecords[0]]); + + await connector.run({ alerts, groupingBy, owner, rule }); + + expect(bulkCreateRecord).not.toHaveBeenCalled(); }); }); }); diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts index 6b25aa7703d65..0ed9ba47e0546 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts @@ -10,15 +10,25 @@ import type { ServiceParams } from '@kbn/actions-plugin/server'; import { SubActionConnector } from '@kbn/actions-plugin/server'; import { pick } from 'lodash'; import { CASES_CONNECTOR_SUB_ACTION } from './constants'; -import type { CasesConnectorConfig, CasesConnectorRunParams, CasesConnectorSecrets } from './types'; +import type { + BulkCreateOracleRecordRequest, + CasesConnectorConfig, + CasesConnectorRunParams, + CasesConnectorSecrets, + OracleRecord, + OracleRecordCreateRequest, +} from './types'; import { CasesConnectorRunParamsSchema } from './schema'; import { CasesOracleService } from './cases_oracle_service'; +import { partitionRecords } from './utils'; -interface GroupingMapValue { +interface GroupedAlerts { alerts: CasesConnectorRunParams['alerts']; grouping: Record; } +type GroupedAlertsWithOracleKey = GroupedAlerts & { oracleKey: string }; + export class CasesConnector extends SubActionConnector< CasesConnectorConfig, CasesConnectorSecrets @@ -59,9 +69,9 @@ export class CasesConnector extends SubActionConnector< private groupAlerts({ alerts, groupingBy, - }: Pick): Map { + }: Pick): GroupedAlerts[] { const uniqueGroupingByFields = Array.from(new Set(groupingBy)); - const groupingMap = new Map(); + const groupingMap = new Map(); const filteredAlerts = alerts.filter((alert) => uniqueGroupingByFields.every((groupingByField) => Object.hasOwn(alert, groupingByField)) @@ -78,22 +88,22 @@ export class CasesConnector extends SubActionConnector< } } - return groupingMap; + return Array.from(groupingMap.values()); } private generateOracleKeys( params: CasesConnectorRunParams, - groupingMap: Map - ): Map { + groupedAlerts: GroupedAlerts[] + ): GroupedAlertsWithOracleKey[] { const { rule, owner } = params; /** * TODO: Take spaceId from the actions framework */ const spaceId = 'default'; - const oracleMap = new Map(); + const oracleMap = new Map(); - for (const { grouping, alerts } of groupingMap.values()) { + for (const { grouping, alerts } of groupedAlerts) { const oracleKey = this.casesOracleService.getRecordId({ ruleId: rule.id, grouping, @@ -101,25 +111,73 @@ export class CasesConnector extends SubActionConnector< spaceId, }); - oracleMap.set(oracleKey, { grouping, alerts }); + oracleMap.set(oracleKey, { oracleKey, grouping, alerts }); } - return oracleMap; + return Array.from(oracleMap.values()); } - private getOracleRecord(groupingMap: Map): Promise {} + private async bulkGetOrCreateOracleRecords( + groupedAlertsWithOracleKey: GroupedAlertsWithOracleKey[] + ): Promise { + const bulkCreateReq: BulkCreateOracleRecordRequest = []; + + const ids = groupedAlertsWithOracleKey.map(({ oracleKey }) => oracleKey); + + const bulkGetRes = await this.casesOracleService.bulkGetRecords(ids); + const [bulkGetValidRecords, bulkGetRecordsErrors] = partitionRecords(bulkGetRes); + + if (bulkGetRecordsErrors.length === 0) { + return bulkGetValidRecords; + } + + const recordsMap = new Map( + groupedAlertsWithOracleKey.map(({ oracleKey, grouping }) => [ + oracleKey, + // TODO: Add the rule info + { cases: [], rules: [], grouping }, + ]) + ); + + for (const error of bulkGetRecordsErrors) { + if (error.id && recordsMap.has(error.id)) { + bulkCreateReq.push({ + recordId: error.id, + payload: recordsMap.get(error.id) ?? { cases: [], rules: [], grouping: {} }, + }); + } + } + + /** + * TODO: Create records with only 404 errors + * All others should throw an error and retry again + */ + const bulkCreateRes = await this.casesOracleService.bulkCreateRecord(bulkCreateReq); + + /** + * TODO: Retry on errors + */ + const [bulkCreateValidRecords, _] = partitionRecords(bulkCreateRes); + + return [...bulkGetValidRecords, ...bulkCreateValidRecords]; + } public async run(params: CasesConnectorRunParams) { const { alerts, groupingBy } = params; + /** * TODO: Handle when grouping is not defined * One case should be created per rule */ - const groupingMap = this.groupAlerts({ alerts, groupingBy }); - const oracleMap = this.generateOracleKeys(params, groupingMap); - - const oracleKeys = oracleMap.keys(); - - const oracleBulkGetRes = this.casesOracleService.bulkGetOrCreateRecords(Array.from(oracleKeys)); + const groupedAlerts = this.groupAlerts({ alerts, groupingBy }); + const groupedAlertsWithOracleKey = this.generateOracleKeys(params, groupedAlerts); + console.log( + '🚀 ~ file: cases_connector.ts:175 ~ run ~ groupedAlertsWithOracleKey:', + groupedAlertsWithOracleKey + ); + /** + * Add circuit breakers to the number of oracles they can be created or retrieved + */ + const oracleRecords = this.bulkGetOrCreateOracleRecords(groupedAlertsWithOracleKey); } } diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts index 3b555590a34b5..84ef73c9c1ea8 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts @@ -424,115 +424,4 @@ describe('CasesOracleService', () => { ); }); }); - - describe('bulkGetOrCreateRecords', () => { - const cases = [{ id: 'test-case-id' }]; - const rules = [{ id: 'test-rule-id' }]; - const grouping = { 'host.ip': '0.0.0.1', 'agent.id': '8a4f500d' }; - - const bulkSOs = [ - { - id: 'so-id', - version: 'so-version', - attributes: { - counter: 1, - cases, - rules, - grouping, - createdAt: '2023-10-10T10:23:42.769Z', - updatedAt: '2023-10-10T10:23:42.769Z', - }, - type: CASE_ORACLE_SAVED_OBJECT, - references: [], - }, - { - id: 'so-id-2', - type: CASE_ORACLE_SAVED_OBJECT, - error: { - message: 'Not found', - statusCode: 404, - error: 'Not found', - }, - }, - ]; - - beforeEach(() => { - // @ts-expect-error: types of the SO client are wrong and they do not accept errors - unsecuredSavedObjectsClient.bulkGet.mockResolvedValue({ saved_objects: bulkSOs }); - unsecuredSavedObjectsClient.bulkCreate.mockResolvedValue({ - saved_objects: [ - // @ts-expect-error: types of the SO client are wrong and they do not accept errors - { - ...bulkSOs[0], - id: 'so-id-2', - attributes: { ...bulkSOs[0].attributes, cases: [{ id: 'test-case-id-2' }] }, - }, - ], - }); - }); - - it('creates the new records if they do not exist', async () => { - const res = await service.bulkGetOrCreateRecords([ - { recordId: 'so-id', payload: { cases, rules, grouping } }, - { recordId: 'so-id-2', payload: { cases: [{ id: 'test-case-id-2' }], rules, grouping } }, - ]); - - expect(res).toEqual([ - { - id: 'so-id', - cases: [{ id: 'test-case-id' }], - counter: 1, - createdAt: '2023-10-10T10:23:42.769Z', - grouping: { 'agent.id': '8a4f500d', 'host.ip': '0.0.0.1' }, - rules: [{ id: 'test-rule-id' }], - updatedAt: '2023-10-10T10:23:42.769Z', - version: 'so-version', - }, - { - grouping: { 'agent.id': '8a4f500d', 'host.ip': '0.0.0.1' }, - id: 'so-id-2', - cases: [{ id: 'test-case-id-2' }], - counter: 1, - createdAt: '2023-10-10T10:23:42.769Z', - rules: [{ id: 'test-rule-id' }], - updatedAt: '2023-10-10T10:23:42.769Z', - version: 'so-version', - }, - ]); - }); - - it('calls the unsecuredSavedObjectsClient.bulkGet correctly', async () => { - await service.bulkGetOrCreateRecords([ - { recordId: 'so-id', payload: { cases, rules, grouping } }, - { recordId: 'so-id-2', payload: { cases: [{ id: 'test-case-id-2' }], rules, grouping } }, - ]); - - expect(unsecuredSavedObjectsClient.bulkGet).toHaveBeenCalledWith([ - { id: 'so-id', type: 'cases-oracle' }, - { id: 'so-id-2', type: 'cases-oracle' }, - ]); - }); - - it('calls the unsecuredSavedObjectsClient.bulkCreate correctly', async () => { - await service.bulkGetOrCreateRecords([ - { recordId: 'so-id', payload: { cases, rules, grouping } }, - { recordId: 'so-id-2', payload: { cases: [{ id: 'test-case-id-2' }], rules, grouping } }, - ]); - - expect(unsecuredSavedObjectsClient.bulkCreate).toHaveBeenCalledWith([ - { - attributes: { - cases: [{ id: 'test-case-id-2' }], - counter: 1, - createdAt: expect.anything(), - grouping: { 'agent.id': '8a4f500d', 'host.ip': '0.0.0.1' }, - rules: [{ id: 'test-rule-id' }], - updatedAt: null, - }, - id: 'so-id-2', - type: 'cases-oracle', - }, - ]); - }); - }); }); diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts index 8de7541f91552..9f21b6b0aa728 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts @@ -11,18 +11,14 @@ import type { SavedObjectsBulkResponseWithErrors } from '../../common/types'; import { isSOError } from '../../common/utils'; import { CryptoService } from './crypto_service'; import type { - BulkGetRecordsResponse, + BulkCreateOracleRecordRequest, + BulkGetOracleRecordsResponse, OracleKey, OracleRecord, OracleRecordCreateRequest, } from './types'; -import { partitionRecords } from './utils'; type OracleRecordAttributes = Omit; -type BulkCreateRequest = Array<{ - recordId: string; - payload: OracleRecordCreateRequest; -}>; export class CasesOracleService { private readonly log: Logger; @@ -73,7 +69,7 @@ export class CasesOracleService { return this.getRecordResponse(oracleRecord); } - public async bulkGetRecords(ids: string[]): Promise { + public async bulkGetRecords(ids: string[]): Promise { this.log.debug(`Getting oracle records with IDs: ${ids}`); const oracleRecords = (await this.unsecuredSavedObjectsClient.bulkGet( @@ -107,7 +103,9 @@ export class CasesOracleService { return this.getRecordResponse(oracleRecord); } - public async bulkCreateRecord(records: BulkCreateRequest): Promise { + public async bulkCreateRecord( + records: BulkCreateOracleRecordRequest + ): Promise { const recordIds = records.map((record) => record.recordId); this.log.debug(`Creating oracle record with ID: ${recordIds}`); @@ -133,40 +131,6 @@ export class CasesOracleService { return this.getBulkRecordsResponse(oracleRecords); } - public async bulkGetOrCreateRecords(records: BulkCreateRequest): Promise { - const recordsMap = new Map( - records.map(({ recordId, payload }) => [recordId, payload]) - ); - const bulkCreateReq: BulkCreateRequest = []; - - const ids = records.map(({ recordId }) => recordId); - - const bulkGetRes = await this.bulkGetRecords(ids); - const [bulkGetValidRecords, bulkGetRecordsErrors] = partitionRecords(bulkGetRes); - - for (const error of bulkGetRecordsErrors) { - if (error.id && recordsMap.has(error.id)) { - bulkCreateReq.push({ - recordId: error.id, - payload: recordsMap.get(error.id) ?? { cases: [], rules: [], grouping: {} }, - }); - } - } - - /** - * TODO: Create records with only 404 errors - * All others should throw an error and retry again - */ - const bulkCreateRes = await this.bulkCreateRecord(bulkCreateReq); - - /** - * TODO: Retry on errors - */ - const [bulkCreateValidRecords, _] = partitionRecords(bulkCreateRes); - - return [...bulkGetValidRecords, ...bulkCreateValidRecords]; - } - public async increaseCounter(recordId: string): Promise { const { id: _, version, ...record } = await this.getRecord(recordId); const newCounter = record.counter + 1; @@ -204,7 +168,7 @@ export class CasesOracleService { private getBulkRecordsResponse( oracleRecords: SavedObjectsBulkResponseWithErrors - ): BulkGetRecordsResponse { + ): BulkGetOracleRecordsResponse { return oracleRecords.saved_objects.map((oracleRecord) => { if (isSOError(oracleRecord)) { return { ...oracleRecord.error, id: oracleRecord.id }; diff --git a/x-pack/plugins/cases/server/connectors/cases/types.ts b/x-pack/plugins/cases/server/connectors/cases/types.ts index c208346b46cca..d784a7b630683 100644 --- a/x-pack/plugins/cases/server/connectors/cases/types.ts +++ b/x-pack/plugins/cases/server/connectors/cases/types.ts @@ -51,4 +51,9 @@ export interface OracleRecordCreateRequest { grouping: Record; } -export type BulkGetRecordsResponse = Array; +export type BulkGetOracleRecordsResponse = Array; + +export type BulkCreateOracleRecordRequest = Array<{ + recordId: string; + payload: OracleRecordCreateRequest; +}>; diff --git a/x-pack/plugins/cases/server/connectors/cases/utils.ts b/x-pack/plugins/cases/server/connectors/cases/utils.ts index 24064efab1439..38f6a23c490fb 100644 --- a/x-pack/plugins/cases/server/connectors/cases/utils.ts +++ b/x-pack/plugins/cases/server/connectors/cases/utils.ts @@ -6,13 +6,13 @@ */ import { partition } from 'lodash'; -import type { BulkGetRecordsResponse, OracleRecord, OracleRecordError } from './types'; +import type { BulkGetOracleRecordsResponse, OracleRecord, OracleRecordError } from './types'; export const isRecordError = (so: OracleRecord | OracleRecordError): so is OracleRecordError => (so as OracleRecordError).error != null; export const partitionRecords = ( - res: BulkGetRecordsResponse + res: BulkGetOracleRecordsResponse ): [OracleRecord[], OracleRecordError[]] => { const [errors, validRecords] = partition(res, isRecordError) as [ OracleRecordError[], From 40f865e2719f8c1cebc195367e3052169d9245b5 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Fri, 20 Oct 2023 16:51:39 +0300 Subject: [PATCH 26/40] Generate case ids --- .../connectors/cases/cases_connector.test.ts | 90 +++++++++-- .../connectors/cases/cases_connector.ts | 80 ++++++++-- .../connectors/cases/cases_service.test.ts | 148 ++++++++++++++++++ .../server/connectors/cases/cases_service.ts | 35 +++++ .../cases/server/connectors/cases/types.ts | 2 + 5 files changed, 321 insertions(+), 34 deletions(-) create mode 100644 x-pack/plugins/cases/server/connectors/cases/cases_service.test.ts create mode 100644 x-pack/plugins/cases/server/connectors/cases/cases_service.ts diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts index 1b064934d101b..4b479106ddaf3 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts @@ -12,10 +12,13 @@ import { CasesConnector } from './cases_connector'; import { CASES_CONNECTOR_ID } from './constants'; import { CASE_ORACLE_SAVED_OBJECT } from '../../../common/constants'; import { CasesOracleService } from './cases_oracle_service'; +import { CasesService } from './cases_service'; jest.mock('./cases_oracle_service'); +jest.mock('./cases_service'); const CasesOracleServiceMock = CasesOracleService as jest.Mock; +const CasesServiceMock = CasesService as jest.Mock; describe('CasesConnector', () => { const services = actionsMock.createServices(); @@ -81,8 +84,9 @@ describe('CasesConnector', () => { ]; const mockGetRecordId = jest.fn(); - const bulkGetRecords = jest.fn(); - const bulkCreateRecord = jest.fn(); + const mockBulkGetRecords = jest.fn(); + const mockBulkCreateRecord = jest.fn(); + const mockGetCaseId = jest.fn(); let connector: CasesConnector; @@ -90,17 +94,29 @@ describe('CasesConnector', () => { jest.clearAllMocks(); CasesOracleServiceMock.mockImplementation(() => { - let idCounter = 0; + let oracleIdCounter = 0; return { - getRecordId: mockGetRecordId.mockImplementation(() => `so-oracle-record-${idCounter++}`), - bulkGetRecords: bulkGetRecords.mockResolvedValue(oracleRecords), - bulkCreateRecord: bulkCreateRecord.mockResolvedValue({ - ...oracleRecords[0], - id: groupedAlertsWithOracleKey[2].oracleKey, - grouping: groupedAlertsWithOracleKey[2].grouping, - version: 'so-version-2', - }), + getRecordId: mockGetRecordId.mockImplementation( + () => `so-oracle-record-${oracleIdCounter++}` + ), + bulkGetRecords: mockBulkGetRecords.mockResolvedValue(oracleRecords), + bulkCreateRecord: mockBulkCreateRecord.mockResolvedValue([ + { + ...oracleRecords[0], + id: groupedAlertsWithOracleKey[2].oracleKey, + grouping: groupedAlertsWithOracleKey[2].grouping, + version: 'so-version-2', + }, + ]), + }; + }); + + CasesServiceMock.mockImplementation(() => { + let caseIdCounter = 0; + + return { + getCaseId: mockGetCaseId.mockImplementation(() => `so-case-id-${caseIdCounter++}`), }; }); @@ -116,9 +132,31 @@ describe('CasesConnector', () => { describe('run', () => { describe('Oracle records', () => { - it('generates the oracle keys correctly', async () => { + it('generates the oracle keys correctly with grouping by one field', async () => { + await connector.run({ alerts, groupingBy: ['host.name'], owner, rule }); + + expect(mockGetRecordId).toHaveBeenCalledTimes(2); + + expect(mockGetRecordId).nthCalledWith(1, { + ruleId: rule.id, + grouping: { 'host.name': 'A' }, + owner, + spaceId: 'default', + }); + + expect(mockGetRecordId).nthCalledWith(2, { + ruleId: rule.id, + grouping: { 'host.name': 'B' }, + owner, + spaceId: 'default', + }); + }); + + it('generates the oracle keys correct with grouping by multiple fields', async () => { await connector.run({ alerts, groupingBy, owner, rule }); + expect(mockGetRecordId).toHaveBeenCalledTimes(3); + for (const [index, { grouping }] of groupedAlertsWithOracleKey.entries()) { expect(mockGetRecordId).nthCalledWith(index + 1, { ruleId: rule.id, @@ -132,17 +170,17 @@ describe('CasesConnector', () => { it('gets the oracle records correctly', async () => { await connector.run({ alerts, groupingBy, owner, rule }); - expect(bulkGetRecords).toHaveBeenCalledWith([ + expect(mockBulkGetRecords).toHaveBeenCalledWith([ groupedAlertsWithOracleKey[0].oracleKey, groupedAlertsWithOracleKey[1].oracleKey, groupedAlertsWithOracleKey[2].oracleKey, ]); }); - it('created the no found oracle records correctly', async () => { + it('created the non found oracle records correctly', async () => { await connector.run({ alerts, groupingBy, owner, rule }); - expect(bulkCreateRecord).toHaveBeenCalledWith([ + expect(mockBulkCreateRecord).toHaveBeenCalledWith([ { recordId: groupedAlertsWithOracleKey[2].oracleKey, payload: { @@ -155,11 +193,29 @@ describe('CasesConnector', () => { }); it('does not create oracle records if there are no 404 errors', async () => { - bulkGetRecords.mockResolvedValue([oracleRecords[0]]); + mockBulkGetRecords.mockResolvedValue([oracleRecords[0]]); await connector.run({ alerts, groupingBy, owner, rule }); - expect(bulkCreateRecord).not.toHaveBeenCalled(); + expect(mockBulkCreateRecord).not.toHaveBeenCalled(); + }); + }); + + describe('Cases', () => { + it('generates the case ids correctly', async () => { + await connector.run({ alerts, groupingBy, owner, rule }); + + expect(mockGetCaseId).toHaveBeenCalledTimes(3); + + for (const [index, { grouping }] of groupedAlertsWithOracleKey.entries()) { + expect(mockGetCaseId).nthCalledWith(index + 1, { + ruleId: rule.id, + grouping, + owner, + spaceId: 'default', + counter: 1, + }); + } }); }); }); diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts index 0ed9ba47e0546..f5768102579e0 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts @@ -21,6 +21,7 @@ import type { import { CasesConnectorRunParamsSchema } from './schema'; import { CasesOracleService } from './cases_oracle_service'; import { partitionRecords } from './utils'; +import { CasesService } from './cases_service'; interface GroupedAlerts { alerts: CasesConnectorRunParams['alerts']; @@ -28,15 +29,18 @@ interface GroupedAlerts { } type GroupedAlertsWithOracleKey = GroupedAlerts & { oracleKey: string }; +type GroupedAlertsWithCaseId = GroupedAlertsWithOracleKey & { caseId: string }; export class CasesConnector extends SubActionConnector< CasesConnectorConfig, CasesConnectorSecrets > { - private readonly casesOracleService; + private readonly casesOracleService: CasesOracleService; + private readonly casesService: CasesService; constructor(params: ServiceParams) { super(params); + this.casesOracleService = new CasesOracleService({ log: this.logger, /** @@ -46,6 +50,9 @@ export class CasesConnector extends SubActionConnector< */ unsecuredSavedObjectsClient: this.savedObjectsClient, }); + + this.casesService = new CasesService(); + this.registerSubActions(); } @@ -66,6 +73,30 @@ export class CasesConnector extends SubActionConnector< throw new Error('Method not implemented.'); } + public async run(params: CasesConnectorRunParams) { + const { alerts, groupingBy } = params; + + /** + * TODO: Handle when grouping is not defined + * One case should be created per rule + */ + const groupedAlerts = this.groupAlerts({ alerts, groupingBy }); + const groupedAlertsWithOracleKey = this.generateOracleKeys(params, groupedAlerts); + + /** + * Add circuit breakers to the number of oracles they can be created or retrieved + */ + const oracleRecords = await this.bulkGetOrCreateOracleRecords( + Array.from(groupedAlertsWithOracleKey.values()) + ); + + const groupedAlertsWithCaseId = this.generateCaseIds( + params, + groupedAlertsWithOracleKey, + oracleRecords + ); + } + private groupAlerts({ alerts, groupingBy, @@ -94,7 +125,7 @@ export class CasesConnector extends SubActionConnector< private generateOracleKeys( params: CasesConnectorRunParams, groupedAlerts: GroupedAlerts[] - ): GroupedAlertsWithOracleKey[] { + ): Map { const { rule, owner } = params; /** * TODO: Take spaceId from the actions framework @@ -114,7 +145,7 @@ export class CasesConnector extends SubActionConnector< oracleMap.set(oracleKey, { oracleKey, grouping, alerts }); } - return Array.from(oracleMap.values()); + return oracleMap; } private async bulkGetOrCreateOracleRecords( @@ -162,22 +193,37 @@ export class CasesConnector extends SubActionConnector< return [...bulkGetValidRecords, ...bulkCreateValidRecords]; } - public async run(params: CasesConnectorRunParams) { - const { alerts, groupingBy } = params; + private generateCaseIds( + params: CasesConnectorRunParams, + groupedAlertsWithOracleKey: Map, + oracleRecords: OracleRecord[] + ): Map { + const { rule, owner } = params; /** - * TODO: Handle when grouping is not defined - * One case should be created per rule - */ - const groupedAlerts = this.groupAlerts({ alerts, groupingBy }); - const groupedAlertsWithOracleKey = this.generateOracleKeys(params, groupedAlerts); - console.log( - '🚀 ~ file: cases_connector.ts:175 ~ run ~ groupedAlertsWithOracleKey:', - groupedAlertsWithOracleKey - ); - /** - * Add circuit breakers to the number of oracles they can be created or retrieved + * TODO: Take spaceId from the actions framework */ - const oracleRecords = this.bulkGetOrCreateOracleRecords(groupedAlertsWithOracleKey); + const spaceId = 'default'; + + const casesMap = new Map(); + + for (const oracleRecord of oracleRecords) { + const { alerts, grouping } = groupedAlertsWithOracleKey.get(oracleRecord.id) ?? { + alerts: [], + grouping: {}, + }; + + const caseId = this.casesService.getCaseId({ + ruleId: rule.id, + grouping, + owner, + spaceId, + counter: oracleRecord.counter, + }); + + casesMap.set(caseId, { caseId, alerts, grouping, oracleKey: oracleRecord.id }); + } + + return casesMap; } } diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_service.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_service.test.ts new file mode 100644 index 0000000000000..5ea9b51bad3ab --- /dev/null +++ b/x-pack/plugins/cases/server/connectors/cases/cases_service.test.ts @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createHash } from 'node:crypto'; +import stringify from 'json-stable-stringify'; + +import { isEmpty, set } from 'lodash'; +import { CasesService } from './cases_service'; + +describe('CasesService', () => { + let service: CasesService; + + beforeEach(() => { + jest.resetAllMocks(); + service = new CasesService(); + }); + + describe('getCaseId', () => { + it('return the record ID correctly', async () => { + const ruleId = 'test-rule-id'; + const spaceId = 'default'; + const owner = 'cases'; + const grouping = { 'host.ip': '0.0.0.1' }; + const counter = 1; + + const payload = `${ruleId}:${spaceId}:${owner}:${stringify(grouping)}:${counter}`; + const hash = createHash('sha256'); + + hash.update(payload); + + const hex = hash.digest('hex'); + + expect(service.getCaseId({ ruleId, spaceId, owner, grouping, counter })).toEqual(hex); + }); + + it('sorts the grouping definition correctly', async () => { + const ruleId = 'test-rule-id'; + const spaceId = 'default'; + const owner = 'cases'; + const grouping = { 'host.ip': '0.0.0.1', 'agent.id': '8a4f500d' }; + const sortedGrouping = { 'agent.id': '8a4f500d', 'host.ip': '0.0.0.1' }; + const counter = 1; + + const payload = `${ruleId}:${spaceId}:${owner}:${stringify(sortedGrouping)}:${counter}`; + const hash = createHash('sha256'); + + hash.update(payload); + + const hex = hash.digest('hex'); + + expect(service.getCaseId({ ruleId, spaceId, owner, grouping, counter })).toEqual(hex); + }); + + it('return the record ID correctly without grouping', async () => { + const ruleId = 'test-rule-id'; + const spaceId = 'default'; + const owner = 'cases'; + const counter = 1; + + const payload = `${ruleId}:${spaceId}:${owner}:${counter}`; + const hash = createHash('sha256'); + + hash.update(payload); + + const hex = hash.digest('hex'); + + expect(service.getCaseId({ ruleId, spaceId, owner, counter })).toEqual(hex); + }); + + it('return the record ID correctly with empty grouping', async () => { + const ruleId = 'test-rule-id'; + const spaceId = 'default'; + const owner = 'cases'; + const grouping = {}; + const counter = 1; + + const payload = `${ruleId}:${spaceId}:${owner}:${stringify(grouping)}:${counter}`; + const hash = createHash('sha256'); + + hash.update(payload); + + const hex = hash.digest('hex'); + + expect(service.getCaseId({ ruleId, spaceId, owner, grouping, counter })).toEqual(hex); + }); + + it('return the record ID correctly without rule', async () => { + const spaceId = 'default'; + const owner = 'cases'; + const grouping = { 'host.ip': '0.0.0.1' }; + const counter = 1; + + const payload = `${spaceId}:${owner}:${stringify(grouping)}:${counter}`; + const hash = createHash('sha256'); + + hash.update(payload); + + const hex = hash.digest('hex'); + + expect(service.getCaseId({ spaceId, owner, grouping, counter })).toEqual(hex); + }); + + it('throws an error when the ruleId and the grouping is missing', async () => { + const spaceId = 'default'; + const owner = 'cases'; + const counter = 1; + + expect(() => + // @ts-expect-error: ruleId and grouping are omitted for testing + service.getCaseId({ spaceId, owner, counter }) + ).toThrowErrorMatchingInlineSnapshot(`"ruleID or grouping is required"`); + }); + + it.each(['ruleId', 'spaceId', 'owner'])( + 'return the record ID correctly with empty string for %s', + async (key) => { + const getPayloadValue = (value: string) => (isEmpty(value) ? '' : `${value}:`); + + const params = { + ruleId: 'test-rule-id', + spaceId: 'default', + owner: 'cases', + }; + + const grouping = { 'host.ip': '0.0.0.1' }; + const counter = 1; + + set(params, key, ''); + + const payload = `${getPayloadValue(params.ruleId)}${getPayloadValue( + params.spaceId + )}${getPayloadValue(params.owner)}${stringify(grouping)}:${counter}`; + + const hash = createHash('sha256'); + + hash.update(payload); + + const hex = hash.digest('hex'); + + expect(service.getCaseId({ ...params, grouping, counter })).toEqual(hex); + } + ); + }); +}); diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_service.ts b/x-pack/plugins/cases/server/connectors/cases/cases_service.ts new file mode 100644 index 0000000000000..214049e5acc0c --- /dev/null +++ b/x-pack/plugins/cases/server/connectors/cases/cases_service.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CryptoService } from './crypto_service'; +import type { CaseIdPayload } from './types'; + +export class CasesService { + private cryptoService: CryptoService; + + constructor() { + this.cryptoService = new CryptoService(); + } + + public getCaseId({ ruleId, spaceId, owner, grouping, counter }: CaseIdPayload): string { + if (grouping == null && ruleId == null) { + throw new Error('ruleID or grouping is required'); + } + + const payload = [ + ruleId, + spaceId, + owner, + this.cryptoService.stringifyDeterministically(grouping), + counter, + ] + .filter(Boolean) + .join(':'); + + return this.cryptoService.getHash(payload); + } +} diff --git a/x-pack/plugins/cases/server/connectors/cases/types.ts b/x-pack/plugins/cases/server/connectors/cases/types.ts index d784a7b630683..ac55e90b0b61f 100644 --- a/x-pack/plugins/cases/server/connectors/cases/types.ts +++ b/x-pack/plugins/cases/server/connectors/cases/types.ts @@ -32,6 +32,8 @@ type OracleKeyWithOptionalGrouping = Optional; export type OracleKey = ExclusiveUnion; +export type CaseIdPayload = OracleKey & { counter: number }; + export interface OracleRecord { id: string; counter: number; From f70bf1e0af41a6d34d5f25ec6c7cc4d6934c7174 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Sat, 21 Oct 2023 15:07:22 +0300 Subject: [PATCH 27/40] Get service with a factory --- .../sub_action_framework/executor.test.ts | 5 +++-- .../server/sub_action_framework/executor.ts | 2 +- .../sub_action_framework/register.test.ts | 4 +++- .../server/sub_action_framework/register.ts | 17 +---------------- .../server/sub_action_framework/types.ts | 2 +- .../sub_action_framework/validators.test.ts | 8 +++++--- .../server/connector_types/bedrock/index.ts | 2 +- .../server/connector_types/d3security/index.ts | 2 +- .../server/connector_types/openai/index.ts | 2 +- .../server/connector_types/opsgenie/index.ts | 2 +- .../server/connector_types/sentinelone/index.ts | 2 +- .../server/connector_types/tines/index.ts | 2 +- .../alerts/server/sub_action_connector.ts | 5 +++-- 13 files changed, 23 insertions(+), 32 deletions(-) diff --git a/x-pack/plugins/actions/server/sub_action_framework/executor.test.ts b/x-pack/plugins/actions/server/sub_action_framework/executor.test.ts index 92467f049ae3f..1280c70d0cd84 100644 --- a/x-pack/plugins/actions/server/sub_action_framework/executor.test.ts +++ b/x-pack/plugins/actions/server/sub_action_framework/executor.test.ts @@ -19,7 +19,7 @@ import { TestSecrets, TestExecutor, } from './mocks'; -import { IService } from './types'; +import { IService, ServiceParams } from './types'; describe('Executor', () => { const actionId = 'test-action-id'; @@ -40,7 +40,8 @@ describe('Executor', () => { config: TestConfigSchema, secrets: TestSecretsSchema, }, - Service, + getService: (serviceParams: ServiceParams) => + new Service(serviceParams), }; return buildExecutor({ configurationUtilities: mockedActionsConfig, logger, connector }); diff --git a/x-pack/plugins/actions/server/sub_action_framework/executor.ts b/x-pack/plugins/actions/server/sub_action_framework/executor.ts index 12368a834ea7e..9ac7a63dc2e96 100644 --- a/x-pack/plugins/actions/server/sub_action_framework/executor.ts +++ b/x-pack/plugins/actions/server/sub_action_framework/executor.ts @@ -33,7 +33,7 @@ export const buildExecutor = < const subAction = params.subAction; const subActionParams = params.subActionParams; - const service = new connector.Service({ + const service = connector.getService({ connector: { id: actionId, type: connector.id }, config, secrets, diff --git a/x-pack/plugins/actions/server/sub_action_framework/register.test.ts b/x-pack/plugins/actions/server/sub_action_framework/register.test.ts index 6788ca6e5e6fb..df454a0f0020b 100644 --- a/x-pack/plugins/actions/server/sub_action_framework/register.test.ts +++ b/x-pack/plugins/actions/server/sub_action_framework/register.test.ts @@ -16,6 +16,7 @@ import { TestSubActionConnector, } from './mocks'; import { register } from './register'; +import { ServiceParams } from './types'; describe('Registration', () => { const renderedVariables = { body: '' }; @@ -30,7 +31,8 @@ describe('Registration', () => { config: TestConfigSchema, secrets: TestSecretsSchema, }, - Service: TestSubActionConnector, + getService: (serviceParams: ServiceParams) => + new TestSubActionConnector(serviceParams), renderParameterTemplates: mockRenderParameterTemplates, }; diff --git a/x-pack/plugins/actions/server/sub_action_framework/register.ts b/x-pack/plugins/actions/server/sub_action_framework/register.ts index 9d5bd91a88866..80331648244ae 100644 --- a/x-pack/plugins/actions/server/sub_action_framework/register.ts +++ b/x-pack/plugins/actions/server/sub_action_framework/register.ts @@ -9,24 +9,11 @@ import { PublicMethodsOf } from '@kbn/utility-types'; import { Logger } from '@kbn/core/server'; import { ActionsConfigurationUtilities } from '../actions_config'; import { ActionTypeRegistry } from '../action_type_registry'; -import { SubActionConnector } from './sub_action_connector'; -import { CaseConnector } from './case'; import { ActionTypeConfig, ActionTypeSecrets } from '../types'; import { buildExecutor } from './executor'; -import { ExecutorParams, SubActionConnectorType, IService } from './types'; +import { ExecutorParams, SubActionConnectorType } from './types'; import { buildValidators } from './validators'; -const validateService = (Service: IService) => { - if ( - !(Service.prototype instanceof CaseConnector) && - !(Service.prototype instanceof SubActionConnector) - ) { - throw new Error( - 'Service must be extend one of the abstract classes: SubActionConnector or CaseConnector' - ); - } -}; - export const register = ({ actionTypeRegistry, connector, @@ -38,8 +25,6 @@ export const register = ; logger: Logger; }) => { - validateService(connector.Service); - const validators = buildValidators({ connector, configurationUtilities }); const executor = buildExecutor({ connector, diff --git a/x-pack/plugins/actions/server/sub_action_framework/types.ts b/x-pack/plugins/actions/server/sub_action_framework/types.ts index 26b1fd20020d2..2f5ae6bad769b 100644 --- a/x-pack/plugins/actions/server/sub_action_framework/types.ts +++ b/x-pack/plugins/actions/server/sub_action_framework/types.ts @@ -79,7 +79,7 @@ export interface SubActionConnectorType { secrets: Type; }; validators?: Array | SecretsValidator>; - Service: IService; + getService: (params: ServiceParams) => SubActionConnector; renderParameterTemplates?: RenderParameterTemplates; } diff --git a/x-pack/plugins/actions/server/sub_action_framework/validators.test.ts b/x-pack/plugins/actions/server/sub_action_framework/validators.test.ts index b28adc0b545bf..244873c657431 100644 --- a/x-pack/plugins/actions/server/sub_action_framework/validators.test.ts +++ b/x-pack/plugins/actions/server/sub_action_framework/validators.test.ts @@ -14,7 +14,7 @@ import { TestSecrets, TestSubActionConnector, } from './mocks'; -import { IService, SubActionConnectorType, ValidatorType } from './types'; +import { IService, ServiceParams, SubActionConnectorType, ValidatorType } from './types'; import { buildValidators } from './validators'; describe('Validators', () => { @@ -30,7 +30,8 @@ describe('Validators', () => { config: TestConfigSchema, secrets: TestSecretsSchema, }, - Service, + getService: (serviceParams: ServiceParams) => + new Service(serviceParams), }; return buildValidators({ configurationUtilities: mockedActionsConfig, connector }); @@ -59,7 +60,8 @@ describe('Validators', () => { validator: secretsValidator, }, ], - Service, + getService: (serviceParams: ServiceParams) => + new Service(serviceParams), }; return { diff --git a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/index.ts b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/index.ts index 02b2bff9a93ae..e9ab583277282 100644 --- a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/index.ts +++ b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/index.ts @@ -23,7 +23,7 @@ import { renderParameterTemplates } from './render'; export const getConnectorType = (): SubActionConnectorType => ({ id: BEDROCK_CONNECTOR_ID, name: BEDROCK_TITLE, - Service: BedrockConnector, + getService: (params) => new BedrockConnector(params), schema: { config: ConfigSchema, secrets: SecretsSchema, diff --git a/x-pack/plugins/stack_connectors/server/connector_types/d3security/index.ts b/x-pack/plugins/stack_connectors/server/connector_types/d3security/index.ts index d3fb6f0d326ad..4e98938335808 100644 --- a/x-pack/plugins/stack_connectors/server/connector_types/d3security/index.ts +++ b/x-pack/plugins/stack_connectors/server/connector_types/d3security/index.ts @@ -26,7 +26,7 @@ export function getConnectorType(): D3SecurityConnectorType { id: D3_SECURITY_CONNECTOR_ID, minimumLicenseRequired: 'gold', name: D3_SECURITY_TITLE, - Service: D3SecurityConnector, + getService: (params) => new D3SecurityConnector(params), supportedFeatureIds: [AlertingConnectorFeatureId, SecurityConnectorFeatureId], schema: { config: D3SecurityConfigSchema, diff --git a/x-pack/plugins/stack_connectors/server/connector_types/openai/index.ts b/x-pack/plugins/stack_connectors/server/connector_types/openai/index.ts index fb6a27b17bad5..9184b14b4f9c7 100644 --- a/x-pack/plugins/stack_connectors/server/connector_types/openai/index.ts +++ b/x-pack/plugins/stack_connectors/server/connector_types/openai/index.ts @@ -27,7 +27,7 @@ import { renderParameterTemplates } from './render'; export const getConnectorType = (): SubActionConnectorType => ({ id: OPENAI_CONNECTOR_ID, name: OPENAI_TITLE, - Service: OpenAIConnector, + getService: (params) => new OpenAIConnector(params), schema: { config: ConfigSchema, secrets: SecretsSchema, diff --git a/x-pack/plugins/stack_connectors/server/connector_types/opsgenie/index.ts b/x-pack/plugins/stack_connectors/server/connector_types/opsgenie/index.ts index 2f7a5d6b39680..7e92a3b7f3332 100644 --- a/x-pack/plugins/stack_connectors/server/connector_types/opsgenie/index.ts +++ b/x-pack/plugins/stack_connectors/server/connector_types/opsgenie/index.ts @@ -24,7 +24,7 @@ import { renderParameterTemplates } from './render_template_variables'; export const getOpsgenieConnectorType = (): SubActionConnectorType => { return { - Service: OpsgenieConnector, + getService: (params) => new OpsgenieConnector(params), minimumLicenseRequired: 'platinum', name: i18n.OPSGENIE_NAME, id: OpsgenieConnectorTypeId, diff --git a/x-pack/plugins/stack_connectors/server/connector_types/sentinelone/index.ts b/x-pack/plugins/stack_connectors/server/connector_types/sentinelone/index.ts index 1ce534079e829..849d54e276e11 100644 --- a/x-pack/plugins/stack_connectors/server/connector_types/sentinelone/index.ts +++ b/x-pack/plugins/stack_connectors/server/connector_types/sentinelone/index.ts @@ -26,7 +26,7 @@ export const getSentinelOneConnectorType = (): SubActionConnectorType< > => ({ id: SENTINELONE_CONNECTOR_ID, name: SENTINELONE_TITLE, - Service: SentinelOneConnector, + getService: (params) => new SentinelOneConnector(params), schema: { config: SentinelOneConfigSchema, secrets: SentinelOneSecretsSchema, diff --git a/x-pack/plugins/stack_connectors/server/connector_types/tines/index.ts b/x-pack/plugins/stack_connectors/server/connector_types/tines/index.ts index ba04fed6df58a..1f0a4560c47b0 100644 --- a/x-pack/plugins/stack_connectors/server/connector_types/tines/index.ts +++ b/x-pack/plugins/stack_connectors/server/connector_types/tines/index.ts @@ -20,7 +20,7 @@ import { renderParameterTemplates } from './render'; export const getTinesConnectorType = (): SubActionConnectorType => ({ id: TINES_CONNECTOR_ID, name: TINES_TITLE, - Service: TinesConnector, + getService: (params) => new TinesConnector(params), schema: { config: TinesConfigSchema, secrets: TinesSecretsSchema, diff --git a/x-pack/test/alerting_api_integration/common/plugins/alerts/server/sub_action_connector.ts b/x-pack/test/alerting_api_integration/common/plugins/alerts/server/sub_action_connector.ts index daf2be6dd40eb..7bd9db83dcc00 100644 --- a/x-pack/test/alerting_api_integration/common/plugins/alerts/server/sub_action_connector.ts +++ b/x-pack/test/alerting_api_integration/common/plugins/alerts/server/sub_action_connector.ts @@ -79,13 +79,14 @@ export const getTestSubActionConnector = ( public async noData() {} } + return { id: 'test.sub-action-connector', name: 'Test: Sub action connector', minimumLicenseRequired: 'platinum' as const, supportedFeatureIds: ['alerting'], schema: { config: TestConfigSchema, secrets: TestSecretsSchema }, - Service: TestSubActionConnector, + getService: (params) => new TestSubActionConnector(params), }; }; @@ -106,6 +107,6 @@ export const getTestSubActionConnectorWithoutSubActions = ( minimumLicenseRequired: 'platinum' as const, supportedFeatureIds: ['alerting'], schema: { config: TestConfigSchema, secrets: TestSecretsSchema }, - Service: TestNoSubActions, + getService: (params) => new TestNoSubActions(params), }; }; From 04805ac5460fc084fa3cfab111853236dda90bd1 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Sat, 21 Oct 2023 15:09:38 +0300 Subject: [PATCH 28/40] Fix docs --- x-pack/plugins/actions/server/sub_action_framework/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/actions/server/sub_action_framework/README.md b/x-pack/plugins/actions/server/sub_action_framework/README.md index 4aa74f6e8362e..28ff2e0e358fc 100644 --- a/x-pack/plugins/actions/server/sub_action_framework/README.md +++ b/x-pack/plugins/actions/server/sub_action_framework/README.md @@ -350,7 +350,7 @@ plugins.actions.registerSubActionConnectorType({ name: 'Test: Sub action connector', minimumLicenseRequired: 'platinum' as const, schema: { config: TestConfigSchema, secrets: TestSecretsSchema }, - Service: TestSubActionConnector, + getService: (params) => new TestSubActionConnector(params), renderParameterTemplates: renderTestTemplate }); ``` @@ -368,6 +368,6 @@ plugins.actions.registerSubActionConnectorType({ minimumLicenseRequired: 'platinum' as const, schema: { config: TestConfigSchema, secrets: TestSecretsSchema }, validators: [{type: ValidatorType.CONFIG, validate: urlAllowListValidator('url')}] - Service: TestSubActionConnector, + getService: (params) => new TestSubActionConnector(params), }); ``` From af1b8ae8c2f09de64588cb5311bf50d4fdaa7ed9 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Sat, 21 Oct 2023 15:57:47 +0300 Subject: [PATCH 29/40] Pass the cases client to the case connector --- .../connectors/cases/cases_connector.ts | 11 +++++-- .../cases/server/connectors/cases/index.ts | 13 ++++++-- .../plugins/cases/server/connectors/index.ts | 12 +++++-- x-pack/plugins/cases/server/plugin.ts | 31 ++++++++++++------- 4 files changed, 49 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts index f5768102579e0..5dbd8afd6fd07 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts @@ -9,6 +9,7 @@ import stringify from 'json-stable-stringify'; import type { ServiceParams } from '@kbn/actions-plugin/server'; import { SubActionConnector } from '@kbn/actions-plugin/server'; import { pick } from 'lodash'; +import type { KibanaRequest } from '@kbn/core-http-server'; import { CASES_CONNECTOR_SUB_ACTION } from './constants'; import type { BulkCreateOracleRecordRequest, @@ -22,6 +23,12 @@ import { CasesConnectorRunParamsSchema } from './schema'; import { CasesOracleService } from './cases_oracle_service'; import { partitionRecords } from './utils'; import { CasesService } from './cases_service'; +import type { CasesClient } from '../../client'; + +interface CasesConnectorParams { + connectorParams: ServiceParams; + casesParams: { getCasesClient: (request: KibanaRequest) => Promise }; +} interface GroupedAlerts { alerts: CasesConnectorRunParams['alerts']; @@ -38,8 +45,8 @@ export class CasesConnector extends SubActionConnector< private readonly casesOracleService: CasesOracleService; private readonly casesService: CasesService; - constructor(params: ServiceParams) { - super(params); + constructor({ connectorParams, casesParams }: CasesConnectorParams) { + super(connectorParams); this.casesOracleService = new CasesOracleService({ log: this.logger, diff --git a/x-pack/plugins/cases/server/connectors/cases/index.ts b/x-pack/plugins/cases/server/connectors/cases/index.ts index 244474f286b8f..b45a1b1bdffcb 100644 --- a/x-pack/plugins/cases/server/connectors/cases/index.ts +++ b/x-pack/plugins/cases/server/connectors/cases/index.ts @@ -7,18 +7,27 @@ import { SecurityConnectorFeatureId, UptimeConnectorFeatureId } from '@kbn/actions-plugin/common'; import type { SubActionConnectorType } from '@kbn/actions-plugin/server/sub_action_framework/types'; +import type { KibanaRequest } from '@kbn/core-http-server'; import { CasesConnector } from './cases_connector'; import { CASES_CONNECTOR_ID, CASES_CONNECTOR_TITLE } from './constants'; import type { CasesConnectorConfig, CasesConnectorSecrets } from './types'; import { CasesConnectorConfigSchema, CasesConnectorSecretsSchema } from './schema'; +import type { CasesClient } from '../../client'; -export const getCasesConnectorType = (): SubActionConnectorType< +interface GetCasesConnectorTypeArgs { + getCasesClient: (request: KibanaRequest) => Promise; +} + +export const getCasesConnectorType = ({ + getCasesClient, +}: GetCasesConnectorTypeArgs): SubActionConnectorType< CasesConnectorConfig, CasesConnectorSecrets > => ({ id: CASES_CONNECTOR_ID, name: CASES_CONNECTOR_TITLE, - Service: CasesConnector, + getService: (params) => + new CasesConnector({ casesParams: { getCasesClient }, connectorParams: params }), schema: { config: CasesConnectorConfigSchema, secrets: CasesConnectorSecretsSchema, diff --git a/x-pack/plugins/cases/server/connectors/index.ts b/x-pack/plugins/cases/server/connectors/index.ts index 329eb6a911133..7e83e97b2e6e1 100644 --- a/x-pack/plugins/cases/server/connectors/index.ts +++ b/x-pack/plugins/cases/server/connectors/index.ts @@ -6,11 +6,19 @@ */ import type { PluginSetupContract as ActionsPluginSetupContract } from '@kbn/actions-plugin/server'; +import type { KibanaRequest } from '@kbn/core-http-server'; +import type { CasesClient } from '../client'; import { getCasesConnectorType } from './cases'; export * from './types'; export { casesConnectors } from './factory'; -export function registerConnectorTypes({ actions }: { actions: ActionsPluginSetupContract }) { - actions.registerSubActionConnectorType(getCasesConnectorType()); +export function registerConnectorTypes({ + actions, + getCasesClient, +}: { + actions: ActionsPluginSetupContract; + getCasesClient: (request: KibanaRequest) => Promise; +}) { + actions.registerSubActionConnectorType(getCasesConnectorType({ getCasesClient })); } diff --git a/x-pack/plugins/cases/server/plugin.ts b/x-pack/plugins/cases/server/plugin.ts index 66460404556b0..0ce4eb626cb35 100644 --- a/x-pack/plugins/cases/server/plugin.ts +++ b/x-pack/plugins/cases/server/plugin.ts @@ -179,7 +179,12 @@ export class CasePlugin { plugins.licensing.featureUsage.register(LICENSING_CASE_ASSIGNMENT_FEATURE, 'platinum'); - registerConnectorTypes({ actions: plugins.actions }); + const getCasesClient = async (request: KibanaRequest): Promise => { + const [coreStart] = await core.getStartServices(); + return this.getCasesClientWithRequest(coreStart)(request); + }; + + registerConnectorTypes({ actions: plugins.actions, getCasesClient }); return { attachmentFramework: { @@ -232,18 +237,8 @@ export class CasePlugin { filesPluginStart: plugins.files, }); - const client = core.elasticsearch.client; - - const getCasesClientWithRequest = async (request: KibanaRequest): Promise => { - return this.clientFactory.create({ - request, - scopedClusterClient: client.asScoped(request).asCurrentUser, - savedObjectsService: core.savedObjects, - }); - }; - return { - getCasesClientWithRequest, + getCasesClientWithRequest: this.getCasesClientWithRequest(core), getExternalReferenceAttachmentTypeRegistry: () => this.externalReferenceAttachmentTypeRegistry, getPersistableStateAttachmentTypeRegistry: () => this.persistableStateAttachmentTypeRegistry, @@ -274,4 +269,16 @@ export class CasePlugin { }; }; }; + + private getCasesClientWithRequest = + (core: CoreStart) => + async (request: KibanaRequest): Promise => { + const client = core.elasticsearch.client; + + return this.clientFactory.create({ + request, + scopedClusterClient: client.asScoped(request).asCurrentUser, + savedObjectsService: core.savedObjects, + }); + }; } From 4c6f5ae8552951d6449799f7a7a7943f2a81903d Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Mon, 23 Oct 2023 18:14:53 +0300 Subject: [PATCH 30/40] Attach alerts to a case --- .../connectors/cases/cases_connector.test.ts | 415 +++++++++++++----- .../connectors/cases/cases_connector.ts | 84 +++- .../server/connectors/cases/constants.ts | 1 + 3 files changed, 400 insertions(+), 100 deletions(-) diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts index 4b479106ddaf3..81cd5564d0a85 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts @@ -13,6 +13,9 @@ import { CASES_CONNECTOR_ID } from './constants'; import { CASE_ORACLE_SAVED_OBJECT } from '../../../common/constants'; import { CasesOracleService } from './cases_oracle_service'; import { CasesService } from './cases_service'; +import { createCasesClientMock } from '../../client/mocks'; +import { mockCases } from '../../mocks'; +import type { Cases } from '../../../common'; jest.mock('./cases_oracle_service'); jest.mock('./cases_service'); @@ -24,11 +27,23 @@ describe('CasesConnector', () => { const services = actionsMock.createServices(); const alerts = [ - { 'host.name': 'A', 'dest.ip': '0.0.0.1', 'source.ip': '0.0.0.2' }, - { 'host.name': 'B', 'dest.ip': '0.0.0.1', 'file.hash': '12345' }, - { 'host.name': 'A', 'dest.ip': '0.0.0.1' }, - { 'host.name': 'B', 'dest.ip': '0.0.0.3' }, - { 'host.name': 'A', 'source.ip': '0.0.0.5' }, + { + _id: 'alert-id-0', + _index: 'alert-index-0', + 'host.name': 'A', + 'dest.ip': '0.0.0.1', + 'source.ip': '0.0.0.2', + }, + { + _id: 'alert-id-1', + _index: 'alert-index-1', + 'host.name': 'B', + 'dest.ip': '0.0.0.1', + 'file.hash': '12345', + }, + { _id: 'alert-id-2', _index: 'alert-index-2', 'host.name': 'A', 'dest.ip': '0.0.0.1' }, + { _id: 'alert-id-3', _index: 'alert-index-3', 'host.name': 'B', 'dest.ip': '0.0.0.3' }, + { _id: 'alert-id-4', _index: 'alert-index-4', 'host.name': 'A', 'source.ip': '0.0.0.5' }, ]; const groupingBy = ['host.name', 'dest.ip']; @@ -83,139 +98,341 @@ describe('CasesConnector', () => { }, ]; + const cases: Cases = mockCases.map((so) => ({ + ...so.attributes, + id: so.id, + version: so.version ?? '', + totalComment: 0, + totalAlerts: 0, + })); + const mockGetRecordId = jest.fn(); const mockBulkGetRecords = jest.fn(); const mockBulkCreateRecord = jest.fn(); const mockGetCaseId = jest.fn(); + const getCasesClient = jest.fn(); + const casesClientMock = createCasesClientMock(); + let connector: CasesConnector; - beforeEach(() => { - jest.clearAllMocks(); - - CasesOracleServiceMock.mockImplementation(() => { - let oracleIdCounter = 0; - - return { - getRecordId: mockGetRecordId.mockImplementation( - () => `so-oracle-record-${oracleIdCounter++}` - ), - bulkGetRecords: mockBulkGetRecords.mockResolvedValue(oracleRecords), - bulkCreateRecord: mockBulkCreateRecord.mockResolvedValue([ - { - ...oracleRecords[0], - id: groupedAlertsWithOracleKey[2].oracleKey, - grouping: groupedAlertsWithOracleKey[2].grouping, - version: 'so-version-2', - }, - ]), - }; - }); + describe('With grouping', () => { + beforeEach(() => { + jest.clearAllMocks(); + + CasesOracleServiceMock.mockImplementation(() => { + let oracleIdCounter = 0; + + return { + getRecordId: mockGetRecordId.mockImplementation( + () => `so-oracle-record-${oracleIdCounter++}` + ), + bulkGetRecords: mockBulkGetRecords.mockResolvedValue(oracleRecords), + bulkCreateRecord: mockBulkCreateRecord.mockResolvedValue([ + { + ...oracleRecords[0], + id: groupedAlertsWithOracleKey[2].oracleKey, + grouping: groupedAlertsWithOracleKey[2].grouping, + version: 'so-version-2', + }, + ]), + }; + }); - CasesServiceMock.mockImplementation(() => { - let caseIdCounter = 0; + CasesServiceMock.mockImplementation(() => { + let caseIdCounter = 0; - return { - getCaseId: mockGetCaseId.mockImplementation(() => `so-case-id-${caseIdCounter++}`), - }; - }); + return { + getCaseId: mockGetCaseId.mockImplementation(() => `mock-id-${++caseIdCounter}`), + }; + }); - connector = new CasesConnector({ - configurationUtilities: actionsConfigMock.create(), - config: {}, - secrets: {}, - connector: { id: '1', type: CASES_CONNECTOR_ID }, - logger: loggingSystemMock.createLogger(), - services, + casesClientMock.cases.bulkGet.mockResolvedValue({ cases, errors: [] }); + + getCasesClient.mockReturnValue(casesClientMock); + + connector = new CasesConnector({ + casesParams: { getCasesClient }, + connectorParams: { + configurationUtilities: actionsConfigMock.create(), + config: {}, + secrets: {}, + connector: { id: '1', type: CASES_CONNECTOR_ID }, + logger: loggingSystemMock.createLogger(), + services, + }, + }); }); - }); - describe('run', () => { - describe('Oracle records', () => { - it('generates the oracle keys correctly with grouping by one field', async () => { - await connector.run({ alerts, groupingBy: ['host.name'], owner, rule }); + describe('run', () => { + describe('Oracle records', () => { + it('generates the oracle keys correctly with grouping by one field', async () => { + await connector.run({ alerts, groupingBy: ['host.name'], owner, rule }); - expect(mockGetRecordId).toHaveBeenCalledTimes(2); + expect(mockGetRecordId).toHaveBeenCalledTimes(2); - expect(mockGetRecordId).nthCalledWith(1, { - ruleId: rule.id, - grouping: { 'host.name': 'A' }, - owner, - spaceId: 'default', + expect(mockGetRecordId).nthCalledWith(1, { + ruleId: rule.id, + grouping: { 'host.name': 'A' }, + owner, + spaceId: 'default', + }); + + expect(mockGetRecordId).nthCalledWith(2, { + ruleId: rule.id, + grouping: { 'host.name': 'B' }, + owner, + spaceId: 'default', + }); }); - expect(mockGetRecordId).nthCalledWith(2, { - ruleId: rule.id, - grouping: { 'host.name': 'B' }, - owner, - spaceId: 'default', + it('generates the oracle keys correct with grouping by multiple fields', async () => { + await connector.run({ alerts, groupingBy, owner, rule }); + + expect(mockGetRecordId).toHaveBeenCalledTimes(3); + + for (const [index, { grouping }] of groupedAlertsWithOracleKey.entries()) { + expect(mockGetRecordId).nthCalledWith(index + 1, { + ruleId: rule.id, + grouping, + owner, + spaceId: 'default', + }); + } + }); + + it('gets the oracle records correctly', async () => { + await connector.run({ alerts, groupingBy, owner, rule }); + + expect(mockBulkGetRecords).toHaveBeenCalledWith([ + groupedAlertsWithOracleKey[0].oracleKey, + groupedAlertsWithOracleKey[1].oracleKey, + groupedAlertsWithOracleKey[2].oracleKey, + ]); + }); + + it('created the non found oracle records correctly', async () => { + await connector.run({ alerts, groupingBy, owner, rule }); + + expect(mockBulkCreateRecord).toHaveBeenCalledWith([ + { + recordId: groupedAlertsWithOracleKey[2].oracleKey, + payload: { + cases: [], + grouping: groupedAlertsWithOracleKey[2].grouping, + rules: [], + }, + }, + ]); + }); + + it('does not create oracle records if there are no 404 errors', async () => { + mockBulkGetRecords.mockResolvedValue([oracleRecords[0]]); + + await connector.run({ alerts, groupingBy, owner, rule }); + + expect(mockBulkCreateRecord).not.toHaveBeenCalled(); }); }); - it('generates the oracle keys correct with grouping by multiple fields', async () => { - await connector.run({ alerts, groupingBy, owner, rule }); + describe('Cases', () => { + it('generates the case ids correctly', async () => { + await connector.run({ alerts, groupingBy, owner, rule }); + + expect(mockGetCaseId).toHaveBeenCalledTimes(3); + + for (const [index, { grouping }] of groupedAlertsWithOracleKey.entries()) { + expect(mockGetCaseId).nthCalledWith(index + 1, { + ruleId: rule.id, + grouping, + owner, + spaceId: 'default', + counter: 1, + }); + } + }); - expect(mockGetRecordId).toHaveBeenCalledTimes(3); + it('gets the cases correctly', async () => { + await connector.run({ alerts, groupingBy, owner, rule }); - for (const [index, { grouping }] of groupedAlertsWithOracleKey.entries()) { - expect(mockGetRecordId).nthCalledWith(index + 1, { - ruleId: rule.id, - grouping, - owner, - spaceId: 'default', + expect(casesClientMock.cases.bulkGet).toHaveBeenCalledWith({ + ids: ['mock-id-1', 'mock-id-2', 'mock-id-3'], }); - } + }); }); - it('gets the oracle records correctly', async () => { - await connector.run({ alerts, groupingBy, owner, rule }); + describe('Alerts', () => { + it('attach the alerts to the correct cases correctly', async () => { + await connector.run({ alerts, groupingBy, owner, rule }); + + expect(casesClientMock.attachments.bulkCreate).toHaveBeenCalledTimes(3); + + expect(casesClientMock.attachments.bulkCreate).nthCalledWith(1, { + caseId: 'mock-id-1', + attachments: [ + { + alertId: 'alert-id-0', + index: 'alert-index-0', + owner: 'securitySolution', + rule: { + id: 'rule-test-id', + name: 'Test rule', + }, + type: 'alert', + }, + { + alertId: 'alert-id-2', + index: 'alert-index-2', + owner: 'securitySolution', + rule: { + id: 'rule-test-id', + name: 'Test rule', + }, + type: 'alert', + }, + ], + }); + + expect(casesClientMock.attachments.bulkCreate).nthCalledWith(2, { + caseId: 'mock-id-2', + attachments: [ + { + alertId: 'alert-id-1', + index: 'alert-index-1', + owner: 'securitySolution', + rule: { + id: 'rule-test-id', + name: 'Test rule', + }, + type: 'alert', + }, + ], + }); - expect(mockBulkGetRecords).toHaveBeenCalledWith([ - groupedAlertsWithOracleKey[0].oracleKey, - groupedAlertsWithOracleKey[1].oracleKey, - groupedAlertsWithOracleKey[2].oracleKey, - ]); + expect(casesClientMock.attachments.bulkCreate).nthCalledWith(3, { + caseId: 'mock-id-3', + attachments: [ + { + alertId: 'alert-id-3', + index: 'alert-index-3', + owner: 'securitySolution', + rule: { + id: 'rule-test-id', + name: 'Test rule', + }, + type: 'alert', + }, + ], + }); + }); }); + }); + }); - it('created the non found oracle records correctly', async () => { - await connector.run({ alerts, groupingBy, owner, rule }); + describe('Without grouping', () => { + beforeEach(() => { + jest.clearAllMocks(); - expect(mockBulkCreateRecord).toHaveBeenCalledWith([ - { - recordId: groupedAlertsWithOracleKey[2].oracleKey, - payload: { - cases: [], - grouping: groupedAlertsWithOracleKey[2].grouping, - rules: [], - }, - }, - ]); + CasesOracleServiceMock.mockImplementation(() => { + let oracleIdCounter = 0; + + return { + getRecordId: mockGetRecordId.mockImplementation( + () => `so-oracle-record-${oracleIdCounter++}` + ), + bulkGetRecords: mockBulkGetRecords.mockResolvedValue([oracleRecords[0]]), + bulkCreateRecord: mockBulkCreateRecord.mockResolvedValue([]), + }; + }); + + CasesServiceMock.mockImplementation(() => { + let caseIdCounter = 0; + + return { + getCaseId: mockGetCaseId.mockImplementation(() => `mock-id-${++caseIdCounter}`), + }; + }); + + casesClientMock.cases.bulkGet.mockResolvedValue({ cases: [cases[0]], errors: [] }); + + getCasesClient.mockReturnValue(casesClientMock); + + connector = new CasesConnector({ + casesParams: { getCasesClient }, + connectorParams: { + configurationUtilities: actionsConfigMock.create(), + config: {}, + secrets: {}, + connector: { id: '1', type: CASES_CONNECTOR_ID }, + logger: loggingSystemMock.createLogger(), + services, + }, }); + }); + + describe('Oracle records', () => { + it('generates the oracle keys correctly with no grouping', async () => { + await connector.run({ alerts, groupingBy: [], owner, rule }); + + expect(mockGetRecordId).toHaveBeenCalledTimes(1); - it('does not create oracle records if there are no 404 errors', async () => { - mockBulkGetRecords.mockResolvedValue([oracleRecords[0]]); + expect(mockGetRecordId).nthCalledWith(1, { + ruleId: rule.id, + grouping: {}, + owner, + spaceId: 'default', + }); + }); - await connector.run({ alerts, groupingBy, owner, rule }); + it('gets the oracle records correctly', async () => { + await connector.run({ alerts, groupingBy: [], owner, rule }); - expect(mockBulkCreateRecord).not.toHaveBeenCalled(); + expect(mockBulkGetRecords).toHaveBeenCalledWith(['so-oracle-record-0']); }); }); describe('Cases', () => { it('generates the case ids correctly', async () => { - await connector.run({ alerts, groupingBy, owner, rule }); + await connector.run({ alerts, groupingBy: [], owner, rule }); - expect(mockGetCaseId).toHaveBeenCalledTimes(3); + expect(mockGetCaseId).toHaveBeenCalledTimes(1); - for (const [index, { grouping }] of groupedAlertsWithOracleKey.entries()) { - expect(mockGetCaseId).nthCalledWith(index + 1, { - ruleId: rule.id, - grouping, - owner, - spaceId: 'default', - counter: 1, - }); - } + expect(mockGetCaseId).nthCalledWith(1, { + ruleId: rule.id, + grouping: {}, + owner, + spaceId: 'default', + counter: 1, + }); + }); + + it('gets the cases correctly', async () => { + await connector.run({ alerts, groupingBy: [], owner, rule }); + + expect(casesClientMock.cases.bulkGet).toHaveBeenCalledWith({ + ids: ['mock-id-1'], + }); + }); + }); + + describe('Alerts', () => { + it('attach all alerts to the same case when the grouping is not defined', async () => { + await connector.run({ alerts, groupingBy: [], owner, rule }); + expect(casesClientMock.attachments.bulkCreate).toHaveBeenCalledTimes(1); + + expect(casesClientMock.attachments.bulkCreate).nthCalledWith(1, { + caseId: 'mock-id-1', + attachments: alerts.map((alert) => ({ + alertId: alert._id, + index: alert._index, + owner: 'securitySolution', + rule: { + id: 'rule-test-id', + name: 'Test rule', + }, + type: 'alert', + })), + }); }); }); }); diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts index 5dbd8afd6fd07..7ccd8576c1afb 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts @@ -10,7 +10,11 @@ import type { ServiceParams } from '@kbn/actions-plugin/server'; import { SubActionConnector } from '@kbn/actions-plugin/server'; import { pick } from 'lodash'; import type { KibanaRequest } from '@kbn/core-http-server'; -import { CASES_CONNECTOR_SUB_ACTION } from './constants'; +import { CoreKibanaRequest } from '@kbn/core/server'; +import pMap from 'p-map'; +import type { Case } from '../../../common'; +import { AttachmentType } from '../../../common'; +import { CASES_CONNECTOR_SUB_ACTION, MAX_CONCURRENT_REQUEST_ATTACH_ALERTS } from './constants'; import type { BulkCreateOracleRecordRequest, CasesConnectorConfig, @@ -24,6 +28,7 @@ import { CasesOracleService } from './cases_oracle_service'; import { partitionRecords } from './utils'; import { CasesService } from './cases_service'; import type { CasesClient } from '../../client'; +import type { BulkCreateArgs as BulkCreateAlertsReq } from '../../client/attachments/types'; interface CasesConnectorParams { connectorParams: ServiceParams; @@ -37,6 +42,7 @@ interface GroupedAlerts { type GroupedAlertsWithOracleKey = GroupedAlerts & { oracleKey: string }; type GroupedAlertsWithCaseId = GroupedAlertsWithOracleKey & { caseId: string }; +type GroupedAlertsWithCases = GroupedAlertsWithCaseId & { theCase: Case }; export class CasesConnector extends SubActionConnector< CasesConnectorConfig, @@ -44,6 +50,8 @@ export class CasesConnector extends SubActionConnector< > { private readonly casesOracleService: CasesOracleService; private readonly casesService: CasesService; + private readonly kibanaRequest: KibanaRequest; + private readonly casesParams: CasesConnectorParams['casesParams']; constructor({ connectorParams, casesParams }: CasesConnectorParams) { super(connectorParams); @@ -60,6 +68,14 @@ export class CasesConnector extends SubActionConnector< this.casesService = new CasesService(); + /** + * TODO: Get request from the actions framework. + * Should be set in the SubActionConnector's constructor + */ + this.kibanaRequest = CoreKibanaRequest.from({ path: '/', headers: {} }); + + this.casesParams = casesParams; + this.registerSubActions(); } @@ -82,6 +98,7 @@ export class CasesConnector extends SubActionConnector< public async run(params: CasesConnectorRunParams) { const { alerts, groupingBy } = params; + const casesClient = await this.casesParams.getCasesClient(this.kibanaRequest); /** * TODO: Handle when grouping is not defined @@ -102,6 +119,13 @@ export class CasesConnector extends SubActionConnector< groupedAlertsWithOracleKey, oracleRecords ); + + const groupedAlertsWithCases = await this.bulkGetOrCreateCases( + casesClient, + groupedAlertsWithCaseId + ); + + await this.attachAlertsToCases(casesClient, groupedAlertsWithCases, params); } private groupAlerts({ @@ -233,4 +257,62 @@ export class CasesConnector extends SubActionConnector< return casesMap; } + + private async bulkGetOrCreateCases( + casesClient: CasesClient, + groupedAlertsWithCaseId: Map + ): Promise> { + const casesMap = new Map(); + + const ids = Array.from(groupedAlertsWithCaseId.values()).map(({ caseId }) => caseId); + const { cases, errors } = await casesClient.cases.bulkGet({ ids }); + + for (const theCase of cases) { + if (groupedAlertsWithCaseId.has(theCase.id)) { + const data = groupedAlertsWithCaseId.get(theCase.id) as GroupedAlertsWithCaseId; + casesMap.set(theCase.id, { ...data, theCase }); + } + } + + if (errors.length === 0) { + return casesMap; + } + + /** + * TODO: Bulk create cases that do not exist (404) + */ + return casesMap; + } + + private async attachAlertsToCases( + casesClient: CasesClient, + groupedAlertsWithCases: Map, + params: CasesConnectorRunParams + ): Promise { + const { rule } = params; + + const bulkCreateAlertsRequest: BulkCreateAlertsReq[] = Array.from( + groupedAlertsWithCases.values() + ).map(({ theCase, alerts }) => ({ + caseId: theCase.id, + /** + * TODO: Verify _id, _index + */ + attachments: alerts.map((alert) => ({ + type: AttachmentType.alert, + alertId: alert._id, + index: alert._index, + rule: { id: rule.id, name: rule.name }, + owner: theCase.owner, + })), + })); + + await pMap( + bulkCreateAlertsRequest, + (req: BulkCreateAlertsReq) => casesClient.attachments.bulkCreate(req), + { + concurrency: MAX_CONCURRENT_REQUEST_ATTACH_ALERTS, + } + ); + } } diff --git a/x-pack/plugins/cases/server/connectors/cases/constants.ts b/x-pack/plugins/cases/server/connectors/cases/constants.ts index bea96fcb4f387..2987881feb56a 100644 --- a/x-pack/plugins/cases/server/connectors/cases/constants.ts +++ b/x-pack/plugins/cases/server/connectors/cases/constants.ts @@ -7,6 +7,7 @@ export const CASES_CONNECTOR_ID = '.cases'; export const CASES_CONNECTOR_TITLE = 'Cases'; +export const MAX_CONCURRENT_REQUEST_ATTACH_ALERTS = 5; export enum CASES_CONNECTOR_SUB_ACTION { RUN = 'run', From c8e82e5ac397a02dc235f7cdb0a1932dc45a5a78 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Wed, 8 Nov 2023 13:31:20 +0200 Subject: [PATCH 31/40] Bulk create non existing cases --- .../connectors/cases/cases_connector.test.ts | 62 +++++++++++++++++++ .../connectors/cases/cases_connector.ts | 34 +++++++++- 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts index 81cd5564d0a85..0a594cc36a906 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts @@ -148,6 +148,7 @@ describe('CasesConnector', () => { }); casesClientMock.cases.bulkGet.mockResolvedValue({ cases, errors: [] }); + casesClientMock.cases.bulkCreate.mockResolvedValue({ cases: [] }); getCasesClient.mockReturnValue(casesClientMock); @@ -259,6 +260,67 @@ describe('CasesConnector', () => { ids: ['mock-id-1', 'mock-id-2', 'mock-id-3'], }); }); + + it('creates non existing cases', async () => { + casesClientMock.cases.bulkCreate.mockResolvedValue({ cases: [cases[2]] }); + casesClientMock.cases.bulkGet.mockResolvedValue({ + cases: [cases[0], cases[1]], + errors: [ + { + error: 'Not found', + message: 'Not found', + status: 404, + caseId: 'mock-id-3', + }, + { + error: 'Forbidden', + message: 'Unauthorized to access case', + status: 403, + caseId: 'mock-id-3', + }, + ], + }); + + await connector.run({ alerts, groupingBy, owner, rule }); + + expect(casesClientMock.cases.bulkCreate).toHaveBeenCalledWith({ + cases: [ + { + title: '', + description: '', + owner: 'cases', + settings: { + syncAlerts: false, + }, + tags: [], + connector: { + fields: null, + id: 'none', + name: 'none', + type: '.none', + }, + }, + ], + }); + }); + + it('does not creates when there are no 404 errors', async () => { + casesClientMock.cases.bulkGet.mockResolvedValue({ + cases: [cases[0], cases[1]], + errors: [ + { + error: 'Forbidden', + message: 'Unauthorized to access case', + status: 403, + caseId: 'mock-id-3', + }, + ], + }); + + await connector.run({ alerts, groupingBy, owner, rule }); + + expect(casesClientMock.cases.bulkCreate).not.toHaveBeenCalled(); + }); }); describe('Alerts', () => { diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts index 7ccd8576c1afb..7eaee16a458de 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts @@ -12,8 +12,9 @@ import { pick } from 'lodash'; import type { KibanaRequest } from '@kbn/core-http-server'; import { CoreKibanaRequest } from '@kbn/core/server'; import pMap from 'p-map'; +import type { BulkCreateCasesRequest } from '../../../common/types/api'; import type { Case } from '../../../common'; -import { AttachmentType } from '../../../common'; +import { ConnectorTypes, AttachmentType } from '../../../common'; import { CASES_CONNECTOR_SUB_ACTION, MAX_CONCURRENT_REQUEST_ATTACH_ALERTS } from './constants'; import type { BulkCreateOracleRecordRequest, @@ -121,6 +122,7 @@ export class CasesConnector extends SubActionConnector< ); const groupedAlertsWithCases = await this.bulkGetOrCreateCases( + params, casesClient, groupedAlertsWithCaseId ); @@ -259,9 +261,11 @@ export class CasesConnector extends SubActionConnector< } private async bulkGetOrCreateCases( + params: CasesConnectorRunParams, casesClient: CasesClient, groupedAlertsWithCaseId: Map ): Promise> { + const bulkCreateReq: BulkCreateCasesRequest['cases'] = []; const casesMap = new Map(); const ids = Array.from(groupedAlertsWithCaseId.values()).map(({ caseId }) => caseId); @@ -279,8 +283,34 @@ export class CasesConnector extends SubActionConnector< } /** - * TODO: Bulk create cases that do not exist (404) + * TODO: Handle different type of errors */ + for (const error of errors) { + if (groupedAlertsWithCaseId.has(error.caseId) && error.status === 404) { + bulkCreateReq.push({ + description: '', + tags: [], + title: '', + connector: { id: 'none', name: 'none', type: ConnectorTypes.none, fields: null }, + settings: { syncAlerts: false }, + owner: params.owner, + }); + } + } + + if (bulkCreateReq.length === 0) { + return casesMap; + } + + const bulkCreateCasesResponse = await casesClient.cases.bulkCreate({ cases: bulkCreateReq }); + + for (const res of bulkCreateCasesResponse.cases) { + if (groupedAlertsWithCaseId.has(res.id)) { + const data = groupedAlertsWithCaseId.get(res.id) as GroupedAlertsWithCaseId; + casesMap.set(res.id, { ...data, theCase: res }); + } + } + return casesMap; } From 1ead04985b32a55a50d2f325b6ec0d3db86c3e96 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Fri, 10 Nov 2023 13:13:30 +0200 Subject: [PATCH 32/40] Improve the case request --- .../connectors/cases/cases_connector.test.ts | 16 +++- .../connectors/cases/cases_connector.ts | 93 ++++++++++++++----- .../cases/server/connectors/cases/schema.ts | 3 +- 3 files changed, 81 insertions(+), 31 deletions(-) diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts index 0a594cc36a906..6e264fca8d15d 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts @@ -47,7 +47,12 @@ describe('CasesConnector', () => { ]; const groupingBy = ['host.name', 'dest.ip']; - const rule = { id: 'rule-test-id', name: 'Test rule', tags: ['rule', 'test'] }; + const rule = { + id: 'rule-test-id', + name: 'Test rule', + tags: ['rule', 'test'], + ruleUrl: 'https://example.com/rules/rule-test-id', + }; const owner = 'cases'; const groupedAlertsWithOracleKey = [ @@ -261,7 +266,7 @@ describe('CasesConnector', () => { }); }); - it('creates non existing cases', async () => { + it('creates non existing cases correctly', async () => { casesClientMock.cases.bulkCreate.mockResolvedValue({ cases: [cases[2]] }); casesClientMock.cases.bulkGet.mockResolvedValue({ cases: [cases[0], cases[1]], @@ -286,13 +291,14 @@ describe('CasesConnector', () => { expect(casesClientMock.cases.bulkCreate).toHaveBeenCalledWith({ cases: [ { - title: '', - description: '', + title: 'Test rule (Auto-created)', + description: + 'This case is auto-created by [Test rule](https://example.com/rules/rule-test-id). \n\n Grouping: `host.name` equals `B` and `dest.ip` equals `0.0.0.3`', owner: 'cases', settings: { syncAlerts: false, }, - tags: [], + tags: ['auto-generated', ...rule.tags], connector: { fields: null, id: 'none', diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts index 7eaee16a458de..f3d8d549a0e4d 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts @@ -101,10 +101,6 @@ export class CasesConnector extends SubActionConnector< const { alerts, groupingBy } = params; const casesClient = await this.casesParams.getCasesClient(this.kibanaRequest); - /** - * TODO: Handle when grouping is not defined - * One case should be created per rule - */ const groupedAlerts = this.groupAlerts({ alerts, groupingBy }); const groupedAlertsWithOracleKey = this.generateOracleKeys(params, groupedAlerts); @@ -203,8 +199,13 @@ export class CasesConnector extends SubActionConnector< ]) ); - for (const error of bulkGetRecordsErrors) { - if (error.id && recordsMap.has(error.id)) { + /** + * TODO: Throw/retry for other errors + */ + const nonFoundErrors = bulkGetRecordsErrors.filter((error) => error.statusCode === 404); + + for (const error of nonFoundErrors) { + if (error.id && error.statusCode === 404 && recordsMap.has(error.id)) { bulkCreateReq.push({ recordId: error.id, payload: recordsMap.get(error.id) ?? { cases: [], rules: [], grouping: {} }, @@ -212,14 +213,10 @@ export class CasesConnector extends SubActionConnector< } } - /** - * TODO: Create records with only 404 errors - * All others should throw an error and retry again - */ const bulkCreateRes = await this.casesOracleService.bulkCreateRecord(bulkCreateReq); /** - * TODO: Retry on errors + * TODO: Throw/Retry on errors */ const [bulkCreateValidRecords, _] = partitionRecords(bulkCreateRes); @@ -283,18 +280,15 @@ export class CasesConnector extends SubActionConnector< } /** - * TODO: Handle different type of errors + * TODO: Throw/retry for other errors */ - for (const error of errors) { - if (groupedAlertsWithCaseId.has(error.caseId) && error.status === 404) { - bulkCreateReq.push({ - description: '', - tags: [], - title: '', - connector: { id: 'none', name: 'none', type: ConnectorTypes.none, fields: null }, - settings: { syncAlerts: false }, - owner: params.owner, - }); + const nonFoundErrors = errors.filter((error) => error.status === 404); + + for (const error of nonFoundErrors) { + if (groupedAlertsWithCaseId.has(error.caseId)) { + const data = groupedAlertsWithCaseId.get(error.caseId) as GroupedAlertsWithCaseId; + + bulkCreateReq.push(this.getCreateCaseRequest(params, data)); } } @@ -302,6 +296,9 @@ export class CasesConnector extends SubActionConnector< return casesMap; } + /** + * TODO: bulkCreate throws an error. Retry on errors. + */ const bulkCreateCasesResponse = await casesClient.cases.bulkCreate({ cases: bulkCreateReq }); for (const res of bulkCreateCasesResponse.cases) { @@ -314,6 +311,55 @@ export class CasesConnector extends SubActionConnector< return casesMap; } + private getCreateCaseRequest( + params: CasesConnectorRunParams, + groupingData: GroupedAlertsWithCaseId + ) { + const { grouping } = groupingData; + + const ruleName = params.rule.ruleUrl + ? `[${params.rule.name}](${params.rule.ruleUrl})` + : params.rule.name; + + const groupingDescription = this.getGroupingDescription(grouping); + + const description = `This case is auto-created by ${ruleName}. \n\n Grouping: ${groupingDescription}`; + + const tags = Array.isArray(params.rule.tags) ? params.rule.tags : []; + + /** + * TODO: Add grouping info to + */ + return { + description, + tags: ['auto-generated', ...tags], + /** + * TODO: Append the counter to the name + */ + title: `${params.rule.name} (Auto-created)`, + connector: { id: 'none', name: 'none', type: ConnectorTypes.none, fields: null }, + /** + * Turn on for Security solution + */ + settings: { syncAlerts: false }, + owner: params.owner, + }; + } + + private getGroupingDescription(grouping: GroupedAlerts['grouping']) { + /** + * TODO: Handle multi values + */ + return Object.entries(grouping) + .map(([key, value]) => { + const keyAsCodeBlock = `\`${key}\``; + const valueAsCodeBlock = `\`${value}\``; + + return `${keyAsCodeBlock} equals ${valueAsCodeBlock}`; + }) + .join(' and '); + } + private async attachAlertsToCases( casesClient: CasesClient, groupedAlertsWithCases: Map, @@ -325,9 +371,6 @@ export class CasesConnector extends SubActionConnector< groupedAlertsWithCases.values() ).map(({ theCase, alerts }) => ({ caseId: theCase.id, - /** - * TODO: Verify _id, _index - */ attachments: alerts.map((alert) => ({ type: AttachmentType.alert, alertId: alert._id, diff --git a/x-pack/plugins/cases/server/connectors/cases/schema.ts b/x-pack/plugins/cases/server/connectors/cases/schema.ts index 3d38ed15ccb35..d8d08a54c87a6 100644 --- a/x-pack/plugins/cases/server/connectors/cases/schema.ts +++ b/x-pack/plugins/cases/server/connectors/cases/schema.ts @@ -9,7 +9,7 @@ import { schema } from '@kbn/config-schema'; const AlertSchema = schema.recordOf(schema.string(), schema.any(), { validate: (value) => { - if (!Object.hasOwn(value, 'id') || !Object.hasOwn(value, 'index')) { + if (!Object.hasOwn(value, '_id') || !Object.hasOwn(value, '_index')) { return 'Alert ID and index must be defined'; } }, @@ -27,6 +27,7 @@ const RuleSchema = schema.object({ * TODO: Verify limits */ tags: schema.arrayOf(schema.string({ minLength: 1, maxLength: 50 }), { minSize: 0, maxSize: 10 }), + ruleUrl: schema.nullable(schema.string()), }); /** From b31c167ee389e99528161426f772fd79da6ac207 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Fri, 10 Nov 2023 14:30:12 +0200 Subject: [PATCH 33/40] Small improvements --- .../server/connectors/cases/cases_connector.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts index f3d8d549a0e4d..b75baff8e3b49 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts @@ -133,6 +133,11 @@ export class CasesConnector extends SubActionConnector< const uniqueGroupingByFields = Array.from(new Set(groupingBy)); const groupingMap = new Map(); + /** + * We are interested in alerts that have a value for any + * of the groupingBy fields defined by the users. All other + * alerts will not be attached to any case. + */ const filteredAlerts = alerts.filter((alert) => uniqueGroupingByFields.every((groupingByField) => Object.hasOwn(alert, groupingByField)) ); @@ -205,7 +210,7 @@ export class CasesConnector extends SubActionConnector< const nonFoundErrors = bulkGetRecordsErrors.filter((error) => error.statusCode === 404); for (const error of nonFoundErrors) { - if (error.id && error.statusCode === 404 && recordsMap.has(error.id)) { + if (error.id && recordsMap.has(error.id)) { bulkCreateReq.push({ recordId: error.id, payload: recordsMap.get(error.id) ?? { cases: [], rules: [], grouping: {} }, @@ -284,6 +289,10 @@ export class CasesConnector extends SubActionConnector< */ const nonFoundErrors = errors.filter((error) => error.status === 404); + if (nonFoundErrors.length === 0) { + return casesMap; + } + for (const error of nonFoundErrors) { if (groupedAlertsWithCaseId.has(error.caseId)) { const data = groupedAlertsWithCaseId.get(error.caseId) as GroupedAlertsWithCaseId; @@ -292,10 +301,6 @@ export class CasesConnector extends SubActionConnector< } } - if (bulkCreateReq.length === 0) { - return casesMap; - } - /** * TODO: bulkCreate throws an error. Retry on errors. */ From 2621c7d7e74e376f6441c09c50e2930732fb64dc Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Fri, 10 Nov 2023 16:06:23 +0200 Subject: [PATCH 34/40] Add time window to schema --- .../server/connectors/cases/schema.test.ts | 125 ++++++++++++++++++ .../cases/server/connectors/cases/schema.ts | 31 ++++- 2 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/cases/server/connectors/cases/schema.test.ts diff --git a/x-pack/plugins/cases/server/connectors/cases/schema.test.ts b/x-pack/plugins/cases/server/connectors/cases/schema.test.ts new file mode 100644 index 0000000000000..0246a99dddbd8 --- /dev/null +++ b/x-pack/plugins/cases/server/connectors/cases/schema.test.ts @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CasesConnectorRunParamsSchema } from './schema'; + +describe('CasesConnectorRunParamsSchema', () => { + const getParams = (overrides = {}) => ({ + alerts: [{ _id: 'alert-id', _index: 'alert-index' }], + groupingBy: ['host.name'], + rule: { id: 'rule-id', name: 'Test rule', tags: [], ruleUrl: 'https://example.com' }, + owner: 'cases', + ...overrides, + }); + + it('accepts valid params', () => { + expect(() => CasesConnectorRunParamsSchema.validate(getParams())).not.toThrow(); + }); + + describe('alerts', () => { + it('throws if the alerts do not contain _id and _index', () => { + expect(() => + CasesConnectorRunParamsSchema.validate(getParams({ alerts: [{ foo: 'bar' }] })) + ).toThrow(); + }); + }); + + describe('groupingBy', () => { + it('accept an empty groupingBy array', () => { + expect(() => + CasesConnectorRunParamsSchema.validate(getParams({ groupingBy: [] })) + ).not.toThrow(); + }); + + it('does not accept more than one groupingBy key', () => { + expect(() => + CasesConnectorRunParamsSchema.validate( + getParams({ groupingBy: ['host.name', 'source.ip'] }) + ) + ).toThrow(); + }); + }); + + describe('rule', () => { + it('accept empty tags', () => { + const params = getParams(); + + expect(() => + CasesConnectorRunParamsSchema.validate({ ...params, rule: { ...params.rule, tags: [] } }) + ).not.toThrow(); + }); + + it('does not accept more than 10 tags', () => { + const params = getParams(); + + expect(() => + CasesConnectorRunParamsSchema.validate({ + ...params, + rule: { ...params.rule, tags: Array(11).fill('test') }, + }) + ).toThrow(); + }); + + it('does not accept a tag that is more than 50 characters', () => { + const params = getParams(); + + expect(() => + CasesConnectorRunParamsSchema.validate({ + ...params, + rule: { ...params.rule, tags: ['x'.repeat(51)] }, + }) + ).toThrow(); + }); + + it('does not accept an empty tag', () => { + const params = getParams(); + + expect(() => + CasesConnectorRunParamsSchema.validate({ + ...params, + rule: { ...params.rule, tags: '' }, + }) + ).toThrow(); + }); + }); + + describe('timeWindow', () => { + it('throws if the first digit starts with zero', () => { + expect(() => + CasesConnectorRunParamsSchema.validate(getParams({ timeWindow: '0d' })) + ).toThrow(); + }); + + it('throws if the timeWindow does not start with a number', () => { + expect(() => + CasesConnectorRunParamsSchema.validate(getParams({ timeWindow: 'd1' })) + ).toThrow(); + }); + + it('accepts double digit numbers', () => { + expect(() => + CasesConnectorRunParamsSchema.validate(getParams({ timeWindow: '10d' })) + ).not.toThrow(); + }); + + it.each(['s', 'm', 'H', 'h'])('does not allow time unit %s', (unit) => { + expect(() => + CasesConnectorRunParamsSchema.validate(getParams({ timeWindow: `5${unit}` })) + ).toThrow(); + }); + + it.each(['d', 'w', 'M', 'y'])('allows time unit %s', (unit) => { + expect(() => + CasesConnectorRunParamsSchema.validate(getParams({ timeWindow: `5${unit}` })) + ).not.toThrow(); + }); + + it('defaults the timeWindow to 7d', () => { + expect(CasesConnectorRunParamsSchema.validate(getParams()).timeWindow).toBe('7d'); + }); + }); +}); diff --git a/x-pack/plugins/cases/server/connectors/cases/schema.ts b/x-pack/plugins/cases/server/connectors/cases/schema.ts index d8d08a54c87a6..3a3e8fe6209fb 100644 --- a/x-pack/plugins/cases/server/connectors/cases/schema.ts +++ b/x-pack/plugins/cases/server/connectors/cases/schema.ts @@ -6,6 +6,7 @@ */ import { schema } from '@kbn/config-schema'; +import dateMath from '@kbn/datemath'; const AlertSchema = schema.recordOf(schema.string(), schema.any(), { validate: (value) => { @@ -26,7 +27,10 @@ const RuleSchema = schema.object({ /** * TODO: Verify limits */ - tags: schema.arrayOf(schema.string({ minLength: 1, maxLength: 50 }), { minSize: 0, maxSize: 10 }), + tags: schema.arrayOf(schema.string({ minLength: 1, maxLength: 50 }), { + minSize: 0, + maxSize: 10, + }), ruleUrl: schema.nullable(schema.string()), }); @@ -42,4 +46,29 @@ export const CasesConnectorRunParamsSchema = schema.object({ groupingBy: GroupingSchema, owner: schema.string(), rule: RuleSchema, + timeWindow: schema.string({ + defaultValue: '7d', + validate: (value) => { + /** + * Validates the time window. + * Acceptable format: + * - First character should be a digit from 1 to 9 + * - All next characters should be a digit from 0 to 9 + * - The last character should be d (day) or w (week) or M (month) or Y (year) + * + * Example: 20d, 2w, 1M, etc + */ + const timeWindowRegex = new RegExp(/[1-9][0-9]*[d,w,M,y]/, 'g'); + + if (!timeWindowRegex.test(value)) { + return 'Not a valid time window'; + } + + const date = dateMath.parse(`now-${value}`); + + if (!date || !date.isValid()) { + return 'Not a valid time window'; + } + }, + }), }); From 5d340aa62a60d1e7a3ef3417bef7a9386b58cd7b Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 22 Nov 2023 15:35:26 +0000 Subject: [PATCH 35/40] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/plugins/cases/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/cases/tsconfig.json b/x-pack/plugins/cases/tsconfig.json index 1f7d76113b0ec..6215c28ae6a6b 100644 --- a/x-pack/plugins/cases/tsconfig.json +++ b/x-pack/plugins/cases/tsconfig.json @@ -71,6 +71,7 @@ "@kbn/alerting-plugin", "@kbn/content-management-plugin", "@kbn/core-logging-server-mocks", + "@kbn/datemath", ], "exclude": [ "target/**/*", From b20ef532b1534c62715a6d067062e05043f96caa Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Thu, 23 Nov 2023 19:46:59 +0200 Subject: [PATCH 36/40] Upsert oracle records --- .../cases/cases_oracle_service.test.ts | 172 ++++++++++++++++++ .../connectors/cases/cases_oracle_service.ts | 74 ++++++-- .../server/connectors/cases/constants.ts | 2 +- .../cases/server/connectors/cases/types.ts | 10 + 4 files changed, 237 insertions(+), 21 deletions(-) diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts index 84ef73c9c1ea8..4c4efd08648ed 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts @@ -424,4 +424,176 @@ describe('CasesOracleService', () => { ); }); }); + + describe('upsertRecord', () => { + const cases = [{ id: 'test-case-id' }]; + const rules = [{ id: 'test-rule-id' }]; + const grouping = { 'host.ip': '0.0.0.1', 'agent.id': '8a4f500d' }; + + const createPayload = { cases, rules, grouping }; + + const oracleSO = { + id: 'so-id', + version: 'so-version', + attributes: { + counter: 2, + cases, + rules, + grouping, + createdAt: '2023-10-10T10:23:42.769Z', + updatedAt: '2023-10-10T10:23:42.769Z', + }, + type: CASE_ORACLE_SAVED_OBJECT, + references: [], + }; + + const updateDatePayload = { ...oracleSO.attributes, version: oracleSO.version }; + + beforeEach(() => { + unsecuredSavedObjectsClient.update.mockResolvedValue(oracleSO); + }); + + it('creates a record correctly', async () => { + const record = await service.upsertRecord({ + recordId: oracleSO.id, + createPayload, + updateDatePayload, + }); + + expect(record).toEqual({ ...oracleSO.attributes, id: 'so-id', version: 'so-version' }); + }); + + it('calls the unsecuredSavedObjectsClient.create method correctly', async () => { + await service.upsertRecord({ + recordId: oracleSO.id, + createPayload, + updateDatePayload, + }); + + expect(unsecuredSavedObjectsClient.update).toHaveBeenCalledWith( + 'cases-oracle', + 'so-id', + updateDatePayload, + { + upsert: { + cases, + counter: 1, + createdAt: expect.anything(), + rules, + grouping, + updatedAt: null, + }, + version: 'so-version', + } + ); + }); + }); + + describe('bulkUpsertRecord', () => { + const cases = [{ id: 'test-case-id' }]; + const rules = [{ id: 'test-rule-id' }]; + const grouping = { 'host.ip': '0.0.0.1', 'agent.id': '8a4f500d' }; + + const createPayload = { cases, rules, grouping }; + + const oracleSO = { + id: 'so-id', + version: 'so-version', + attributes: { + counter: 2, + cases, + rules, + grouping, + createdAt: '2023-10-10T10:23:42.769Z', + updatedAt: '2023-10-10T10:23:42.769Z', + }, + type: CASE_ORACLE_SAVED_OBJECT, + references: [], + }; + + const updateDatePayload = { ...oracleSO.attributes, version: oracleSO.version }; + + beforeEach(() => { + unsecuredSavedObjectsClient.update.mockResolvedValueOnce(oracleSO); + unsecuredSavedObjectsClient.update.mockResolvedValueOnce({ + ...oracleSO, + id: 'so-id-2', + version: 'so-version-2', + attributes: { ...oracleSO.attributes, counter: 3 }, + }); + }); + + it('creates a record correctly', async () => { + const records = await service.bulkUpsertRecord([ + { + recordId: oracleSO.id, + createPayload, + updateDatePayload, + }, + { + recordId: 'so-id-2', + createPayload, + updateDatePayload: { ...updateDatePayload, version: 'so-version-2', counter: 3 }, + }, + ]); + + expect(records).toEqual([ + { ...oracleSO.attributes, id: 'so-id', version: 'so-version' }, + { ...oracleSO.attributes, id: 'so-id-2', version: 'so-version-2', counter: 3 }, + ]); + }); + + it('calls the unsecuredSavedObjectsClient.create method correctly', async () => { + await service.bulkUpsertRecord([ + { + recordId: oracleSO.id, + createPayload, + updateDatePayload, + }, + { + recordId: 'so-id-2', + createPayload, + updateDatePayload: { ...updateDatePayload, version: 'so-version-2', counter: 3 }, + }, + ]); + + expect(unsecuredSavedObjectsClient.update).toHaveBeenCalledTimes(2); + + expect(unsecuredSavedObjectsClient.update).nthCalledWith( + 1, + 'cases-oracle', + 'so-id', + updateDatePayload, + { + upsert: { + cases, + counter: 1, + createdAt: expect.anything(), + rules, + grouping, + updatedAt: null, + }, + version: 'so-version', + } + ); + + expect(unsecuredSavedObjectsClient.update).nthCalledWith( + 2, + 'cases-oracle', + 'so-id-2', + { ...updateDatePayload, version: 'so-version-2', counter: 3 }, + { + upsert: { + cases, + counter: 1, + createdAt: expect.anything(), + rules, + grouping, + updatedAt: null, + }, + version: 'so-version-2', + } + ); + }); + }); }); diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts index 9f21b6b0aa728..43d7cefc73dd4 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts @@ -6,20 +6,23 @@ */ import type { Logger, SavedObject, SavedObjectsClientContract } from '@kbn/core/server'; +import pMap from 'p-map'; import { CASE_ORACLE_SAVED_OBJECT } from '../../../common/constants'; import type { SavedObjectsBulkResponseWithErrors } from '../../common/types'; import { isSOError } from '../../common/utils'; +import { MAX_CONCURRENT_ES_REQUEST } from './constants'; import { CryptoService } from './crypto_service'; import type { BulkCreateOracleRecordRequest, BulkGetOracleRecordsResponse, + BulkUpsertOracleRecordRequest, OracleKey, OracleRecord, + OracleRecordAttributes, OracleRecordCreateRequest, + OracleRecordUpsertRequest, } from './types'; -type OracleRecordAttributes = Omit; - export class CasesOracleService { private readonly log: Logger; /** @@ -83,26 +86,53 @@ export class CasesOracleService { recordId: string, payload: OracleRecordCreateRequest ): Promise { - const { cases, rules, grouping } = payload; - this.log.debug(`Creating oracle record with ID: ${recordId}`); const oracleRecord = await this.unsecuredSavedObjectsClient.create( CASE_ORACLE_SAVED_OBJECT, - { - counter: 1, - cases, - rules, - grouping, - createdAt: new Date().toISOString(), - updatedAt: null, - }, + this.getCreateRecordAttributes(payload), { id: recordId } ); return this.getRecordResponse(oracleRecord); } + public async upsertRecord({ + recordId, + createPayload, + updateDatePayload, + }: OracleRecordUpsertRequest): Promise { + const createAttributes = this.getCreateRecordAttributes(createPayload); + + const { version, ...updateAttributes } = updateDatePayload; + + this.log.debug(`Updating or creating if not exist oracle record with ID: ${recordId}`); + + const oracleRecord = await this.unsecuredSavedObjectsClient.update( + CASE_ORACLE_SAVED_OBJECT, + recordId, + updateDatePayload, + { upsert: createAttributes, version } + ); + + return this.getRecordResponse({ + ...oracleRecord, + attributes: { ...updateAttributes, ...oracleRecord.attributes }, + references: oracleRecord.references ?? [], + }); + } + + public async bulkUpsertRecord(records: BulkUpsertOracleRecordRequest): Promise { + /** + * The SO client does not supports bulk upsert + */ + const res = await pMap(records, (req: OracleRecordUpsertRequest) => this.upsertRecord(req), { + concurrency: MAX_CONCURRENT_ES_REQUEST, + }); + + return res; + } + public async bulkCreateRecord( records: BulkCreateOracleRecordRequest ): Promise { @@ -113,14 +143,7 @@ export class CasesOracleService { const req = records.map((record) => ({ id: record.recordId, type: CASE_ORACLE_SAVED_OBJECT, - attributes: { - counter: 1, - cases: record.payload.cases, - rules: record.payload.rules, - grouping: record.payload.grouping, - createdAt: new Date().toISOString(), - updatedAt: null, - }, + attributes: this.getCreateRecordAttributes(record.payload), })); const oracleRecords = @@ -177,4 +200,15 @@ export class CasesOracleService { return this.getRecordResponse(oracleRecord); }); } + + private getCreateRecordAttributes({ cases, rules, grouping }: OracleRecordCreateRequest) { + return { + counter: 1, + cases, + rules, + grouping, + createdAt: new Date().toISOString(), + updatedAt: null, + }; + } } diff --git a/x-pack/plugins/cases/server/connectors/cases/constants.ts b/x-pack/plugins/cases/server/connectors/cases/constants.ts index 2987881feb56a..03d84f8701dd8 100644 --- a/x-pack/plugins/cases/server/connectors/cases/constants.ts +++ b/x-pack/plugins/cases/server/connectors/cases/constants.ts @@ -7,7 +7,7 @@ export const CASES_CONNECTOR_ID = '.cases'; export const CASES_CONNECTOR_TITLE = 'Cases'; -export const MAX_CONCURRENT_REQUEST_ATTACH_ALERTS = 5; +export const MAX_CONCURRENT_ES_REQUEST = 5; export enum CASES_CONNECTOR_SUB_ACTION { RUN = 'run', diff --git a/x-pack/plugins/cases/server/connectors/cases/types.ts b/x-pack/plugins/cases/server/connectors/cases/types.ts index ac55e90b0b61f..28f5e64fea295 100644 --- a/x-pack/plugins/cases/server/connectors/cases/types.ts +++ b/x-pack/plugins/cases/server/connectors/cases/types.ts @@ -59,3 +59,13 @@ export type BulkCreateOracleRecordRequest = Array<{ recordId: string; payload: OracleRecordCreateRequest; }>; + +export type OracleRecordAttributes = Omit; + +export interface OracleRecordUpsertRequest { + recordId: string; + updateDatePayload: OracleRecordAttributes & { version: string }; + createPayload: OracleRecordCreateRequest; +} + +export type BulkUpsertOracleRecordRequest = OracleRecordUpsertRequest[]; From b3806ef38c243f303b75ba3ecaf833671e1d370e Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Wed, 29 Nov 2023 14:29:30 +0200 Subject: [PATCH 37/40] Increase the counter when the time window has passed --- .../connectors/cases/cases_connector.test.ts | 186 ++++++++++++++++-- .../connectors/cases/cases_connector.ts | 72 +++++-- .../cases/cases_oracle_service.test.ts | 172 ---------------- .../connectors/cases/cases_oracle_service.ts | 63 +++--- .../cases/server/connectors/cases/types.ts | 14 +- .../server/connectors/cases/utils.test.ts | 6 +- .../cases/server/connectors/cases/utils.ts | 2 +- 7 files changed, 263 insertions(+), 252 deletions(-) diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts index 36cdcaadc99fe..0452c2f43e515 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts @@ -5,6 +5,8 @@ * 2.0. */ +import dateMath from '@kbn/datemath'; +import moment from 'moment'; import { actionsConfigMock } from '@kbn/actions-plugin/server/actions_config.mock'; import { actionsMock } from '@kbn/actions-plugin/server/mocks'; import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; @@ -19,9 +21,11 @@ import type { Cases } from '../../../common'; jest.mock('./cases_oracle_service'); jest.mock('./cases_service'); +jest.mock('@kbn/datemath'); const CasesOracleServiceMock = CasesOracleService as jest.Mock; const CasesServiceMock = CasesService as jest.Mock; +const dateMathMock = dateMath as jest.Mocked; describe('CasesConnector', () => { const services = actionsMock.createServices(); @@ -53,7 +57,9 @@ describe('CasesConnector', () => { tags: ['rule', 'test'], ruleUrl: 'https://example.com/rules/rule-test-id', }; + const owner = 'cases'; + const timeWindow = '7d'; const groupedAlertsWithOracleKey = [ { @@ -91,8 +97,8 @@ describe('CasesConnector', () => { cases: [], rules: [], grouping: groupedAlertsWithOracleKey[1].grouping, - createdAt: '2023-10-10T10:23:42.769Z', - updatedAt: '2023-10-10T10:23:42.769Z', + createdAt: '2023-10-12T10:23:42.769Z', + updatedAt: '2023-10-12T10:23:42.769Z', }, { id: groupedAlertsWithOracleKey[2].oracleKey, @@ -114,6 +120,7 @@ describe('CasesConnector', () => { const mockGetRecordId = jest.fn(); const mockBulkGetRecords = jest.fn(); const mockBulkCreateRecord = jest.fn(); + const mockBulkUpdateRecord = jest.fn(); const mockGetCaseId = jest.fn(); const getCasesClient = jest.fn(); @@ -141,6 +148,7 @@ describe('CasesConnector', () => { version: 'so-version-2', }, ]), + bulkUpdateRecord: mockBulkUpdateRecord.mockResolvedValue([]), }; }); @@ -168,12 +176,14 @@ describe('CasesConnector', () => { services, }, }); + + dateMathMock.parse.mockImplementation(() => moment('2023-10-09T10:23:42.769Z')); }); describe('run', () => { describe('Oracle records', () => { it('generates the oracle keys correctly with grouping by one field', async () => { - await connector.run({ alerts, groupingBy: ['host.name'], owner, rule }); + await connector.run({ alerts, groupingBy: ['host.name'], owner, rule, timeWindow }); expect(mockGetRecordId).toHaveBeenCalledTimes(2); @@ -193,7 +203,7 @@ describe('CasesConnector', () => { }); it('generates the oracle keys correct with grouping by multiple fields', async () => { - await connector.run({ alerts, groupingBy, owner, rule }); + await connector.run({ alerts, groupingBy, owner, rule, timeWindow }); expect(mockGetRecordId).toHaveBeenCalledTimes(3); @@ -208,7 +218,7 @@ describe('CasesConnector', () => { }); it('gets the oracle records correctly', async () => { - await connector.run({ alerts, groupingBy, owner, rule }); + await connector.run({ alerts, groupingBy, owner, rule, timeWindow }); expect(mockBulkGetRecords).toHaveBeenCalledWith([ groupedAlertsWithOracleKey[0].oracleKey, @@ -218,7 +228,7 @@ describe('CasesConnector', () => { }); it('created the non found oracle records correctly', async () => { - await connector.run({ alerts, groupingBy, owner, rule }); + await connector.run({ alerts, groupingBy, owner, rule, timeWindow }); expect(mockBulkCreateRecord).toHaveBeenCalledWith([ { @@ -235,15 +245,152 @@ describe('CasesConnector', () => { it('does not create oracle records if there are no 404 errors', async () => { mockBulkGetRecords.mockResolvedValue([oracleRecords[0]]); - await connector.run({ alerts, groupingBy, owner, rule }); + await connector.run({ alerts, groupingBy, owner, rule, timeWindow }); expect(mockBulkCreateRecord).not.toHaveBeenCalled(); }); + + it('does not create oracle records if there are other errors than 404', async () => { + mockBulkGetRecords.mockResolvedValue([ + { + id: groupedAlertsWithOracleKey[2].oracleKey, + type: CASE_ORACLE_SAVED_OBJECT, + message: 'Conflict', + statusCode: 409, + error: 'Conflict', + }, + ]); + + await connector.run({ alerts, groupingBy, owner, rule, timeWindow }); + + /** + * TODO: Change it to: expect(mockBulkCreateRecord).not.toHaveBeenCalled(); + */ + expect(mockBulkCreateRecord).toHaveBeenCalledWith([]); + }); + + it('does not increase the counter if the time window has not passed', async () => { + mockBulkGetRecords.mockResolvedValue([oracleRecords[0]]); + await connector.run({ alerts, groupingBy, owner, rule, timeWindow }); + + expect(mockBulkUpdateRecord).not.toHaveBeenCalled(); + }); + + it('updates the counter correctly if the time window has passed', async () => { + dateMathMock.parse.mockImplementation(() => moment('2023-11-10T10:23:42.769Z')); + await connector.run({ alerts, groupingBy, owner, rule, timeWindow }); + + expect(mockBulkUpdateRecord).toHaveBeenCalledWith([ + { payload: { counter: 2 }, recordId: 'so-oracle-record-0', version: 'so-version-0' }, + { payload: { counter: 2 }, recordId: 'so-oracle-record-1', version: 'so-version-1' }, + ]); + }); + + it('run correctly with all records: valid, counter increased, counter did not increased, created', async () => { + dateMathMock.parse.mockImplementation(() => moment('2023-10-11T10:23:42.769Z')); + mockBulkCreateRecord.mockResolvedValue([ + { + ...oracleRecords[0], + id: groupedAlertsWithOracleKey[2].oracleKey, + grouping: groupedAlertsWithOracleKey[2].grouping, + version: 'so-version-2', + }, + // Returning errors to verify that the code does not return them + { + id: 'test-id', + type: CASE_ORACLE_SAVED_OBJECT, + message: 'Conflict', + statusCode: 409, + error: 'Conflict', + }, + ]); + + mockBulkUpdateRecord.mockResolvedValue([ + { ...oracleRecords[0], counter: 2 }, + // Returning errors to verify that the code does not return them + { + id: 'test-id', + type: CASE_ORACLE_SAVED_OBJECT, + message: 'Conflict', + statusCode: 409, + error: 'Conflict', + }, + ]); + + await connector.run({ alerts, groupingBy, owner, rule, timeWindow }); + + // 1. Get all records + expect(mockBulkGetRecords).toHaveBeenCalledWith([ + groupedAlertsWithOracleKey[0].oracleKey, + groupedAlertsWithOracleKey[1].oracleKey, + groupedAlertsWithOracleKey[2].oracleKey, + ]); + + // 2. Create the non found records + expect(mockBulkCreateRecord).toHaveBeenCalledWith([ + { + recordId: groupedAlertsWithOracleKey[2].oracleKey, + payload: { + cases: [], + grouping: groupedAlertsWithOracleKey[2].grouping, + rules: [], + }, + }, + ]); + + // 3. Update the counter for the records where the time window has passed + expect(mockBulkUpdateRecord).toHaveBeenCalledWith([ + { payload: { counter: 2 }, recordId: 'so-oracle-record-0', version: 'so-version-0' }, + ]); + + /** + * By checking the getCaseId function we ensure + * that no errors are being returned to the next steps + */ + + expect(mockGetCaseId).toBeCalledTimes(3); + + /** + * Oracle record index: 1 + * Should not update the counter + */ + expect(mockGetCaseId).nthCalledWith(1, { + counter: 1, + grouping: { 'dest.ip': '0.0.0.1', 'host.name': 'B' }, + owner: 'cases', + ruleId: 'rule-test-id', + spaceId: 'default', + }); + + /** + * Oracle record index: 3 + * Not found. Created. + */ + expect(mockGetCaseId).nthCalledWith(2, { + counter: 1, + grouping: { 'dest.ip': '0.0.0.3', 'host.name': 'B' }, + owner: 'cases', + ruleId: 'rule-test-id', + spaceId: 'default', + }); + + /** + * Oracle record index: 0 + * Should update the counter + */ + expect(mockGetCaseId).nthCalledWith(3, { + counter: 2, + grouping: { 'dest.ip': '0.0.0.1', 'host.name': 'A' }, + owner: 'cases', + ruleId: 'rule-test-id', + spaceId: 'default', + }); + }); }); describe('Cases', () => { it('generates the case ids correctly', async () => { - await connector.run({ alerts, groupingBy, owner, rule }); + await connector.run({ alerts, groupingBy, owner, rule, timeWindow }); expect(mockGetCaseId).toHaveBeenCalledTimes(3); @@ -259,7 +406,7 @@ describe('CasesConnector', () => { }); it('gets the cases correctly', async () => { - await connector.run({ alerts, groupingBy, owner, rule }); + await connector.run({ alerts, groupingBy, owner, rule, timeWindow }); expect(casesClientMock.cases.bulkGet).toHaveBeenCalledWith({ ids: ['mock-id-1', 'mock-id-2', 'mock-id-3'], @@ -286,7 +433,7 @@ describe('CasesConnector', () => { ], }); - await connector.run({ alerts, groupingBy, owner, rule }); + await connector.run({ alerts, groupingBy, owner, rule, timeWindow }); expect(casesClientMock.cases.bulkCreate).toHaveBeenCalledWith({ cases: [ @@ -323,7 +470,7 @@ describe('CasesConnector', () => { ], }); - await connector.run({ alerts, groupingBy, owner, rule }); + await connector.run({ alerts, groupingBy, owner, rule, timeWindow }); expect(casesClientMock.cases.bulkCreate).not.toHaveBeenCalled(); }); @@ -331,7 +478,7 @@ describe('CasesConnector', () => { describe('Alerts', () => { it('attach the alerts to the correct cases correctly', async () => { - await connector.run({ alerts, groupingBy, owner, rule }); + await connector.run({ alerts, groupingBy, owner, rule, timeWindow }); expect(casesClientMock.attachments.bulkCreate).toHaveBeenCalledTimes(3); @@ -397,6 +544,11 @@ describe('CasesConnector', () => { }); }); + /** + * In this testing group we test + * only the functionality that differs + * from the testing with grouping + */ describe('Without grouping', () => { beforeEach(() => { jest.clearAllMocks(); @@ -440,7 +592,7 @@ describe('CasesConnector', () => { describe('Oracle records', () => { it('generates the oracle keys correctly with no grouping', async () => { - await connector.run({ alerts, groupingBy: [], owner, rule }); + await connector.run({ alerts, groupingBy: [], owner, rule, timeWindow }); expect(mockGetRecordId).toHaveBeenCalledTimes(1); @@ -453,7 +605,7 @@ describe('CasesConnector', () => { }); it('gets the oracle records correctly', async () => { - await connector.run({ alerts, groupingBy: [], owner, rule }); + await connector.run({ alerts, groupingBy: [], owner, rule, timeWindow }); expect(mockBulkGetRecords).toHaveBeenCalledWith(['so-oracle-record-0']); }); @@ -461,7 +613,7 @@ describe('CasesConnector', () => { describe('Cases', () => { it('generates the case ids correctly', async () => { - await connector.run({ alerts, groupingBy: [], owner, rule }); + await connector.run({ alerts, groupingBy: [], owner, rule, timeWindow }); expect(mockGetCaseId).toHaveBeenCalledTimes(1); @@ -475,7 +627,7 @@ describe('CasesConnector', () => { }); it('gets the cases correctly', async () => { - await connector.run({ alerts, groupingBy: [], owner, rule }); + await connector.run({ alerts, groupingBy: [], owner, rule, timeWindow }); expect(casesClientMock.cases.bulkGet).toHaveBeenCalledWith({ ids: ['mock-id-1'], @@ -485,7 +637,7 @@ describe('CasesConnector', () => { describe('Alerts', () => { it('attach all alerts to the same case when the grouping is not defined', async () => { - await connector.run({ alerts, groupingBy: [], owner, rule }); + await connector.run({ alerts, groupingBy: [], owner, rule, timeWindow }); expect(casesClientMock.attachments.bulkCreate).toHaveBeenCalledTimes(1); expect(casesClientMock.attachments.bulkCreate).nthCalledWith(1, { diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts index bd5b84fff356a..bd223b115f6b9 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts @@ -6,16 +6,17 @@ */ import stringify from 'json-stable-stringify'; +import pMap from 'p-map'; import type { ServiceParams } from '@kbn/actions-plugin/server'; import { SubActionConnector } from '@kbn/actions-plugin/server'; -import { pick } from 'lodash'; +import { partition, pick } from 'lodash'; import type { KibanaRequest } from '@kbn/core-http-server'; import { CoreKibanaRequest } from '@kbn/core/server'; -import pMap from 'p-map'; +import dateMath from '@kbn/datemath'; import type { BulkCreateCasesRequest } from '../../../common/types/api'; import type { Case } from '../../../common'; import { ConnectorTypes, AttachmentType } from '../../../common'; -import { CASES_CONNECTOR_SUB_ACTION, MAX_CONCURRENT_REQUEST_ATTACH_ALERTS } from './constants'; +import { CASES_CONNECTOR_SUB_ACTION, MAX_CONCURRENT_ES_REQUEST } from './constants'; import type { BulkCreateOracleRecordRequest, CasesConnectorConfig, @@ -26,7 +27,7 @@ import type { } from './types'; import { CasesConnectorRunParamsSchema } from './schema'; import { CasesOracleService } from './cases_oracle_service'; -import { partitionRecords } from './utils'; +import { partitionRecordsByError } from './utils'; import { CasesService } from './cases_service'; import type { CasesClient } from '../../client'; import type { BulkCreateArgs as BulkCreateAlertsReq } from '../../client/attachments/types'; @@ -107,7 +108,8 @@ export class CasesConnector extends SubActionConnector< /** * Add circuit breakers to the number of oracles they can be created or retrieved */ - const oracleRecords = await this.bulkGetOrCreateOracleRecords( + const oracleRecords = await this.upsertOracleRecords( + params, Array.from(groupedAlertsWithOracleKey.values()) ); @@ -182,17 +184,24 @@ export class CasesConnector extends SubActionConnector< return oracleMap; } - private async bulkGetOrCreateOracleRecords( + private async upsertOracleRecords( + params: CasesConnectorRunParams, groupedAlertsWithOracleKey: GroupedAlertsWithOracleKey[] ): Promise { + const { timeWindow } = params; const bulkCreateReq: BulkCreateOracleRecordRequest = []; const ids = groupedAlertsWithOracleKey.map(({ oracleKey }) => oracleKey); const bulkGetRes = await this.casesOracleService.bulkGetRecords(ids); - const [bulkGetValidRecords, bulkGetRecordsErrors] = partitionRecords(bulkGetRes); + const [bulkGetValidRecords, bulkGetRecordsErrors] = partitionRecordsByError(bulkGetRes); + + const [recordsToIncreaseCounter, recordsWithoutIncreasedCounter] = partition( + bulkGetValidRecords, + (req) => this.isTimeWindowPassed(timeWindow, req.updatedAt ?? req.createdAt) + ); - if (bulkGetRecordsErrors.length === 0) { + if (bulkGetRecordsErrors.length === 0 && recordsToIncreaseCounter.length === 0) { return bulkGetValidRecords; } @@ -218,14 +227,55 @@ export class CasesConnector extends SubActionConnector< } } + const bulkUpdateReq = recordsToIncreaseCounter.map((record) => ({ + recordId: record.id, + version: record.version, + /** + * TODO: Add new cases or any other related info + */ + payload: { counter: record.counter + 1 }, + })); + const bulkCreateRes = await this.casesOracleService.bulkCreateRecord(bulkCreateReq); + const bulkUpdateRes = await this.casesOracleService.bulkUpdateRecord(bulkUpdateReq); /** * TODO: Throw/Retry on errors */ - const [bulkCreateValidRecords, _] = partitionRecords(bulkCreateRes); + const [bulkCreateValidRecords, _bulkCreateErrors] = partitionRecordsByError(bulkCreateRes); + const [bulkUpdateValidRecords, _bulkUpdateErrors] = partitionRecordsByError(bulkUpdateRes); + + /** + * TODO: Should we check if the records in the + * arrays are unique? + */ + return [ + ...recordsWithoutIncreasedCounter, + ...bulkCreateValidRecords, + ...bulkUpdateValidRecords, + ]; + } + + private isTimeWindowPassed(timeWindow: string, counterLastUpdatedAt: string) { + const parsedDate = dateMath.parse(`now-${timeWindow}`); + + /** + * TODO: Should we throw? + */ + if (!parsedDate || !parsedDate.isValid()) { + return false; + } + + const counterLastUpdatedAtAsDate = new Date(counterLastUpdatedAt); + + /** + * TODO: Should we throw? + */ + if (isNaN(counterLastUpdatedAtAsDate.getTime())) { + return false; + } - return [...bulkGetValidRecords, ...bulkCreateValidRecords]; + return counterLastUpdatedAtAsDate < parsedDate.toDate(); } private generateCaseIds( @@ -392,7 +442,7 @@ export class CasesConnector extends SubActionConnector< bulkCreateAlertsRequest, (req: BulkCreateAlertsReq) => casesClient.attachments.bulkCreate(req), { - concurrency: MAX_CONCURRENT_REQUEST_ATTACH_ALERTS, + concurrency: MAX_CONCURRENT_ES_REQUEST, } ); } diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts index 4c4efd08648ed..84ef73c9c1ea8 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.test.ts @@ -424,176 +424,4 @@ describe('CasesOracleService', () => { ); }); }); - - describe('upsertRecord', () => { - const cases = [{ id: 'test-case-id' }]; - const rules = [{ id: 'test-rule-id' }]; - const grouping = { 'host.ip': '0.0.0.1', 'agent.id': '8a4f500d' }; - - const createPayload = { cases, rules, grouping }; - - const oracleSO = { - id: 'so-id', - version: 'so-version', - attributes: { - counter: 2, - cases, - rules, - grouping, - createdAt: '2023-10-10T10:23:42.769Z', - updatedAt: '2023-10-10T10:23:42.769Z', - }, - type: CASE_ORACLE_SAVED_OBJECT, - references: [], - }; - - const updateDatePayload = { ...oracleSO.attributes, version: oracleSO.version }; - - beforeEach(() => { - unsecuredSavedObjectsClient.update.mockResolvedValue(oracleSO); - }); - - it('creates a record correctly', async () => { - const record = await service.upsertRecord({ - recordId: oracleSO.id, - createPayload, - updateDatePayload, - }); - - expect(record).toEqual({ ...oracleSO.attributes, id: 'so-id', version: 'so-version' }); - }); - - it('calls the unsecuredSavedObjectsClient.create method correctly', async () => { - await service.upsertRecord({ - recordId: oracleSO.id, - createPayload, - updateDatePayload, - }); - - expect(unsecuredSavedObjectsClient.update).toHaveBeenCalledWith( - 'cases-oracle', - 'so-id', - updateDatePayload, - { - upsert: { - cases, - counter: 1, - createdAt: expect.anything(), - rules, - grouping, - updatedAt: null, - }, - version: 'so-version', - } - ); - }); - }); - - describe('bulkUpsertRecord', () => { - const cases = [{ id: 'test-case-id' }]; - const rules = [{ id: 'test-rule-id' }]; - const grouping = { 'host.ip': '0.0.0.1', 'agent.id': '8a4f500d' }; - - const createPayload = { cases, rules, grouping }; - - const oracleSO = { - id: 'so-id', - version: 'so-version', - attributes: { - counter: 2, - cases, - rules, - grouping, - createdAt: '2023-10-10T10:23:42.769Z', - updatedAt: '2023-10-10T10:23:42.769Z', - }, - type: CASE_ORACLE_SAVED_OBJECT, - references: [], - }; - - const updateDatePayload = { ...oracleSO.attributes, version: oracleSO.version }; - - beforeEach(() => { - unsecuredSavedObjectsClient.update.mockResolvedValueOnce(oracleSO); - unsecuredSavedObjectsClient.update.mockResolvedValueOnce({ - ...oracleSO, - id: 'so-id-2', - version: 'so-version-2', - attributes: { ...oracleSO.attributes, counter: 3 }, - }); - }); - - it('creates a record correctly', async () => { - const records = await service.bulkUpsertRecord([ - { - recordId: oracleSO.id, - createPayload, - updateDatePayload, - }, - { - recordId: 'so-id-2', - createPayload, - updateDatePayload: { ...updateDatePayload, version: 'so-version-2', counter: 3 }, - }, - ]); - - expect(records).toEqual([ - { ...oracleSO.attributes, id: 'so-id', version: 'so-version' }, - { ...oracleSO.attributes, id: 'so-id-2', version: 'so-version-2', counter: 3 }, - ]); - }); - - it('calls the unsecuredSavedObjectsClient.create method correctly', async () => { - await service.bulkUpsertRecord([ - { - recordId: oracleSO.id, - createPayload, - updateDatePayload, - }, - { - recordId: 'so-id-2', - createPayload, - updateDatePayload: { ...updateDatePayload, version: 'so-version-2', counter: 3 }, - }, - ]); - - expect(unsecuredSavedObjectsClient.update).toHaveBeenCalledTimes(2); - - expect(unsecuredSavedObjectsClient.update).nthCalledWith( - 1, - 'cases-oracle', - 'so-id', - updateDatePayload, - { - upsert: { - cases, - counter: 1, - createdAt: expect.anything(), - rules, - grouping, - updatedAt: null, - }, - version: 'so-version', - } - ); - - expect(unsecuredSavedObjectsClient.update).nthCalledWith( - 2, - 'cases-oracle', - 'so-id-2', - { ...updateDatePayload, version: 'so-version-2', counter: 3 }, - { - upsert: { - cases, - counter: 1, - createdAt: expect.anything(), - rules, - grouping, - updatedAt: null, - }, - version: 'so-version-2', - } - ); - }); - }); }); diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts index 43d7cefc73dd4..822be943e1e77 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_oracle_service.ts @@ -6,21 +6,18 @@ */ import type { Logger, SavedObject, SavedObjectsClientContract } from '@kbn/core/server'; -import pMap from 'p-map'; import { CASE_ORACLE_SAVED_OBJECT } from '../../../common/constants'; import type { SavedObjectsBulkResponseWithErrors } from '../../common/types'; import { isSOError } from '../../common/utils'; -import { MAX_CONCURRENT_ES_REQUEST } from './constants'; import { CryptoService } from './crypto_service'; import type { BulkCreateOracleRecordRequest, BulkGetOracleRecordsResponse, - BulkUpsertOracleRecordRequest, + BulkUpdateOracleRecordRequest, OracleKey, OracleRecord, OracleRecordAttributes, OracleRecordCreateRequest, - OracleRecordUpsertRequest, } from './types'; export class CasesOracleService { @@ -97,42 +94,6 @@ export class CasesOracleService { return this.getRecordResponse(oracleRecord); } - public async upsertRecord({ - recordId, - createPayload, - updateDatePayload, - }: OracleRecordUpsertRequest): Promise { - const createAttributes = this.getCreateRecordAttributes(createPayload); - - const { version, ...updateAttributes } = updateDatePayload; - - this.log.debug(`Updating or creating if not exist oracle record with ID: ${recordId}`); - - const oracleRecord = await this.unsecuredSavedObjectsClient.update( - CASE_ORACLE_SAVED_OBJECT, - recordId, - updateDatePayload, - { upsert: createAttributes, version } - ); - - return this.getRecordResponse({ - ...oracleRecord, - attributes: { ...updateAttributes, ...oracleRecord.attributes }, - references: oracleRecord.references ?? [], - }); - } - - public async bulkUpsertRecord(records: BulkUpsertOracleRecordRequest): Promise { - /** - * The SO client does not supports bulk upsert - */ - const res = await pMap(records, (req: OracleRecordUpsertRequest) => this.upsertRecord(req), { - concurrency: MAX_CONCURRENT_ES_REQUEST, - }); - - return res; - } - public async bulkCreateRecord( records: BulkCreateOracleRecordRequest ): Promise { @@ -176,6 +137,28 @@ export class CasesOracleService { }); } + public async bulkUpdateRecord( + records: BulkUpdateOracleRecordRequest + ): Promise { + const recordIds = records.map((record) => record.recordId); + + this.log.debug(`Updating oracle record with ID: ${recordIds}`); + + const req = records.map((record) => ({ + id: record.recordId, + type: CASE_ORACLE_SAVED_OBJECT, + version: record.version, + attributes: { ...record.payload, updatedAt: new Date().toISOString() }, + })); + + const oracleRecords = + (await this.unsecuredSavedObjectsClient.bulkUpdate( + req + )) as SavedObjectsBulkResponseWithErrors; + + return this.getBulkRecordsResponse(oracleRecords); + } + private getRecordResponse = ( oracleRecord: SavedObject ): OracleRecord => ({ diff --git a/x-pack/plugins/cases/server/connectors/cases/types.ts b/x-pack/plugins/cases/server/connectors/cases/types.ts index 28f5e64fea295..4c8820968260b 100644 --- a/x-pack/plugins/cases/server/connectors/cases/types.ts +++ b/x-pack/plugins/cases/server/connectors/cases/types.ts @@ -55,17 +55,15 @@ export interface OracleRecordCreateRequest { export type BulkGetOracleRecordsResponse = Array; +export type OracleRecordAttributes = Omit; + export type BulkCreateOracleRecordRequest = Array<{ recordId: string; payload: OracleRecordCreateRequest; }>; -export type OracleRecordAttributes = Omit; - -export interface OracleRecordUpsertRequest { +export type BulkUpdateOracleRecordRequest = Array<{ recordId: string; - updateDatePayload: OracleRecordAttributes & { version: string }; - createPayload: OracleRecordCreateRequest; -} - -export type BulkUpsertOracleRecordRequest = OracleRecordUpsertRequest[]; + version: string; + payload: Pick; +}>; diff --git a/x-pack/plugins/cases/server/connectors/cases/utils.test.ts b/x-pack/plugins/cases/server/connectors/cases/utils.test.ts index 444dd8da6b27e..9652414bff9c7 100644 --- a/x-pack/plugins/cases/server/connectors/cases/utils.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/utils.test.ts @@ -6,7 +6,7 @@ */ import { oracleRecordError, oracleRecord } from './index.mock'; -import { isRecordError, partitionRecords } from './utils'; +import { isRecordError, partitionRecordsByError } from './utils'; describe('utils', () => { describe('isRecordError', () => { @@ -24,10 +24,10 @@ describe('utils', () => { }); }); - describe('partitionRecords', () => { + describe('partitionRecordsByError', () => { it('partition records correctly', () => { expect( - partitionRecords([oracleRecordError, oracleRecord, oracleRecordError, oracleRecord]) + partitionRecordsByError([oracleRecordError, oracleRecord, oracleRecordError, oracleRecord]) ).toEqual([ [oracleRecord, oracleRecord], [oracleRecordError, oracleRecordError], diff --git a/x-pack/plugins/cases/server/connectors/cases/utils.ts b/x-pack/plugins/cases/server/connectors/cases/utils.ts index 38f6a23c490fb..51137c1a616ec 100644 --- a/x-pack/plugins/cases/server/connectors/cases/utils.ts +++ b/x-pack/plugins/cases/server/connectors/cases/utils.ts @@ -11,7 +11,7 @@ import type { BulkGetOracleRecordsResponse, OracleRecord, OracleRecordError } fr export const isRecordError = (so: OracleRecord | OracleRecordError): so is OracleRecordError => (so as OracleRecordError).error != null; -export const partitionRecords = ( +export const partitionRecordsByError = ( res: BulkGetOracleRecordsResponse ): [OracleRecord[], OracleRecordError[]] => { const [errors, validRecords] = partition(res, isRecordError) as [ From 7e5989e270e6a918a5335f2ea4e8944a44134e1d Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Wed, 29 Nov 2023 14:53:25 +0200 Subject: [PATCH 38/40] Add more tests --- .../connectors/cases/cases_connector.test.ts | 47 ++++++++++--------- .../connectors/cases/cases_connector.ts | 4 +- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts index 0452c2f43e515..9bc09210c6362 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts @@ -342,11 +342,32 @@ describe('CasesConnector', () => { expect(mockBulkUpdateRecord).toHaveBeenCalledWith([ { payload: { counter: 2 }, recordId: 'so-oracle-record-0', version: 'so-version-0' }, ]); + }); + }); - /** - * By checking the getCaseId function we ensure - * that no errors are being returned to the next steps - */ + describe('Cases', () => { + it('generates the case ids correctly', async () => { + await connector.run({ alerts, groupingBy, owner, rule, timeWindow }); + + expect(mockGetCaseId).toHaveBeenCalledTimes(3); + + for (const [index, { grouping }] of groupedAlertsWithOracleKey.entries()) { + expect(mockGetCaseId).nthCalledWith(index + 1, { + ruleId: rule.id, + grouping, + owner, + spaceId: 'default', + counter: 1, + }); + } + }); + + it('generates the case ids correctly when the time window has passed', async () => { + dateMathMock.parse.mockImplementation(() => moment('2023-10-11T10:23:42.769Z')); + + mockBulkUpdateRecord.mockResolvedValue([{ ...oracleRecords[0], counter: 2 }]); + + await connector.run({ alerts, groupingBy, owner, rule, timeWindow }); expect(mockGetCaseId).toBeCalledTimes(3); @@ -386,24 +407,6 @@ describe('CasesConnector', () => { spaceId: 'default', }); }); - }); - - describe('Cases', () => { - it('generates the case ids correctly', async () => { - await connector.run({ alerts, groupingBy, owner, rule, timeWindow }); - - expect(mockGetCaseId).toHaveBeenCalledTimes(3); - - for (const [index, { grouping }] of groupedAlertsWithOracleKey.entries()) { - expect(mockGetCaseId).nthCalledWith(index + 1, { - ruleId: rule.id, - grouping, - owner, - spaceId: 'default', - counter: 1, - }); - } - }); it('gets the cases correctly', async () => { await connector.run({ alerts, groupingBy, owner, rule, timeWindow }); diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts index bd223b115f6b9..54ee2f8b108c1 100644 --- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts +++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts @@ -119,7 +119,7 @@ export class CasesConnector extends SubActionConnector< oracleRecords ); - const groupedAlertsWithCases = await this.bulkGetOrCreateCases( + const groupedAlertsWithCases = await this.upsertCases( params, casesClient, groupedAlertsWithCaseId @@ -312,7 +312,7 @@ export class CasesConnector extends SubActionConnector< return casesMap; } - private async bulkGetOrCreateCases( + private async upsertCases( params: CasesConnectorRunParams, casesClient: CasesClient, groupedAlertsWithCaseId: Map From 81ab22031e97c39f94d79f06db7c0878dc7697a8 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Fri, 8 Dec 2023 12:15:40 +0200 Subject: [PATCH 39/40] Fix bug with Regex --- .../server/connectors/cases/schema.test.ts | 30 +++++++++++++++++++ .../cases/server/connectors/cases/schema.ts | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/cases/server/connectors/cases/schema.test.ts b/x-pack/plugins/cases/server/connectors/cases/schema.test.ts index 0246a99dddbd8..b28f7d9feb3bc 100644 --- a/x-pack/plugins/cases/server/connectors/cases/schema.test.ts +++ b/x-pack/plugins/cases/server/connectors/cases/schema.test.ts @@ -100,6 +100,36 @@ describe('CasesConnectorRunParamsSchema', () => { ).toThrow(); }); + it('should fail for valid date math but not valid time window', () => { + expect(() => + CasesConnectorRunParamsSchema.validate(getParams({ timeWindow: '10d+3d' })) + ).toThrow(); + }); + + it('throws if there is a non valid letter at the end', () => { + expect(() => + CasesConnectorRunParamsSchema.validate(getParams({ timeWindow: '10d#' })) + ).toThrow(); + }); + + it('throws if there is a valid letter at the end', () => { + expect(() => + CasesConnectorRunParamsSchema.validate(getParams({ timeWindow: '10dd' })) + ).toThrow(); + }); + + it('throws if there is a digit at the end', () => { + expect(() => + CasesConnectorRunParamsSchema.validate(getParams({ timeWindow: '10d2' })) + ).toThrow(); + }); + + it('throws if there are two valid formats in sequence', () => { + expect(() => + CasesConnectorRunParamsSchema.validate(getParams({ timeWindow: '1d2d' })) + ).toThrow(); + }); + it('accepts double digit numbers', () => { expect(() => CasesConnectorRunParamsSchema.validate(getParams({ timeWindow: '10d' })) diff --git a/x-pack/plugins/cases/server/connectors/cases/schema.ts b/x-pack/plugins/cases/server/connectors/cases/schema.ts index 3a3e8fe6209fb..6cf9fb43cb700 100644 --- a/x-pack/plugins/cases/server/connectors/cases/schema.ts +++ b/x-pack/plugins/cases/server/connectors/cases/schema.ts @@ -58,7 +58,7 @@ export const CasesConnectorRunParamsSchema = schema.object({ * * Example: 20d, 2w, 1M, etc */ - const timeWindowRegex = new RegExp(/[1-9][0-9]*[d,w,M,y]/, 'g'); + const timeWindowRegex = new RegExp(/^[1-9][0-9]*[d,w,M,y]$/, 'g'); if (!timeWindowRegex.test(value)) { return 'Not a valid time window'; From d4e1e8cc3522ed610f9e59604950176d1ce37397 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 8 Dec 2023 10:35:55 +0000 Subject: [PATCH 40/40] [CI] Auto-commit changed files from 'node scripts/check_mappings_update --fix' --- .../kbn-check-mappings-update-cli/current_fields.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index ad25525f19303..7e658e4ec414d 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -966,5 +966,14 @@ "kuery", "serviceEnvironmentFilterEnabled", "serviceNameFilterEnabled" + ], + "cases-oracle": [ + "cases", + "cases.id", + "counter", + "createdAt", + "rules", + "rules.id", + "updatedAt" ] }