From 08003990cfa951b9fae8ae860eb9371b676351d1 Mon Sep 17 00:00:00 2001 From: Georgii Gorbachev Date: Fri, 16 Dec 2022 18:33:53 +0100 Subject: [PATCH] PoC of the rule upgrade and installation workflows --- package.json | 1 + .../prebuilt_rules/api/urls.ts | 20 +- .../prebuilt_rules/poc/README.md | 1099 +++++++++++++++++ .../request_schema.ts | 24 + .../response_schema.ts | 35 + .../request_schema.ts | 16 + .../response_schema.ts | 38 + .../api/review_rule_upgrade/request_schema.ts | 16 + .../review_rule_upgrade/response_schema.ts | 44 + .../content_model/prebuilt_rule_content.ts | 48 + .../prebuilt_rule_version_info.ts | 16 + .../poc/content_model/semantic_version.ts | 29 + .../poc/diff_algorithm/calculate_rule_diff.ts | 108 ++ .../algorithms/simple_diff_algorithm.ts | 84 ++ .../calculation/calculate_rule_fields_diff.ts | 265 ++++ .../calculation/calculate_rule_json_diff.ts | 89 ++ .../calculation/diff_calculation_helpers.ts | 35 + .../normalization/convert_rule_to_diffable.ts | 264 ++++ .../extract_building_block_object.ts | 21 + .../normalization/extract_rule_data_query.ts | 60 + .../normalization/extract_rule_data_source.ts | 31 + .../extract_rule_name_override_object.ts | 21 + .../normalization/extract_rule_schedule.ts | 86 ++ .../extract_timeline_template_reference.ts | 22 + .../extract_timestamp_override_object.ts | 22 + .../poc/diff_model/fields_diff.ts | 16 + .../poc/diff_model/rule_diff.ts | 60 + .../poc/diff_model/three_way_diff.ts | 114 ++ .../poc/diff_model/three_way_diff_outcome.ts | 60 + .../poc/diff_model/three_way_merge_outcome.ts | 26 + .../poc/diffable_rule_model/build_schema.ts | 23 + .../diffable_field_types.ts | 141 +++ .../poc/diffable_rule_model/diffable_rule.ts | 244 ++++ .../poc/diffable_rule_model/rule_fields.ts | 393 ++++++ .../rule_schema/model/build_rule_schemas.ts | 15 +- .../common_attributes/misc_attributes.ts | 2 +- .../rule_schema/model/rule_schemas.ts | 28 +- .../api/get_prebuilt_rules_status/route.ts | 105 ++ .../api/install_test_assets/route.ts | 206 +++ .../prebuilt_rules/api/register_routes.ts | 15 + .../api/review_rule_installation/route.ts | 125 ++ .../api/review_rule_upgrade/route.ts | 179 +++ .../get_versions_to_install_and_upgrade.ts | 58 + .../logic/poc/prebuilt_rule_content_client.ts | 68 + .../logic/poc/prebuilt_rule_objects_client.ts | 69 ++ ...e_asset_composite2_saved_objects_client.ts | 185 +++ ...ule_asset_composite2_saved_objects_type.ts | 62 + ...le_asset_composite_saved_objects_client.ts | 176 +++ ...rule_asset_composite_saved_objects_type.ts | 62 + .../rule_asset_flat_saved_objects_client.ts | 257 ++++ .../rule_asset_flat_saved_objects_type.ts | 53 + .../security_solution/server/routes/index.ts | 2 +- .../security_solution/server/saved_objects.ts | 6 + yarn.lock | 5 + 54 files changed, 5231 insertions(+), 18 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/README.md create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/get_prebuilt_rules_status/request_schema.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/get_prebuilt_rules_status/response_schema.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/review_rule_installation/request_schema.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/review_rule_installation/response_schema.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/review_rule_upgrade/request_schema.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/review_rule_upgrade/response_schema.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_content.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_version_info.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/content_model/semantic_version.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculate_rule_diff.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculation/algorithms/simple_diff_algorithm.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculation/calculate_rule_fields_diff.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculation/calculate_rule_json_diff.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculation/diff_calculation_helpers.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/convert_rule_to_diffable.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_building_block_object.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_rule_data_query.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_rule_data_source.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_rule_name_override_object.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_rule_schedule.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_timeline_template_reference.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_timestamp_override_object.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_model/fields_diff.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_model/rule_diff.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_model/three_way_diff.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_model/three_way_diff_outcome.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_model/three_way_merge_outcome.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diffable_rule_model/build_schema.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diffable_rule_model/diffable_field_types.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diffable_rule_model/diffable_rule.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diffable_rule_model/rule_fields.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/get_prebuilt_rules_status/route.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/install_test_assets/route.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_installation/route.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/route.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/get_versions_to_install_and_upgrade.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/prebuilt_rule_content_client.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/prebuilt_rule_objects_client.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_composite2_saved_objects_client.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_composite2_saved_objects_type.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_composite_saved_objects_client.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_composite_saved_objects_type.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_flat_saved_objects_client.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_flat_saved_objects_type.ts diff --git a/package.json b/package.json index c4706afabf3e5..869f68704a71e 100644 --- a/package.json +++ b/package.json @@ -582,6 +582,7 @@ "monaco-editor": "^0.24.0", "monaco-yaml": "3.2.1", "mustache": "^2.3.2", + "node-diff3": "^3.1.2", "node-fetch": "^2.6.7", "node-forge": "^1.3.1", "nodemailer": "^6.6.2", diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/urls.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/urls.ts index 449960916f239..b8e4ed241f669 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/urls.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/urls.ts @@ -5,7 +5,21 @@ * 2.0. */ -import { DETECTION_ENGINE_RULES_URL as RULES } from '../../../constants'; +import { + DETECTION_ENGINE_RULES_URL as RULES, + INTERNAL_DETECTION_ENGINE_URL as INTERNAL, +} from '../../../constants'; -export const PREBUILT_RULES_URL = `${RULES}/prepackaged` as const; -export const PREBUILT_RULES_STATUS_URL = `${RULES}/prepackaged/_status` as const; +const OLD_BASE_URL = `${RULES}/prepackaged` as const; +const NEW_BASE_URL = `${INTERNAL}/prebuilt_rules` as const; + +export const PREBUILT_RULES_URL = OLD_BASE_URL; +export const PREBUILT_RULES_STATUS_URL = `${OLD_BASE_URL}/_status` as const; + +export const GET_PREBUILT_RULES_STATUS_URL = `${NEW_BASE_URL}/status` as const; +export const REVIEW_RULE_UPGRADE_URL = `${NEW_BASE_URL}/upgrade/_review` as const; +export const PERFORM_RULE_UPGRADE_URL = `${NEW_BASE_URL}/upgrade/_perform` as const; +export const REVIEW_RULE_INSTALLATION_URL = `${NEW_BASE_URL}/installation/_review` as const; +export const PERFORM_RULE_INSTALLATION_URL = `${NEW_BASE_URL}/installation/_perform` as const; + +export const INSTALL_TEST_ASSETS_URL = `${NEW_BASE_URL}/_install_test_assets` as const; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/README.md b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/README.md new file mode 100644 index 0000000000000..270095561dd5a --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/README.md @@ -0,0 +1,1099 @@ +# PoC of the rule upgrade and installation workflows + +**Resolves:** https://github.com/elastic/kibana/issues/137446 + +NOTE: You can comment this PR description line-by-line in `security_solution/common/detection_engine/prebuilt_rules/poc/README.md`. + +## Summary + +- [x] Revisit Figma designs and discuss separation between installation and upgrade workflows. +- [x] Design API endpoints for the rule upgrade and installation workflows. +- [x] Design a domain model that will be used on the frontend to build the rule upgrade UI. +- [x] Design a data model of prebuilt rule assets (2 versions: flat and composite). + - NOTE: Ended up with 3 versions: one flat and two composite data models. +- [x] Implement helper endpoints for indexing prebuilt rule assets. +- [x] Determine which rule fields will be "customizable" and which will be "technical". + - NOTE: Mostly done but a few specific fields need to be discussed with the team. + - Follow-up: https://github.com/elastic/kibana/issues/147239 +- [x] Find out which fields are actually used in https://github.com/elastic/detection-rules and if we could remove some of them from the `PrebuiltRuleToInstall` schema +- [x] Design an algorithm that returns a 3-way diff between 3 rule versions: base, current, target. +- [x] Implement PoC API endpoints and make them accept a data model in request parameters. + - NOTE: Implemented the `status` and two `_review` endpoints. I think we could skip implementing the `_perform` endpoints within this POC because they seem to be less risky (given the `_review` endpoints). +- [x] Clean up. Address `TODO: https://github.com/elastic/kibana/pull/144060` comments. +- [x] Test performance of the implemented API endpoints. + - NOTE: The flat data model appears to be the most performant. It's up to 2x more performant than the composite models. +- [x] Choose the data model. + - NOTE: We decided to proceed with the flat data model. We will have to fix several issues on the Fleet side in order to do that: + - https://github.com/elastic/kibana/issues/147695 + - https://github.com/elastic/kibana/issues/148174 + - https://github.com/elastic/kibana/issues/148175 + +Open questions to clarify in parallel or later: + +- [Discuss with Design] Design a field diff UI for each rule field or set of related fields. +- [Discuss with PM] What should we do when a user installs the Elastic Defend integration? + - Current behavior: we install and upgrade all prebuilt rules automatically. + - Proposed change: adjust `server/fleet_integration/handlers/install_prepackaged_rules.ts` to install only the promotion "Endpoint Security" rule if it's not installed. Don't install all the rules and don't upgrade rules. +- [Discuss with PM] How/when do we want to install prebuilt timelines? Should we provide a separate API and UI for that? + - https://github.com/elastic/kibana/issues/92553 +- [Discuss with PM] What should be our rule deprecation workflow? Let's discuss the UX and requirements. + - https://github.com/elastic/kibana/issues/118942 + - FYI some of the prebuilt rules were deleted in the prebuilt rules repo (https://github.com/elastic/ia-trade-team/issues/84#issuecomment-1332531334) +- Think about advanced features for the future: + - rollback to the current stock version (revert customizations) + - upgrade to a given next version + - downgrade to a given previous version + +## Workflows + +Stage 1 (both workflows): + +1. Call `GET /internal/detection_engine/prebuilt_rules/status`. +1. Show a callout with 1 or 2 CTA buttons (upgrade rules, install new rules). + +Stage 2 (upgrade workflow): + +1. User clicks "Upgrade X rules" button. +1. Show "Review updates" flyout. +1. Enable the loading indicator. +1. Call `POST /internal/detection_engine/prebuilt_rules/upgrade/_review`. +1. Disable the loading indicator, show the upgrade UI. +1. User selects/deselects rules and fields, resolves conflicts if any. +1. User clicks "Update selected rules" button. +1. Enable the loading indicator. +1. Call `POST /internal/detection_engine/prebuilt_rules/upgrade/_perform`. +1. Disable the loading indicator, close the flyout or show errors. +1. Refresh the Rules table and the status of prebuilt rules. + +Stage 2 (installation workflow): + +1. User clicks "View Y new rules" button. +1. Show "View new rules" flyout. +1. Enable the loading indicator. +1. Call `POST /internal/detection_engine/prebuilt_rules/installation/_review`. +1. Disable the loading indicator, show the installation UI. +1. User selects/deselects rules to be installed. +1. User clicks "Install selected rules" button. +1. Enable the loading indicator. +1. Call `POST /internal/detection_engine/prebuilt_rules/installation/_perform`. +1. Disable the loading indicator, close the flyout or show errors. +1. Refresh the Rules table and the status of prebuilt rules. + +## Data models + +This POC implements and compares 3 new data models for historical versioned rule asset saved objects. +See the implementation in `server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects`. + +We will need to choose the one we will proceed with. Criteria considered so far: + +- Ability to implement queries needed for implementing the 5 endpoints proposed in this POC. +- Flexibility in querying data in general. +- Performance of querying data. + +### Flat model + +Every object is a historical rule version that contains the rule id, the content version and the content itself. + +`server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_flat_saved_objects_type.ts`: + +```ts +const SO_TYPE = 'security-rule-flat'; + +const mappings = { + dynamic: false, + properties: { + name: { + type: 'keyword', + }, + rule_id: { + type: 'keyword', + }, + rule_content_version: { + type: 'version', + }, + stack_version_min: { + type: 'version', + }, + stack_version_max: { + type: 'version', + }, + }, +}; +``` + +### Composite model v1 + +Every object is a rule, all historical content is stored in its nested field (an array). + +`server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_composite_saved_objects_type.ts`: + +```ts +const SO_TYPE = 'security-rule-composite'; + +const mappings = { + dynamic: false, + properties: { + rule_id: { + type: 'keyword', + }, + versions: { + type: 'nested', + properties: { + name: { + type: 'keyword', + }, + rule_content_version: { + type: 'version', + }, + stack_version_min: { + type: 'version', + }, + stack_version_max: { + type: 'version', + }, + }, + }, + }, +}; +``` + +### Composite model v2 + +Every object is a rule. Historical version information is stored as an array of small objects +which is mapped as a nested field. Historical content is stored in a map where keys are formed +in a special way so that we can fetch individual content versions for many rules in bulk. + +`server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_composite2_saved_objects_type.ts`: + +```ts +export const SO_TYPE = 'security-rule-composite2'; + +interface RuleAssetComposite2Attributes { + rule_id: string; + versions: RuleVersionInfo[]; + content: Record; +} + +interface RuleVersionInfo { + rule_content_version: string; + stack_version_min: string; + stack_version_max: string; +} + +const mappings = { + dynamic: 'strict', + properties: { + rule_id: { + type: 'keyword', + }, + versions: { + type: 'nested', + properties: { + rule_content_version: { + type: 'version', + }, + stack_version_min: { + type: 'version', + }, + stack_version_max: { + type: 'version', + }, + }, + }, + content: { + type: 'flattened', + }, + }, +}; +``` + +## API endpoints + +See the implementation in `x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api`. + +### Get status of prebuilt rules + +```txt +GET /internal/detection_engine/prebuilt_rules/status +``` + +Response body: + +```ts + +export interface GetPrebuiltRulesStatusResponseBody { + status_code: number; + message: string; + attributes: { + /** Aggregated info about all prebuilt rules */ + stats: PrebuiltRulesStatusStats; + }; +} + +export interface PrebuiltRulesStatusStats { + /** Total number of existing (known) prebuilt rules */ + num_prebuilt_rules_total: number; // do we need it? + /** Number of installed prebuilt rules */ + num_prebuilt_rules_installed: number; // do we need it? + /** Number of prebuilt rules available for installation (not yet installed) */ + num_prebuilt_rules_to_install: number; + /** Number of installed prebuilt rules available for upgrade (stock + customized) */ + num_prebuilt_rules_to_upgrade: number; + + /** Signature ids ("rule_id") of prebuilt rules available for installation (not yet installed) */ + rule_ids_to_install: string[]; + /** Signature ids ("rule_id") of installed prebuilt rules available for upgrade (stock + customized) */ + rule_ids_to_upgrade: string[]; + + // In the future we could add more stats such as: + // - number of installed prebuilt rules which were deprecated + // - number of installed prebuilt rules which are not compatible with the current version of Kibana +} +``` + +Implementation: `server/lib/detection_engine/prebuilt_rules/api/get_prebuilt_rules_status/route.ts`. + +### Review rules that can be upgraded + +```txt +POST /internal/detection_engine/prebuilt_rules/upgrade/_review +``` + +Response body: + +```ts +export interface ReviewRuleUpgradeResponseBody { + status_code: number; + message: string; + attributes: { + /** Aggregated info about all rules available for upgrade */ + stats: RuleUpgradeStatsForReview; + /** Info about individual rules: one object per each rule available for upgrade */ + rules: RuleUpgradeInfoForReview[]; + }; +} + +export interface RuleUpgradeStatsForReview { + /** Number of installed prebuilt rules available for upgrade (stock + customized) */ + num_rules_to_upgrade: number; + /** Number of installed prebuilt rules available for upgrade which are stock (non-customized) */ + num_stock_rules_to_upgrade: number; + /** Number of installed prebuilt rules available for upgrade which are customized by the user */ + num_customized_rules_to_upgrade: number; + /** A union of all tags of all rules available for upgrade */ + tags: RuleTagArray; + /** A union of all fields "to be upgraded" across all the rules available for upgrade. An array of field names. */ + fields: string[]; +} + +export interface RuleUpgradeInfoForReview { + id: RuleObjectId; + rule_id: RuleSignatureId; + rule: DiffableRule; + diff: { + fields: { + name?: ThreeWayDiff; + description?: ThreeWayDiff; + // etc; only fields that have some changes or conflicts will be returned + }; + has_conflict: boolean; + }; +} +``` + +Implementation: `server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/route.ts`. + +### Perform rule upgrade + +```txt +POST /internal/detection_engine/prebuilt_rules/upgrade/_perform +``` + +Request body: + +```ts +export interface PerformRuleUpgradeRequestBody { + mode: 'ALL_RULES' | 'SPECIFIC_RULES'; + pick_version?: 'BASE' | 'CURRENT' | 'TARGET' | 'MERGED'; + rules: SingleRuleUpgradeRequest[]; // required if mode is SPECIFIC_RULES +} + +export interface SingleRuleUpgradeRequest { + id: RuleObjectId; + pick_version?: 'BASE' | 'CURRENT' | 'TARGET' | 'MERGED'; + fields?: { + name?: FieldUpgradeRequest; + description?: FieldUpgradeRequest; + // etc + // Every non-specified field will default to pick_version: 'MERGED'. + // If pick_version is MERGED and there's a merge conflict the endpoint will throw. + }; + + /** + * This parameter is needed for handling race conditions with Optimistic Concurrency Control. + * Two or more users can call upgrade/_review and upgrade/_perform endpoints concurrently. + * Also, in general the time between these two calls can be anything. + * The idea is to only allow the user to upgrade a rule if the user has reviewed the exact version + * of it that had been returned from the _review endpoint. If the version changed on the BE, + * upgrade/_perform endpoint will return a version mismatch error for this rule. + */ + rule_content_version: SemanticVersion; + + /** + * This parameter is needed for handling race conditions with Optimistic Concurrency Control. + * Two or more users can call upgrade/_review and upgrade/_perform endpoints concurrently. + * Also, in general the time between these two calls can be anything. + * The idea is to only allow the user to upgrade a rule if the user has reviewed the exact revision + * of it that had been returned from the _review endpoint. If the revision changed on the BE, + * upgrade/_perform endpoint will return a revision mismatch error for this rule. + */ + rule_revision: number; +} + +export interface FieldUpgradeRequest { + pick_version: 'BASE' | 'CURRENT' | 'TARGET' | 'MERGED' | 'RESOLVED'; + resolved_value: T; // required if pick_version is RESOLVED; type depends on the rule field type +} +``` + +Response body: + +```ts +export interface PerformRuleUpgradeResponseBody { + status_code: number; + message: string; + attributes: { + summary: { + total: number; + succeeded: number: + skipped: number; + failed: number; + }; + results: { + updated: RuleResponse[]; + skipped: Array<{ + rule_id: RuleSignatureId; + reason_code: 'RULE_NOT_FOUND' | 'RULE_UP_TO_DATE'; // or anything else + }>; + }; + errors: Array<{ + message: string; + error_code: string; // maybe not needed for now + status_code: number; + rules: Array<{ + rule_id: RuleSignatureId; + name?: string; + }>; + }>; + } +} +``` + +Implementation: not implemented in this POC. + +### Review rules that can be installed + +```txt +POST /internal/detection_engine/prebuilt_rules/installation/_review +``` + +Response body: + +```ts +export interface ReviewRuleInstallationResponseBody { + status_code: number; + message: string; + attributes: { + /** Aggregated info about all rules available for installation */ + stats: RuleInstallationStatsForReview; + /** Info about individual rules: one object per each rule available for installation */ + rules: RuleInstallationInfoForReview[]; + }; +} + +export interface RuleInstallationStatsForReview { + /** Number of prebuilt rules available for installation */ + num_rules_to_install: number; + /** A union of all tags of all rules available for installation */ + tags: RuleTagArray; +} + +// Option 1: rule ids and versions + all fields from DiffableRule +// Option 2: rule ids and versions + selected fields from DiffableRule (depending on the rule type) +export type RuleInstallationInfoForReview = DiffableRule & { + rule_id: RuleSignatureId; + rule_content_version: SemanticVersion; + stack_version_min: SemanticVersion; + stack_version_max: SemanticVersion; +}; +``` + +Implementation: `server/lib/detection_engine/prebuilt_rules/api/review_rule_installation/route.ts`. + +### Perform rule installation + +```txt +POST /internal/detection_engine/prebuilt_rules/installation/_perform +``` + +Request body: + +```ts +export interface PerformRuleInstallationRequestBody { + mode: `ALL_RULES` | `SPECIFIC_RULES`; + rules: SingleRuleInstallationRequest[]; // required if mode is `SPECIFIC_RULES` +} + +export interface SingleRuleInstallationRequest { + rule_id: RuleSignatureId; + + /** + * This parameter is needed for handling race conditions with Optimistic Concurrency Control. + * Two or more users can call installation/_review and installation/_perform endpoints concurrently. + * Also, in general the time between these two calls can be anything. + * The idea is to only allow the user to install a rule if the user has reviewed the exact version + * of it that had been returned from the _review endpoint. If the version changed on the BE, + * installation/_perform endpoint will return a version mismatch error for this rule. + */ + rule_content_version: SemanticVersion; +} +``` + +Response body: + +```ts +export interface PerformRuleInstallationResponseBody { + status_code: number; + message: string; + attributes: { + summary: { + total: number; + succeeded: number: + skipped: number; + failed: number; + }; + results: { + created: RuleResponse[]; + skipped: Array<{ + rule_id: RuleSignatureId; + reason_code: 'RULE_NOT_FOUND' | 'RULE_INSTALLED'; // or anything else + }>; + }; + errors: Array<{ + message: string; + error_code: string; // maybe not needed for now + status_code: number; + rules: Array<{ + rule_id: RuleSignatureId; + name?: string; + }>; + }>; + } +} +``` + +Implementation: not implemented in this POC. + +## API performance considerations + +```txt +GET /internal/detection_engine/prebuilt_rules/status +``` + +Should be fast and lightweight (< 1 second). Should: + +- have O(1) complexity +- require as few requests to ES as possible +- not load a lot of data into memory +- not do heavy in-memory calculations + +```txt +POST /internal/detection_engine/prebuilt_rules/*/_review +``` + +Could be slightly slow (1 to 5 seconds). Can: + +- have O(n + k) complexity, where n is installed, k is known prebuilt rules (NOT historical versions) +- do some requests to ES, but not N+1 +- load a lot of data into memory +- do heavy in-memory calculations + +```txt +POST /internal/detection_engine/prebuilt_rules/*/_perform +``` + +Could be moderately slow (< 1 minute). Can: + +- have O(n + k) complexity, where n is installed, k is known prebuilt rules (NOT historical versions) +- do N+1 requests to ES +- load a lot of data into memory +- do heavy in-memory calculations + +## API testing + +All the 3 added endpoints accept a `data_model` parameter so we could test their work and performance +in different conditions and with different data models. + +1. Generate test prebuilt rule assets (the endpoint will do it for all 3 data models). + Pick whatever number of versions you want to be generated per each rule. + + ```txt + POST /internal/detection_engine/prebuilt_rules/_install_test_assets + { + "num_versions_per_rule": 10 + } + ``` + +2. Test get status endpoint + + ```txt + GET /internal/detection_engine/prebuilt_rules/status?data_model=flat + + GET /internal/detection_engine/prebuilt_rules/status?data_model=composite + + GET /internal/detection_engine/prebuilt_rules/status?data_model=composite2 + ``` + +3. Test review installation endpoint + + ```txt + POST /internal/detection_engine/prebuilt_rules/installation/_review + { + "data_model": "flat" + } + + POST /internal/detection_engine/prebuilt_rules/installation/_review + { + "data_model": "composite" + } + + POST /internal/detection_engine/prebuilt_rules/installation/_review + { + "data_model": "composite2" + } + ``` + +4. Test review upgrade endpoint + + ```txt + POST /internal/detection_engine/prebuilt_rules/upgrade/_review + { + "data_model": "flat" + } + + POST /internal/detection_engine/prebuilt_rules/upgrade/_review + { + "data_model": "composite" + } + + POST /internal/detection_engine/prebuilt_rules/upgrade/_review + { + "data_model": "composite2" + } + ``` + +## Rule fields + +I did some research on rule fields to be able to determine which rule fields will be "customizable" and which will be "technical". + +Please find the result and follow-up work to do in a dedicated ticket: + +https://github.com/elastic/kibana/issues/147239 + +## Diff algorithm + +This section describes an algorithm that returns a 3-way diff between 3 rule versions: base, current, target. + +### Definition: diffable rule + +We have two data structures that represent a prebuilt rule: + +- `PrebuiltRuleToInstall`: schema for a prebuilt rule asset (filesystem or fleet package based). +- `RuleResponse`: schema for an Alerting Framework's rule. + +These data structures are similar but different. In order to be able to run a diff between +an already installed prebuilt rule (`RuleResponse`) and its next version shipped by Elastic +(`PrebuiltRuleToInstall`) we would first need to normalize both of them to a common interface +that would be suitable for passing to the diff algorithm. This common interface is `DiffableRule`. + +`common/detection_engine/prebuilt_rules/poc/diffable_rule_model/diffable_rule.ts`: + +
+ +```ts +export type DiffableCommonFields = t.TypeOf; +export const DiffableCommonFields = buildSchema({ + required: { + // Technical fields + // NOTE: We might consider removing them from the schema and returning from the API + // not via the fields diff, but via dedicated properties in the response body. + rule_id: RuleSignatureId, + rule_content_version: SemanticVersion, + stack_version_min: SemanticVersion, + stack_version_max: SemanticVersion, + meta: RuleMetadata, + + // Main domain fields + name: RuleName, + tags: RuleTagArray, + description: RuleDescription, + severity: Severity, + severity_mapping: SeverityMapping, + risk_score: RiskScore, + risk_score_mapping: RiskScoreMapping, + + // About -> Advanced settings + references: RuleReferenceArray, + false_positives: RuleFalsePositiveArray, + threat: ThreatArray, + note: InvestigationGuide, + setup: SetupGuide, + related_integrations: RelatedIntegrationArray, + required_fields: RequiredFieldArray, + author: RuleAuthorArray, + license: RuleLicense, + + // Other domain fields + rule_schedule: RuleSchedule, // NOTE: new field + actions: RuleActionArray, + throttle: RuleActionThrottle, + exceptions_list: ExceptionListArray, + max_signals: MaxSignals, + }, + optional: { + rule_name_override: RuleNameOverrideObject, // NOTE: new field + timestamp_override: TimestampOverrideObject, // NOTE: new field + timeline_template: TimelineTemplateReference, // NOTE: new field + building_block: BuildingBlockObject, // NOTE: new field + }, +}); + +export type DiffableCustomQueryFields = t.TypeOf; +export const DiffableCustomQueryFields = buildSchema({ + required: { + type: t.literal('query'), + data_query: RuleKqlQuery, // NOTE: new field + }, + optional: { + data_source: RuleDataSource, // NOTE: new field + alert_suppression: AlertSuppression, + }, +}); + +export type DiffableSavedQueryFields = t.TypeOf; +export const DiffableSavedQueryFields = buildSchema({ + required: { + type: t.literal('saved_query'), + data_query: RuleKqlQuery, // NOTE: new field + }, + optional: { + data_source: RuleDataSource, // NOTE: new field + alert_suppression: AlertSuppression, + }, +}); + +export type DiffableEqlFields = t.TypeOf; +export const DiffableEqlFields = buildSchema({ + required: { + type: t.literal('eql'), + data_query: RuleEqlQuery, // NOTE: new field + }, + optional: { + data_source: RuleDataSource, // NOTE: new field + event_category_override: EventCategoryOverride, + timestamp_field: TimestampField, + tiebreaker_field: TiebreakerField, + }, +}); + +export type DiffableThreatMatchFields = t.TypeOf; +export const DiffableThreatMatchFields = buildSchema({ + required: { + type: t.literal('threat_match'), + data_query: RuleKqlQuery, // NOTE: new field + threat_query: InlineKqlQuery, // NOTE: new field + threat_index, + threat_mapping, + }, + optional: { + data_source: RuleDataSource, // NOTE: new field + threat_indicator_path, + concurrent_searches, // Should combine concurrent_searches and items_per_search? + items_per_search, + }, +}); + +export type DiffableThresholdFields = t.TypeOf; +export const DiffableThresholdFields = buildSchema({ + required: { + type: t.literal('threshold'), + data_query: RuleKqlQuery, // NOTE: new field + threshold: Threshold, + }, + optional: { + data_source: RuleDataSource, // NOTE: new field + }, +}); + +export type DiffableMachineLearningFields = t.TypeOf; +export const DiffableMachineLearningFields = buildSchema({ + required: { + type: t.literal('machine_learning'), + machine_learning_job_id, + anomaly_threshold, + }, + optional: {}, +}); + +export type DiffableNewTermsFields = t.TypeOf; +export const DiffableNewTermsFields = buildSchema({ + required: { + type: t.literal('new_terms'), + data_query: InlineKqlQuery, // NOTE: new field + new_terms_fields: NewTermsFields, + history_window_start: HistoryWindowStart, + }, + optional: { + data_source: RuleDataSource, // NOTE: new field + }, +}); + +/** + * Represents a normalized rule object that is suitable for passing to the diff algorithm. + * Every top-level field of a diffable rule can be compared separately on its own. + * + * It's important to do such normalization because: + * + * 1. We need to compare installed rules with prebuilt rule content. These objects have similar but not exactly + * the same interfaces. In order to compare them we need to convert them to a common interface. + * + * 2. It only makes sense to compare certain rule fields in combination with other fields. For example, + * we combine `index` and `data_view_id` fields into a `RuleDataSource` object, so that later we could + * calculate a diff for this whole object. If we don't combine them the app would successfully merge the + * following values independently from each other without a conflict: + * + * Base version: index=[logs-*], data_view_id=undefined + * Current version: index=[], data_view_id=some-data-view // user switched to a data view + * Target version: index=[logs-*, filebeat-*], data_view_id=undefined // Elastic added a new index pattern + * Merged version: index=[filebeat-*], data_view_id=some-data-view ??? + * + * Instead, semantically such change represents a conflict because the data source of the rule was changed + * in a potentially incompatible way, and the user might want to review the change and resolve it manually. + * The user must either pick index patterns or a data view, but not both at the same time. + * + * NOTE: Every top-level field in a DiffableRule MUST BE LOGICALLY INDEPENDENT from other + * top-level fields. + */ +export type DiffableRule = t.TypeOf; +export const DiffableRule = t.intersection([ + DiffableCommonFields, + t.union([ + DiffableCustomQueryFields, + DiffableSavedQueryFields, + DiffableEqlFields, + DiffableThreatMatchFields, + DiffableThresholdFields, + DiffableMachineLearningFields, + DiffableNewTermsFields, + ]), +]); +``` + +
+ +### Definition: 3-way diff + +We will use a 3-way diff algorithm for two things: + +- Calculating a 3-way diff result for every "diffable" rule field (every top-level field of a `DiffableRule`). +- Calculating a 3-way diff result for the whole rule serialized into JSON + +In order to calculate a 3-way diff result for a field, we will need 3 input values: + +- The base version of the field. +- The current version of the field. +- The target version of the field. + +And, our goal will be: + +- to try to automatically merge these 3 versions into a 4th one that could be accepted or rejected by the user +- to flag about a merge conflict when these 3 versions can't be automatically merged + +Potential reasons for a conflict: + +- the 3 versions can't be technically merged unambiguously +- it's possible to merge it technically but it wouldn't be safe: it would bring a risk of breaking the + rule's behavior or introducing unintended side-effects in the behavior from the user's point of view + +Below is a `ThreeWayDiff` result's interface. + +`common/detection_engine/prebuilt_rules/poc/diff_model/three_way_diff.ts`: + +
+ +```ts +/** + * Three versions of a value to pass to a diff algorithm. + */ +export interface ThreeVersionsOf { + /** + * Corresponds to the stock version of the currently installed prebuilt rule. + */ + base_version: TValue; + + /** + * Corresponds exactly to the currently installed prebuilt rule: + * - to the customized version (if it's customized) + * - to the stock version (if it's not customized) + */ + current_version: TValue; + + /** + * Corresponds to the "new" stock version that the user is trying to upgrade to. + */ + target_version: TValue; +} + +/** + * Represents a result of an abstract three-way diff/merge operation on a value + * (could be a whole rule JSON or a given rule field). + * + * Typical situations: + * + * 1. base=A, current=A, target=A => merged=A, conflict=false + * Stock rule, the value hasn't changed. + * + * 2. base=A, current=A, target=B => merged=B, conflict=false + * Stock rule, the value has changed. + * + * 3. base=A, current=B, target=A => merged=B, conflict=false + * Customized rule, the value hasn't changed. + * + * 4. base=A, current=B, target=B => merged=B, conflict=false + * Customized rule, the value has changed exactly the same way as in the user customization. + * + * 5. base=A, current=B, target=C => merged=D, conflict=false + * Customized rule, the value has changed, conflict between B and C resolved automatically. + * + * 6. base=A, current=B, target=C => merged=C, conflict=true + * Customized rule, the value has changed, conflict between B and C couldn't be resolved automatically. + */ +export interface ThreeWayDiff extends ThreeVersionsOf { + /** + * The result of an automatic three-way merge of three values: + * - base version + * - current version + * - target version + * + * Exact merge algorithm depends on the value: + * - one algo could be used for single-line strings and keywords (e.g. rule name) + * - another one could be used for multiline text (e.g. rule description) + * - another one could be used for arrays of keywords (e.g. rule tags) + * - another one could be used for the MITRE ATT&CK data structure + * - etc + * + * Merged version always has a value. We do our best to resolve conflicts automatically. + * If they can't be resolved automatically, merged version is equal to target version. + */ + merged_version: TValue; + + /** + * Tells which combination corresponds to the three input versions of the value for this specific diff. + */ + diff_outcome: ThreeWayDiffOutcome; + + /** + * The type of result of an automatic three-way merge of three values: + * - base version + * - current version + * - target version + */ + merge_outcome: ThreeWayMergeOutcome; + + /** + * Tells if the value has changed in the target version and the current version could be updated. + * True if: + * - base=A, current=A, target=B + * - base=A, current=B, target=C + */ + has_value_changed: boolean; + + /** + * True if: + * - current != target and we couldn't automatically resolve the conflict between them + * + * False if: + * - current == target (value won't change) + * - current != target && current == base (stock rule will get a new value) + * - current != target and we automatically resolved the conflict between them + */ + has_conflict: boolean; +} + +/** + * Given the three versions of a value, calculates a three-way diff for it. + */ +export type ThreeWayDiffAlgorithm = ( + versions: ThreeVersionsOf +) => ThreeWayDiff; + +/** + * Result of comparing three versions of a value against each other. + * Defines 5 typical combinations of 3 versions of a value. + */ +export enum ThreeWayDiffOutcome { + /** Stock rule, the value hasn't changed in the target version. */ + StockValueNoUpdate = 'BASE=A, CURRENT=A, TARGET=A', + + /** Stock rule, the value has changed in the target version. */ + StockValueCanUpdate = 'BASE=A, CURRENT=A, TARGET=B', + + /** Customized rule, the value hasn't changed in the target version comparing to the base one. */ + CustomizedValueNoUpdate = 'BASE=A, CURRENT=B, TARGET=A', + + /** Customized rule, the value has changed in the target version exactly the same way as in the user customization. */ + CustomizedValueSameUpdate = 'BASE=A, CURRENT=B, TARGET=B', + + /** Customized rule, the value has changed in the target version and is not equal to the current version. */ + CustomizedValueCanUpdate = 'BASE=A, CURRENT=B, TARGET=C', +} + +/** + * Type of result of an automatic three-way merge of three values: + * - base version + * - current version + * - target version + */ +export enum ThreeWayMergeOutcome { + /** Took current version and returned as the merged one. */ + Current = 'CURRENT', + + /** Took target version and returned as the merged one. */ + Target = 'TARGET', + + /** Merged three versions successfully into a new one. */ + Merged = 'MERGED', + + /** Merged three versions with a conflict. */ + MergedWithConflict = 'MERGED_WITH_CONFLICT', +} +``` + +
+ +### The algorithm itself + +GIVEN a list of prebuilt rules that can be upgraded (`currentVersion[]`) +AND a list of the corresponding base asset saved objects (`baseVersion[]`) +AND a list of the corresponding target asset saved objects (`targetVersion[]`) +DO run the diff algorithm for every match of these 3 versions. + +`common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculate_rule_diff.ts`: + +
+ +```ts +export interface CalculateRuleDiffArgs { + currentVersion: RuleResponse; + baseVersion: PrebuiltRuleContent; + targetVersion: PrebuiltRuleContent; +} + +export interface CalculateRuleDiffResult { + ruleDiff: RuleDiff; + ruleVersions: { + input: { + current: RuleResponse; + base: PrebuiltRuleContent; + target: PrebuiltRuleContent; + }; + output: { + current: DiffableRule; + base: DiffableRule; + target: DiffableRule; + }; + }; +} + +/** + * Calculates a rule diff for a given set of 3 versions of the rule: + * - currenly installed version + * - base version that is the corresponding stock rule content + * - target version which is the stock rule content the user wants to update the rule to + */ +export const calculateRuleDiff = (args: CalculateRuleDiffArgs): CalculateRuleDiffResult => { + /* + 1. Convert current, base and target versions to `DiffableRule`. + 2. Calculate a `RuleFieldsDiff`. For every top-level field of `DiffableRule`: + 2.1. Pick a code path based on the rule type. + 2.2. Pick a concrete diff algorithm (function) per rule field based on the field name or type. + - one algo for rule name and other simple string fields + - another one for tags and other arrays of keywords + - another one for multiline text fields (investigation guide, setup guide, etc) + - another one for `data_source` + - etc + 2.3. Call the picked diff function to get a `ThreeWayDiff` result + 2.4. Add the result to the `RuleFieldsDiff` object as a key-value pair "fieldName: ThreeWayDiff". + 3. Calculate a `RuleJsonDiff` for the whole rule based on the `RuleFieldsDiff` from the previous step. + 4. Return the `RuleFieldsDiff` and `RuleJsonDiff` objects. + */ + + const { baseVersion, currentVersion, targetVersion } = args; + + const diffableBaseVersion = convertRuleToDiffable(baseVersion); + const diffableCurrentVersion = convertRuleToDiffable(currentVersion); + const diffableTargetVersion = convertRuleToDiffable(targetVersion); + + const fieldsDiff = calculateRuleFieldsDiff({ + base_version: diffableBaseVersion, + current_version: diffableCurrentVersion, + target_version: diffableTargetVersion, + }); + + // I'm thinking that maybe instead of eagerly calculating it for many rules on the BE side we should + // calculate it on the FE side on demand, only if the user switches to the corresponding view. + const jsonDiff = calculateRuleJsonDiff(fieldsDiff); + + return { + ruleDiff: combineDiffs(fieldsDiff, jsonDiff), + ruleVersions: { + input: { + current: currentVersion, + base: baseVersion, + target: targetVersion, + }, + output: { + current: diffableCurrentVersion, + base: diffableBaseVersion, + target: diffableTargetVersion, + }, + }, + }; +}; + +const combineDiffs = (fieldsDiff: RuleFieldsDiff, jsonDiff: RuleJsonDiff): RuleDiff => { + const hasFieldsConflict = Object.values>(fieldsDiff).some( + (fieldDiff) => fieldDiff.has_conflict + ); + + const hasJsonConflict = jsonDiff.has_conflict; + + return { + fields: fieldsDiff, + json: jsonDiff, + has_conflict: hasFieldsConflict || hasJsonConflict, + }; +}; +``` + +
+ +The algorithm's overall structure is fully implemented and works, but it uses a simple diff algorithm +for calculating field diffs. This algorithm is kinda nasty: it doesn't try to merge any values +and marks a diff as conflict if base version != current version != target version. + +`common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculation/algorithms/simple_diff_algorithm.ts`. + +The idea is to write more flexible algorithms for different rule fields that would generate fewer +conflicts and would try to automatically merge changes when it can be technically done and it won't +result in inintended changes in the rule from the user standpoint. diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/get_prebuilt_rules_status/request_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/get_prebuilt_rules_status/request_schema.ts new file mode 100644 index 0000000000000..f80f9c3c1ab46 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/get_prebuilt_rules_status/request_schema.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; + +export type PrebuiltRuleContentDataModel = t.TypeOf; +export const PrebuiltRuleContentDataModel = t.union([ + t.literal('flat'), + t.literal('composite'), + t.literal('composite2'), +]); + +export type GetPrebuiltRulesStatusRequestQuery = t.TypeOf< + typeof GetPrebuiltRulesStatusRequestQuery +>; +export const GetPrebuiltRulesStatusRequestQuery = t.exact( + t.type({ + data_model: PrebuiltRuleContentDataModel, + }) +); diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/get_prebuilt_rules_status/response_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/get_prebuilt_rules_status/response_schema.ts new file mode 100644 index 0000000000000..a18113a39ef6e --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/get_prebuilt_rules_status/response_schema.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface GetPrebuiltRulesStatusResponseBody { + status_code: number; + message: string; + attributes: { + /** Aggregated info about all prebuilt rules */ + stats: PrebuiltRulesStatusStats; + }; +} + +export interface PrebuiltRulesStatusStats { + /** Total number of existing (known) prebuilt rules */ + num_prebuilt_rules_total: number; // do we need it? + /** Number of installed prebuilt rules */ + num_prebuilt_rules_installed: number; // do we need it? + /** Number of prebuilt rules available for installation (not yet installed) */ + num_prebuilt_rules_to_install: number; + /** Number of installed prebuilt rules available for upgrade (stock + customized) */ + num_prebuilt_rules_to_upgrade: number; + + /** Signature ids ("rule_id") of prebuilt rules available for installation (not yet installed) */ + rule_ids_to_install: string[]; + /** Signature ids ("rule_id") of installed prebuilt rules available for upgrade (stock + customized) */ + rule_ids_to_upgrade: string[]; + + // In the future we could add more stats such as: + // - number of installed prebuilt rules which were deprecated + // - number of installed prebuilt rules which are not compatible with the current version of Kibana +} diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/review_rule_installation/request_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/review_rule_installation/request_schema.ts new file mode 100644 index 0000000000000..aa721698527ca --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/review_rule_installation/request_schema.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; +import { PrebuiltRuleContentDataModel } from '../get_prebuilt_rules_status/request_schema'; + +export type ReviewRuleInstallationRequestBody = t.TypeOf; +export const ReviewRuleInstallationRequestBody = t.exact( + t.type({ + data_model: PrebuiltRuleContentDataModel, + }) +); diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/review_rule_installation/response_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/review_rule_installation/response_schema.ts new file mode 100644 index 0000000000000..315bbfc48af22 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/review_rule_installation/response_schema.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleSignatureId, RuleTagArray } from '../../../../rule_schema'; +import type { SemanticVersion } from '../../content_model/semantic_version'; +import type { SemanticVersion } from '../../content_model/prebuilt_rule_stack_version'; +import type { DiffableRule } from '../../diffable_rule_model/diffable_rule'; + +export interface ReviewRuleInstallationResponseBody { + status_code: number; + message: string; + attributes: { + /** Aggregated info about all rules available for installation */ + stats: RuleInstallationStatsForReview; + /** Info about individual rules: one object per each rule available for installation */ + rules: RuleInstallationInfoForReview[]; + }; +} + +export interface RuleInstallationStatsForReview { + /** Number of prebuilt rules available for installation */ + num_rules_to_install: number; + /** A union of all tags of all rules available for installation */ + tags: RuleTagArray; +} + +// Option 1: rule ids and versions + all fields from DiffableRule +// Option 2: rule ids and versions + selected fields from DiffableRule (depending on the rule type) +export type RuleInstallationInfoForReview = DiffableRule & { + rule_id: RuleSignatureId; + rule_content_version: SemanticVersion; + stack_version_min: SemanticVersion; + stack_version_max: SemanticVersion; +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/review_rule_upgrade/request_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/review_rule_upgrade/request_schema.ts new file mode 100644 index 0000000000000..93592071cbb5e --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/review_rule_upgrade/request_schema.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; +import { PrebuiltRuleContentDataModel } from '../get_prebuilt_rules_status/request_schema'; + +export type ReviewRuleUpgradeRequestBody = t.TypeOf; +export const ReviewRuleUpgradeRequestBody = t.exact( + t.type({ + data_model: PrebuiltRuleContentDataModel, + }) +); diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/review_rule_upgrade/response_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/review_rule_upgrade/response_schema.ts new file mode 100644 index 0000000000000..dac001a93c695 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/api/review_rule_upgrade/response_schema.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleObjectId, RuleSignatureId, RuleTagArray } from '../../../../rule_schema'; +import type { DiffableRule } from '../../diffable_rule_model/diffable_rule'; +import type { RuleFieldsDiff } from '../../diff_model/rule_diff'; + +export interface ReviewRuleUpgradeResponseBody { + status_code: number; + message: string; + attributes: { + /** Aggregated info about all rules available for upgrade */ + stats: RuleUpgradeStatsForReview; + /** Info about individual rules: one object per each rule available for upgrade */ + rules: RuleUpgradeInfoForReview[]; + }; +} + +export interface RuleUpgradeStatsForReview { + /** Number of installed prebuilt rules available for upgrade (stock + customized) */ + num_rules_to_upgrade: number; + /** Number of installed prebuilt rules available for upgrade which are stock (non-customized) */ + num_stock_rules_to_upgrade: number; + /** Number of installed prebuilt rules available for upgrade which are customized by the user */ + num_customized_rules_to_upgrade: number; + /** A union of all tags of all rules available for upgrade */ + tags: RuleTagArray; + /** A union of all fields "to be upgraded" across all the rules available for upgrade. An array of field names. */ + fields: string[]; +} + +export interface RuleUpgradeInfoForReview { + id: RuleObjectId; + rule_id: RuleSignatureId; + rule: DiffableRule; + diff: { + fields: Partial; + has_conflict: boolean; + }; +} diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_content.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_content.ts new file mode 100644 index 0000000000000..cde0a9a54cdc7 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_content.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; + +import { + BaseCreateProps, + RelatedIntegrationArray, + RequiredFieldArray, + RuleSignatureId, + SetupGuide, + TypeSpecificCreateProps, +} from '../../../rule_schema'; + +import { SemanticVersion } from './semantic_version'; + +/** + * Full content of a prebuilt rule. + * Is defined for each prebuilt rule in https://github.com/elastic/detection-rules. + * Is shipped via the `security_detection_engine` Fleet package. + * Is installed as separate saved objects when the package is installed. + * + * NOTE: This is NOT a schema of the saved objects. It's a model of a single version of the content. + */ +export type PrebuiltRuleContent = t.TypeOf; +export const PrebuiltRuleContent = t.intersection([ + BaseCreateProps, + TypeSpecificCreateProps, + t.exact( + t.type({ + rule_id: RuleSignatureId, + rule_content_version: SemanticVersion, + stack_version_min: SemanticVersion, + stack_version_max: SemanticVersion, + }) + ), + t.exact( + t.partial({ + related_integrations: RelatedIntegrationArray, + required_fields: RequiredFieldArray, + setup: SetupGuide, + }) + ), +]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_version_info.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_version_info.ts new file mode 100644 index 0000000000000..610e1aa334678 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_version_info.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleSignatureId } from '../../../rule_schema'; +import type { SemanticVersion } from './semantic_version'; + +export interface PrebuiltRuleVersionInfo { + rule_id: RuleSignatureId; + rule_content_version: SemanticVersion; + stack_version_min: SemanticVersion; + stack_version_max: SemanticVersion; +} diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/content_model/semantic_version.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/content_model/semantic_version.ts new file mode 100644 index 0000000000000..da14f7e83902f --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/content_model/semantic_version.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as t from 'io-ts'; +import { NonEmptyString } from '@kbn/securitysolution-io-ts-types'; +import type { RuleVersion } from '../../../rule_schema'; + +/** + * Type that should match the `version` field type of Elasticsearch. + * https://www.elastic.co/guide/en/elasticsearch/reference/current/version.html + */ +export type SemanticVersion = t.TypeOf; +export const SemanticVersion = NonEmptyString; + +export const getSemanticVersion = ( + ruleVersion: RuleVersion, + patchVersion: number +): SemanticVersion => `${ruleVersion}.0.${patchVersion}`; + +export const convertLegacyVersionToSemantic = (legacyVersion: RuleVersion): SemanticVersion => + getSemanticVersion(legacyVersion, 0); + +export const convertSemanticVersionToLegacy = (semanticVersion: SemanticVersion): RuleVersion => { + throw new Error('Not implemented'); +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculate_rule_diff.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculate_rule_diff.ts new file mode 100644 index 0000000000000..b72a519f7ed48 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculate_rule_diff.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleResponse } from '../../../rule_schema'; +import type { PrebuiltRuleContent } from '../content_model/prebuilt_rule_content'; +import type { RuleDiff, RuleFieldsDiff, RuleJsonDiff } from '../diff_model/rule_diff'; +import type { ThreeWayDiff } from '../diff_model/three_way_diff'; +import type { DiffableRule } from '../diffable_rule_model/diffable_rule'; + +import { convertRuleToDiffable } from './normalization/convert_rule_to_diffable'; +import { calculateRuleFieldsDiff } from './calculation/calculate_rule_fields_diff'; +import { calculateRuleJsonDiff } from './calculation/calculate_rule_json_diff'; + +export interface CalculateRuleDiffArgs { + currentVersion: RuleResponse; + baseVersion: PrebuiltRuleContent; + targetVersion: PrebuiltRuleContent; +} + +export interface CalculateRuleDiffResult { + ruleDiff: RuleDiff; + ruleVersions: { + input: { + current: RuleResponse; + base: PrebuiltRuleContent; + target: PrebuiltRuleContent; + }; + output: { + current: DiffableRule; + base: DiffableRule; + target: DiffableRule; + }; + }; +} + +/** + * Calculates a rule diff for a given set of 3 versions of the rule: + * - currenly installed version + * - base version that is the corresponding stock rule content + * - target version which is the stock rule content the user wants to update the rule to + */ +export const calculateRuleDiff = (args: CalculateRuleDiffArgs): CalculateRuleDiffResult => { + /* + 1. Convert current, base and target versions to `DiffableRule`. + 2. Calculate a `RuleFieldsDiff`. For every top-level field of `DiffableRule`: + 2.1. Pick a code path based on the rule type. + 2.2. Pick a concrete diff algorithm (function) per rule field based on the field name or type. + - one algo for rule name and other simple string fields + - another one for tags and other arrays of keywords + - another one for multiline text fields (investigation guide, setup guide, etc) + - another one for `data_source` + - etc + 2.3. Call the picked diff function to get a `ThreeWayDiff` result + 2.4. Add the result to the `RuleFieldsDiff` object as a key-value pair "fieldName: ThreeWayDiff". + 3. Calculate a `RuleJsonDiff` for the whole rule based on the `RuleFieldsDiff` from the previous step. + 4. Return the `RuleFieldsDiff` and `RuleJsonDiff` objects. + */ + + const { baseVersion, currentVersion, targetVersion } = args; + + const diffableBaseVersion = convertRuleToDiffable(baseVersion); + const diffableCurrentVersion = convertRuleToDiffable(currentVersion); + const diffableTargetVersion = convertRuleToDiffable(targetVersion); + + const fieldsDiff = calculateRuleFieldsDiff({ + base_version: diffableBaseVersion, + current_version: diffableCurrentVersion, + target_version: diffableTargetVersion, + }); + + // I'm thinking that maybe instead of eagerly calculating it for many rules on the BE side we should + // calculate it on the FE side on demand, only if the user switches to the corresponding view. + const jsonDiff = calculateRuleJsonDiff(fieldsDiff); + + return { + ruleDiff: combineDiffs(fieldsDiff, jsonDiff), + ruleVersions: { + input: { + current: currentVersion, + base: baseVersion, + target: targetVersion, + }, + output: { + current: diffableCurrentVersion, + base: diffableBaseVersion, + target: diffableTargetVersion, + }, + }, + }; +}; + +const combineDiffs = (fieldsDiff: RuleFieldsDiff, jsonDiff: RuleJsonDiff): RuleDiff => { + const hasFieldsConflict = Object.values>(fieldsDiff).some( + (fieldDiff) => fieldDiff.has_conflict + ); + + const hasJsonConflict = jsonDiff.has_conflict; + + return { + fields: fieldsDiff, + json: jsonDiff, + has_conflict: hasFieldsConflict || hasJsonConflict, + }; +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculation/algorithms/simple_diff_algorithm.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculation/algorithms/simple_diff_algorithm.ts new file mode 100644 index 0000000000000..63e67b70ff1ec --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculation/algorithms/simple_diff_algorithm.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { assertUnreachable } from '../../../../../../utility_types'; +import type { ThreeVersionsOf, ThreeWayDiff } from '../../../diff_model/three_way_diff'; +import { + determineDiffOutcome, + determineIfValueChanged, + ThreeWayDiffOutcome, +} from '../../../diff_model/three_way_diff_outcome'; +import { ThreeWayMergeOutcome } from '../../../diff_model/three_way_merge_outcome'; + +export const simpleDiffAlgorithm = ( + versions: ThreeVersionsOf +): ThreeWayDiff => { + const { + base_version: baseVersion, + current_version: currentVersion, + target_version: targetVersion, + } = versions; + + const diffOutcome = determineDiffOutcome(baseVersion, currentVersion, targetVersion); + const hasValueChanged = determineIfValueChanged(diffOutcome); + + const { mergeOutcome, mergedVersion } = mergeVersions( + baseVersion, + currentVersion, + targetVersion, + diffOutcome + ); + + return { + base_version: baseVersion, + current_version: currentVersion, + target_version: targetVersion, + merged_version: mergedVersion, + + diff_outcome: diffOutcome, + merge_outcome: mergeOutcome, + has_value_changed: hasValueChanged, + has_conflict: mergeOutcome === ThreeWayMergeOutcome.MergedWithConflict, + }; +}; + +interface MergeResult { + mergeOutcome: ThreeWayMergeOutcome; + mergedVersion: TValue; +} + +const mergeVersions = ( + baseVersion: TValue, + currentVersion: TValue, + targetVersion: TValue, + diffOutcome: ThreeWayDiffOutcome +): MergeResult => { + switch (diffOutcome) { + case ThreeWayDiffOutcome.StockValueNoUpdate: + case ThreeWayDiffOutcome.CustomizedValueNoUpdate: + case ThreeWayDiffOutcome.CustomizedValueSameUpdate: { + return { + mergeOutcome: ThreeWayMergeOutcome.Current, + mergedVersion: currentVersion, + }; + } + case ThreeWayDiffOutcome.StockValueCanUpdate: { + return { + mergeOutcome: ThreeWayMergeOutcome.Target, + mergedVersion: targetVersion, + }; + } + case ThreeWayDiffOutcome.CustomizedValueCanUpdate: { + return { + mergeOutcome: ThreeWayMergeOutcome.MergedWithConflict, + mergedVersion: targetVersion, + }; + } + default: + return assertUnreachable(diffOutcome); + } +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculation/calculate_rule_fields_diff.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculation/calculate_rule_fields_diff.ts new file mode 100644 index 0000000000000..26fdc71a5ee24 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculation/calculate_rule_fields_diff.ts @@ -0,0 +1,265 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { assertUnreachable } from '../../../../../utility_types'; +import { invariant } from '../../../../../utils/invariant'; + +import type { + DiffableCommonFields, + DiffableCustomQueryFields, + DiffableEqlFields, + DiffableMachineLearningFields, + DiffableNewTermsFields, + DiffableRule, + DiffableSavedQueryFields, + DiffableThreatMatchFields, + DiffableThresholdFields, +} from '../../diffable_rule_model/diffable_rule'; +import type { + CommonFieldsDiff, + CustomQueryFieldsDiff, + EqlFieldsDiff, + MachineLearningFieldsDiff, + NewTermsFieldsDiff, + RuleFieldsDiff, + SavedQueryFieldsDiff, + ThreatMatchFieldsDiff, + ThresholdFieldsDiff, +} from '../../diff_model/rule_diff'; + +import type { FieldsDiffAlgorithmsFor } from '../../diff_model/fields_diff'; +import type { ThreeVersionsOf } from '../../diff_model/three_way_diff'; +import { calculateFieldsDiffFor } from './diff_calculation_helpers'; +import { simpleDiffAlgorithm } from './algorithms/simple_diff_algorithm'; + +const BASE_TYPE_ERROR = `Base version can't be of different rule type`; +const TARGET_TYPE_ERROR = `Target version can't be of different rule type`; + +/** + * Calculates a three-way diff per each top-level rule field. + * Returns an object which keys are equal to rule's field names and values are + * three-way diffs calculated for those fields. + */ +export const calculateRuleFieldsDiff = ( + ruleVersions: ThreeVersionsOf +): RuleFieldsDiff => { + validateRuleVersions(ruleVersions); + + const commonFieldsDiff = calculateCommonFieldsDiff(ruleVersions); + // eslint-disable-next-line @typescript-eslint/naming-convention + const { base_version, current_version, target_version } = ruleVersions; + + switch (current_version.type) { + case 'query': { + invariant(base_version.type === 'query', BASE_TYPE_ERROR); + invariant(target_version.type === 'query', TARGET_TYPE_ERROR); + return { + ...commonFieldsDiff, + ...calculateCustomQueryFieldsDiff({ base_version, current_version, target_version }), + }; + } + case 'saved_query': { + invariant(base_version.type === 'saved_query', BASE_TYPE_ERROR); + invariant(target_version.type === 'saved_query', TARGET_TYPE_ERROR); + return { + ...commonFieldsDiff, + ...calculateSavedQueryFieldsDiff({ base_version, current_version, target_version }), + }; + } + case 'eql': { + invariant(base_version.type === 'eql', BASE_TYPE_ERROR); + invariant(target_version.type === 'eql', TARGET_TYPE_ERROR); + return { + ...commonFieldsDiff, + ...calculateEqlFieldsDiff({ base_version, current_version, target_version }), + }; + } + case 'threat_match': { + invariant(base_version.type === 'threat_match', BASE_TYPE_ERROR); + invariant(target_version.type === 'threat_match', TARGET_TYPE_ERROR); + return { + ...commonFieldsDiff, + ...calculateThreatMatchFieldsDiff({ base_version, current_version, target_version }), + }; + } + case 'threshold': { + invariant(base_version.type === 'threshold', BASE_TYPE_ERROR); + invariant(target_version.type === 'threshold', TARGET_TYPE_ERROR); + return { + ...commonFieldsDiff, + ...calculateThresholdFieldsDiff({ base_version, current_version, target_version }), + }; + } + case 'machine_learning': { + invariant(base_version.type === 'machine_learning', BASE_TYPE_ERROR); + invariant(target_version.type === 'machine_learning', TARGET_TYPE_ERROR); + return { + ...commonFieldsDiff, + ...calculateMachineLearningFieldsDiff({ base_version, current_version, target_version }), + }; + } + case 'new_terms': { + invariant(base_version.type === 'new_terms', BASE_TYPE_ERROR); + invariant(target_version.type === 'new_terms', TARGET_TYPE_ERROR); + return { + ...commonFieldsDiff, + ...calculateNewTermsFieldsDiff({ base_version, current_version, target_version }), + }; + } + default: { + return assertUnreachable(current_version, 'Unhandled rule type'); + } + } +}; + +const validateRuleVersions = (ruleVersions: ThreeVersionsOf): void => { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { base_version, current_version, target_version } = ruleVersions; + const types = new Set([base_version.type, current_version.type, target_version.type]); + + if (types.size > 1) { + throw new Error('Cannot change rule type during rule upgrade'); + } +}; + +const calculateCommonFieldsDiff = ( + fieldsVersions: ThreeVersionsOf +): CommonFieldsDiff => { + return calculateFieldsDiffFor(fieldsVersions, commonFieldsDiffAlgorithms); +}; + +const commonFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor = { + rule_id: simpleDiffAlgorithm, + rule_content_version: simpleDiffAlgorithm, + stack_version_min: simpleDiffAlgorithm, + stack_version_max: simpleDiffAlgorithm, + meta: simpleDiffAlgorithm, + name: simpleDiffAlgorithm, + tags: simpleDiffAlgorithm, + description: simpleDiffAlgorithm, + severity: simpleDiffAlgorithm, + severity_mapping: simpleDiffAlgorithm, + risk_score: simpleDiffAlgorithm, + risk_score_mapping: simpleDiffAlgorithm, + references: simpleDiffAlgorithm, + false_positives: simpleDiffAlgorithm, + threat: simpleDiffAlgorithm, + note: simpleDiffAlgorithm, + setup: simpleDiffAlgorithm, + related_integrations: simpleDiffAlgorithm, + required_fields: simpleDiffAlgorithm, + author: simpleDiffAlgorithm, + license: simpleDiffAlgorithm, + rule_schedule: simpleDiffAlgorithm, + actions: simpleDiffAlgorithm, + throttle: simpleDiffAlgorithm, + exceptions_list: simpleDiffAlgorithm, + max_signals: simpleDiffAlgorithm, + rule_name_override: simpleDiffAlgorithm, + timestamp_override: simpleDiffAlgorithm, + timeline_template: simpleDiffAlgorithm, + building_block: simpleDiffAlgorithm, +}; + +const calculateCustomQueryFieldsDiff = ( + fieldsVersions: ThreeVersionsOf +): CustomQueryFieldsDiff => { + return calculateFieldsDiffFor(fieldsVersions, customQueryFieldsDiffAlgorithms); +}; + +const customQueryFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor = { + type: simpleDiffAlgorithm, + data_query: simpleDiffAlgorithm, + data_source: simpleDiffAlgorithm, + alert_suppression: simpleDiffAlgorithm, +}; + +const calculateSavedQueryFieldsDiff = ( + fieldsVersions: ThreeVersionsOf +): SavedQueryFieldsDiff => { + return calculateFieldsDiffFor(fieldsVersions, savedQueryFieldsDiffAlgorithms); +}; + +const savedQueryFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor = { + type: simpleDiffAlgorithm, + data_query: simpleDiffAlgorithm, + data_source: simpleDiffAlgorithm, + alert_suppression: simpleDiffAlgorithm, +}; + +const calculateEqlFieldsDiff = ( + fieldsVersions: ThreeVersionsOf +): EqlFieldsDiff => { + return calculateFieldsDiffFor(fieldsVersions, eqlFieldsDiffAlgorithms); +}; + +const eqlFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor = { + type: simpleDiffAlgorithm, + data_query: simpleDiffAlgorithm, + data_source: simpleDiffAlgorithm, + event_category_override: simpleDiffAlgorithm, + timestamp_field: simpleDiffAlgorithm, + tiebreaker_field: simpleDiffAlgorithm, +}; + +const calculateThreatMatchFieldsDiff = ( + fieldsVersions: ThreeVersionsOf +): ThreatMatchFieldsDiff => { + return calculateFieldsDiffFor(fieldsVersions, threatMatchFieldsDiffAlgorithms); +}; + +const threatMatchFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor = { + type: simpleDiffAlgorithm, + data_query: simpleDiffAlgorithm, + data_source: simpleDiffAlgorithm, + threat_query: simpleDiffAlgorithm, + threat_index: simpleDiffAlgorithm, + threat_mapping: simpleDiffAlgorithm, + threat_indicator_path: simpleDiffAlgorithm, + concurrent_searches: simpleDiffAlgorithm, + items_per_search: simpleDiffAlgorithm, +}; + +const calculateThresholdFieldsDiff = ( + fieldsVersions: ThreeVersionsOf +): ThresholdFieldsDiff => { + return calculateFieldsDiffFor(fieldsVersions, thresholdFieldsDiffAlgorithms); +}; + +const thresholdFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor = { + type: simpleDiffAlgorithm, + data_query: simpleDiffAlgorithm, + data_source: simpleDiffAlgorithm, + threshold: simpleDiffAlgorithm, +}; + +const calculateMachineLearningFieldsDiff = ( + fieldsVersions: ThreeVersionsOf +): MachineLearningFieldsDiff => { + return calculateFieldsDiffFor(fieldsVersions, machineLearningFieldsDiffAlgorithms); +}; + +const machineLearningFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor = + { + type: simpleDiffAlgorithm, + machine_learning_job_id: simpleDiffAlgorithm, + anomaly_threshold: simpleDiffAlgorithm, + }; + +const calculateNewTermsFieldsDiff = ( + fieldsVersions: ThreeVersionsOf +): NewTermsFieldsDiff => { + return calculateFieldsDiffFor(fieldsVersions, newTermsFieldsDiffAlgorithms); +}; + +const newTermsFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor = { + type: simpleDiffAlgorithm, + data_query: simpleDiffAlgorithm, + data_source: simpleDiffAlgorithm, + new_terms_fields: simpleDiffAlgorithm, + history_window_start: simpleDiffAlgorithm, +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculation/calculate_rule_json_diff.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculation/calculate_rule_json_diff.ts new file mode 100644 index 0000000000000..ad01d565bf920 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculation/calculate_rule_json_diff.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { sortBy } from 'lodash'; +import { mergeDiff3 } from 'node-diff3'; + +import type { RuleFieldsDiff, RuleJsonDiff } from '../../diff_model/rule_diff'; +import type { ThreeWayDiff } from '../../diff_model/three_way_diff'; +import { + determineDiffOutcome, + determineIfValueChanged, +} from '../../diff_model/three_way_diff_outcome'; +import { ThreeWayMergeOutcome } from '../../diff_model/three_way_merge_outcome'; + +export const calculateRuleJsonDiff = (fieldsDiff: RuleFieldsDiff): RuleJsonDiff => { + const baseFields = getFromDiff(fieldsDiff, (diff) => diff.base_version); + const currentFields = getFromDiff(fieldsDiff, (diff) => diff.current_version); + const targetFields = getFromDiff(fieldsDiff, (diff) => diff.target_version); + + const baseJson = serializeToJson(baseFields); + const currentJson = serializeToJson(currentFields); + const targetJson = serializeToJson(targetFields); + + const diffOutcome = determineDiffOutcome(baseJson, currentJson, targetJson); + const hasValueChanged = determineIfValueChanged(diffOutcome); + + // How do we calculate the merged version? + // Option 1: calculate a diff between the 3 JSONs; return the result in a standard format, e.g. git unidiff. + + // https://github.com/bhousel/node-diff3#mergeDiff3 + const mergeResult = mergeDiff3(currentJson, baseJson, targetJson, { + excludeFalseConflicts: true, + label: { + o: 'base version', + a: 'current version', + b: 'target version', + }, + }); + + const mergedJson = mergeResult.result.join('\n'); + const hasConflict = mergeResult.conflict; + + // How do we calculate the merged version? + // Option 2: simply combine the merged fields into an object and convert it to JSON. + + // const mergedFields = getFromDiff(fieldsDiff, (diff) => diff.merged_version); + // const mergedJson = serializeToJson(mergedFields); + // const conflicts = getFromDiff(fieldsDiff, (diff) => diff.has_conflict); + // const hasConflict = conflicts.some((item) => item.value); + + return { + base_version: baseJson, + current_version: currentJson, + target_version: targetJson, + merged_version: mergedJson, + + diff_outcome: diffOutcome, + merge_outcome: hasConflict + ? ThreeWayMergeOutcome.MergedWithConflict + : ThreeWayMergeOutcome.Merged, + + has_value_changed: hasValueChanged, + has_conflict: hasConflict, + }; +}; + +const getFromDiff = ( + fieldsDiff: RuleFieldsDiff, + getter: (diff: ThreeWayDiff) => TValue +): Array<{ fieldName: string; value: TValue }> => { + return Object.entries(fieldsDiff).map(([fieldName, fieldDiff]) => { + const value = getter(fieldDiff as ThreeWayDiff); + return { fieldName, value }; + }); +}; + +const serializeToJson = (fields: Array<{ fieldName: string; value: unknown }>): string => { + const sortedFields = sortBy(fields, (f) => f.fieldName); + const objectWithFields = sortedFields.reduce>((obj, item) => { + obj[item.fieldName] = item.value; + return obj; + }, {}); + + return JSON.stringify(objectWithFields, null, 2); +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculation/diff_calculation_helpers.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculation/diff_calculation_helpers.ts new file mode 100644 index 0000000000000..476e0f4e8d98e --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculation/diff_calculation_helpers.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mapValues } from 'lodash'; +import type { FieldsDiff, FieldsDiffAlgorithmsFor } from '../../diff_model/fields_diff'; +import type { ThreeVersionsOf } from '../../diff_model/three_way_diff'; + +export const calculateFieldsDiffFor = ( + objectVersions: ThreeVersionsOf, + fieldsDiffAlgorithms: FieldsDiffAlgorithmsFor +): FieldsDiff => { + const result = mapValues(fieldsDiffAlgorithms, (calculateFieldDiff, fieldName) => { + const fieldVersions = pickField(fieldName as keyof TObject, objectVersions); + const fieldDiff = calculateFieldDiff(fieldVersions); + return fieldDiff; + }); + + // TODO: try to improve strict typing and get rid of this "as" operator. + return result as FieldsDiff; +}; + +const pickField = ( + fieldName: keyof TObject, + versions: ThreeVersionsOf +): ThreeVersionsOf => { + return { + base_version: versions.base_version[fieldName], + current_version: versions.current_version[fieldName], + target_version: versions.target_version[fieldName], + }; +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/convert_rule_to_diffable.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/convert_rule_to_diffable.ts new file mode 100644 index 0000000000000..416cb8a891aa2 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/convert_rule_to_diffable.ts @@ -0,0 +1,264 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DEFAULT_MAX_SIGNALS } from '../../../../../constants'; +import { assertUnreachable } from '../../../../../utility_types'; +import type { + EqlRule, + EqlRuleCreateProps, + MachineLearningRule, + MachineLearningRuleCreateProps, + NewTermsRule, + NewTermsRuleCreateProps, + QueryRule, + QueryRuleCreateProps, + RuleResponse, + SavedQueryRule, + SavedQueryRuleCreateProps, + ThreatMatchRule, + ThreatMatchRuleCreateProps, + ThresholdRule, + ThresholdRuleCreateProps, +} from '../../../../rule_schema'; +import type { PrebuiltRuleContent } from '../../content_model/prebuilt_rule_content'; +import type { SemanticVersion } from '../../content_model/semantic_version'; +import { convertLegacyVersionToSemantic } from '../../content_model/semantic_version'; +import type { + DiffableCommonFields, + DiffableCustomQueryFields, + DiffableEqlFields, + DiffableMachineLearningFields, + DiffableNewTermsFields, + DiffableRule, + DiffableSavedQueryFields, + DiffableThreatMatchFields, + DiffableThresholdFields, +} from '../../diffable_rule_model/diffable_rule'; +import { extractBuildingBlockObject } from './extract_building_block_object'; +import { + extractInlineKqlQuery, + extractRuleEqlQuery, + extractRuleKqlQuery, +} from './extract_rule_data_query'; +import { extractRuleDataSource } from './extract_rule_data_source'; +import { extractRuleNameOverrideObject } from './extract_rule_name_override_object'; +import { extractRuleSchedule } from './extract_rule_schedule'; +import { extractTimelineTemplateReference } from './extract_timeline_template_reference'; +import { extractTimestampOverrideObject } from './extract_timestamp_override_object'; + +/** + * Normalizes a given rule to the form which is suitable for passing to the diff algorithm. + * Read more in the JSDoc description of DiffableRule. + */ +export const convertRuleToDiffable = (rule: RuleResponse | PrebuiltRuleContent): DiffableRule => { + const commonFields = extractDiffableCommonFields(rule); + + switch (rule.type) { + case 'query': + return { + ...commonFields, + ...extractDiffableCustomQueryFields(rule), + }; + case 'saved_query': + return { + ...commonFields, + ...extractDiffableSavedQueryFieldsFromRuleObject(rule), + }; + case 'eql': + return { + ...commonFields, + ...extractDiffableEqlFieldsFromRuleObject(rule), + }; + case 'threat_match': + return { + ...commonFields, + ...extractDiffableThreatMatchFieldsFromRuleObject(rule), + }; + case 'threshold': + return { + ...commonFields, + ...extractDiffableThresholdFieldsFromRuleObject(rule), + }; + case 'machine_learning': + return { + ...commonFields, + ...extractDiffableMachineLearningFieldsFromRuleObject(rule), + }; + case 'new_terms': + return { + ...commonFields, + ...extractDiffableNewTermsFieldsFromRuleObject(rule), + }; + default: + return assertUnreachable(rule, 'Unhandled rule type'); + } +}; + +const extractDiffableCommonFields = ( + rule: RuleResponse | PrebuiltRuleContent +): DiffableCommonFields => { + return { + // --------------------- REQUIRED FIELDS + // Technical fields + rule_id: rule.rule_id, + rule_content_version: extractDiffableRuleContentVersion(rule), + stack_version_min: extractDiffableStackVersionMin(rule), + stack_version_max: extractDiffableStackVersionMax(rule), + meta: rule.meta ?? {}, + + // Main domain fields + name: rule.name, + tags: rule.tags ?? [], + description: rule.description, + severity: rule.severity, + severity_mapping: rule.severity_mapping ?? [], + risk_score: rule.risk_score, + risk_score_mapping: rule.risk_score_mapping ?? [], + + // About -> Advanced settings + references: rule.references ?? [], + false_positives: rule.false_positives ?? [], + threat: rule.threat ?? [], + note: rule.note ?? '', + setup: rule.setup ?? '', + related_integrations: rule.related_integrations ?? [], + required_fields: rule.required_fields ?? [], + author: rule.author ?? [], + license: rule.license ?? '', + + // Other domain fields + rule_schedule: extractRuleSchedule(rule), + actions: rule.actions ?? [], + throttle: rule.throttle ?? 'no_actions', + exceptions_list: rule.exceptions_list ?? [], + max_signals: rule.max_signals ?? DEFAULT_MAX_SIGNALS, + + // --------------------- OPTIONAL FIELDS + rule_name_override: extractRuleNameOverrideObject(rule), + timestamp_override: extractTimestampOverrideObject(rule), + timeline_template: extractTimelineTemplateReference(rule), + building_block: extractBuildingBlockObject(rule), + }; +}; + +const extractDiffableRuleContentVersion = ( + rule: RuleResponse | PrebuiltRuleContent +): SemanticVersion => { + if ('rule_content_version' in rule) { + return rule.rule_content_version; + } + if ('version' in rule) { + return convertLegacyVersionToSemantic(rule.version); + } + return ''; +}; + +const extractDiffableStackVersionMin = ( + rule: RuleResponse | PrebuiltRuleContent +): SemanticVersion => { + if ('stack_version_min' in rule) { + return rule.stack_version_min; + } + return ''; +}; + +const extractDiffableStackVersionMax = ( + rule: RuleResponse | PrebuiltRuleContent +): SemanticVersion => { + if ('stack_version_max' in rule) { + return rule.stack_version_max; + } + return ''; +}; + +const extractDiffableCustomQueryFields = ( + rule: QueryRule | QueryRuleCreateProps +): DiffableCustomQueryFields => { + return { + type: rule.type, + data_query: extractRuleKqlQuery(rule.query, rule.language, rule.filters, rule.saved_id), + data_source: extractRuleDataSource(rule.index, rule.data_view_id), + alert_suppression: rule.alert_suppression, + }; +}; + +const extractDiffableSavedQueryFieldsFromRuleObject = ( + rule: SavedQueryRule | SavedQueryRuleCreateProps +): DiffableSavedQueryFields => { + return { + type: rule.type, + data_query: extractRuleKqlQuery(rule.query, rule.language, rule.filters, rule.saved_id), + data_source: extractRuleDataSource(rule.index, rule.data_view_id), + alert_suppression: rule.alert_suppression, + }; +}; + +const extractDiffableEqlFieldsFromRuleObject = ( + rule: EqlRule | EqlRuleCreateProps +): DiffableEqlFields => { + return { + type: rule.type, + data_query: extractRuleEqlQuery(rule.query, rule.language, rule.filters), + data_source: extractRuleDataSource(rule.index, rule.data_view_id), + event_category_override: rule.event_category_override, + timestamp_field: rule.timestamp_field, + tiebreaker_field: rule.tiebreaker_field, + }; +}; + +const extractDiffableThreatMatchFieldsFromRuleObject = ( + rule: ThreatMatchRule | ThreatMatchRuleCreateProps +): DiffableThreatMatchFields => { + return { + type: rule.type, + data_query: extractRuleKqlQuery(rule.query, rule.language, rule.filters, rule.saved_id), + data_source: extractRuleDataSource(rule.index, rule.data_view_id), + threat_query: extractInlineKqlQuery( + rule.threat_query, + rule.threat_language, + rule.threat_filters + ), + threat_index: rule.threat_index, + threat_mapping: rule.threat_mapping, + threat_indicator_path: rule.threat_indicator_path, + concurrent_searches: rule.concurrent_searches, + items_per_search: rule.items_per_search, + }; +}; + +const extractDiffableThresholdFieldsFromRuleObject = ( + rule: ThresholdRule | ThresholdRuleCreateProps +): DiffableThresholdFields => { + return { + type: rule.type, + data_query: extractRuleKqlQuery(rule.query, rule.language, rule.filters, rule.saved_id), + data_source: extractRuleDataSource(rule.index, rule.data_view_id), + threshold: rule.threshold, + }; +}; + +const extractDiffableMachineLearningFieldsFromRuleObject = ( + rule: MachineLearningRule | MachineLearningRuleCreateProps +): DiffableMachineLearningFields => { + return { + type: rule.type, + machine_learning_job_id: rule.machine_learning_job_id, + anomaly_threshold: rule.anomaly_threshold, + }; +}; + +const extractDiffableNewTermsFieldsFromRuleObject = ( + rule: NewTermsRule | NewTermsRuleCreateProps +): DiffableNewTermsFields => { + return { + type: rule.type, + data_query: extractInlineKqlQuery(rule.query, rule.language, rule.filters), + data_source: extractRuleDataSource(rule.index, rule.data_view_id), + new_terms_fields: rule.new_terms_fields, + history_window_start: rule.history_window_start, + }; +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_building_block_object.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_building_block_object.ts new file mode 100644 index 0000000000000..9d684c8b7398b --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_building_block_object.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleResponse } from '../../../../rule_schema'; +import type { PrebuiltRuleContent } from '../../content_model/prebuilt_rule_content'; +import type { BuildingBlockObject } from '../../diffable_rule_model/diffable_field_types'; + +export const extractBuildingBlockObject = ( + rule: RuleResponse | PrebuiltRuleContent +): BuildingBlockObject | undefined => { + if (rule.building_block_type == null) { + return undefined; + } + return { + type: rule.building_block_type, + }; +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_rule_data_query.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_rule_data_query.ts new file mode 100644 index 0000000000000..24d61e1c753a4 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_rule_data_query.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + EqlQueryLanguage, + KqlQueryLanguage, + RuleFilterArray, + RuleQuery, +} from '../../../../rule_schema'; +import type { + InlineKqlQuery, + RuleEqlQuery, + RuleKqlQuery, +} from '../../diffable_rule_model/diffable_field_types'; +import { KqlQueryType } from '../../diffable_rule_model/diffable_field_types'; + +export const extractRuleKqlQuery = ( + query: RuleQuery | undefined, + language: KqlQueryLanguage | undefined, + filters: RuleFilterArray | undefined, + savedQueryId: string | undefined +): RuleKqlQuery => { + if (savedQueryId != null) { + return { + type: KqlQueryType.saved_query, + saved_query_id: savedQueryId, + }; + } else { + return extractInlineKqlQuery(query, language, filters); + } +}; + +export const extractInlineKqlQuery = ( + query: RuleQuery | undefined, + language: KqlQueryLanguage | undefined, + filters: RuleFilterArray | undefined +): InlineKqlQuery => { + return { + type: KqlQueryType.inline_query, + query: query ?? '', + language: language ?? 'kuery', + filters: filters ?? [], + }; +}; + +export const extractRuleEqlQuery = ( + query: RuleQuery, + language: EqlQueryLanguage, + filters: RuleFilterArray | undefined +): RuleEqlQuery => { + return { + query, + language, + filters: filters ?? [], + }; +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_rule_data_source.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_rule_data_source.ts new file mode 100644 index 0000000000000..96e207c28957b --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_rule_data_source.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DataViewId, IndexPatternArray } from '../../../../rule_schema'; +import type { RuleDataSource } from '../../diffable_rule_model/diffable_field_types'; +import { DataSourceType } from '../../diffable_rule_model/diffable_field_types'; + +export const extractRuleDataSource = ( + indexPatterns: IndexPatternArray | undefined, + dataViewId: DataViewId | undefined +): RuleDataSource | undefined => { + if (indexPatterns != null) { + return { + type: DataSourceType.index_patterns, + index_patterns: indexPatterns, + }; + } + + if (dataViewId != null) { + return { + type: DataSourceType.data_view, + data_view_id: dataViewId, + }; + } + + return undefined; +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_rule_name_override_object.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_rule_name_override_object.ts new file mode 100644 index 0000000000000..3f5a2a7559e66 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_rule_name_override_object.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleResponse } from '../../../../rule_schema'; +import type { PrebuiltRuleContent } from '../../content_model/prebuilt_rule_content'; +import type { RuleNameOverrideObject } from '../../diffable_rule_model/diffable_field_types'; + +export const extractRuleNameOverrideObject = ( + rule: RuleResponse | PrebuiltRuleContent +): RuleNameOverrideObject | undefined => { + if (rule.rule_name_override == null) { + return undefined; + } + return { + field_name: rule.rule_name_override, + }; +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_rule_schedule.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_rule_schedule.ts new file mode 100644 index 0000000000000..cc34b28509ac2 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_rule_schedule.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import dateMath from '@elastic/datemath'; +import { parseDuration } from '@kbn/alerting-plugin/common'; + +import type { RuleResponse } from '../../../../rule_schema'; +import type { PrebuiltRuleContent } from '../../content_model/prebuilt_rule_content'; +import type { RuleSchedule } from '../../diffable_rule_model/diffable_field_types'; + +export const extractRuleSchedule = (rule: RuleResponse | PrebuiltRuleContent): RuleSchedule => { + const interval = rule.interval ?? '5m'; + const from = rule.from ?? 'now-6m'; + const to = rule.to ?? 'now'; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const ruleMeta = (rule.meta ?? {}) as any; + const lookbackFromMeta = String(ruleMeta.from ?? ''); + + const intervalDuration = parseInterval(interval); + const lookbackFromMetaDuration = parseInterval(lookbackFromMeta); + const driftToleranceDuration = parseDriftTolerance(from, to); + + if (lookbackFromMetaDuration != null) { + if (intervalDuration != null) { + return { + interval, + lookback: lookbackFromMeta, + }; + } + return { + interval: `Cannot parse: interval="${interval}"`, + lookback: lookbackFromMeta, + }; + } + + if (intervalDuration == null) { + return { + interval: `Cannot parse: interval="${interval}"`, + lookback: `Cannot calculate due to invalid interval`, + }; + } + + if (driftToleranceDuration == null) { + return { + interval, + lookback: `Cannot parse: from="${from}", to="${to}"`, + }; + } + + const lookbackDuration = moment.duration().add(driftToleranceDuration).subtract(intervalDuration); + const lookback = `${lookbackDuration.asSeconds()}s`; + + return { interval, lookback }; +}; + +const parseInterval = (intervalString: string): moment.Duration | null => { + try { + const milliseconds = parseDuration(intervalString); + return moment.duration(milliseconds); + } catch (e) { + return null; + } +}; + +const parseDriftTolerance = (from: string, to: string): moment.Duration | null => { + const now = new Date(); + const fromDate = parseDateMathString(from, now); + const toDate = parseDateMathString(to, now); + + if (fromDate == null || toDate == null) { + return null; + } + + return moment.duration(toDate.diff(fromDate)); +}; + +const parseDateMathString = (dateMathString: string, now: Date): moment.Moment | null => { + const parsedDate = dateMath.parse(dateMathString, { forceNow: now }); + return parsedDate != null && parsedDate.isValid() ? parsedDate : null; +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_timeline_template_reference.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_timeline_template_reference.ts new file mode 100644 index 0000000000000..12fe23b20b9ed --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_timeline_template_reference.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleResponse } from '../../../../rule_schema'; +import type { PrebuiltRuleContent } from '../../content_model/prebuilt_rule_content'; +import type { TimelineTemplateReference } from '../../diffable_rule_model/diffable_field_types'; + +export const extractTimelineTemplateReference = ( + rule: RuleResponse | PrebuiltRuleContent +): TimelineTemplateReference | undefined => { + if (rule.timeline_id == null) { + return undefined; + } + return { + timeline_id: rule.timeline_id, + timeline_title: rule.timeline_title ?? '', + }; +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_timestamp_override_object.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_timestamp_override_object.ts new file mode 100644 index 0000000000000..02e25ae1b1277 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/extract_timestamp_override_object.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleResponse } from '../../../../rule_schema'; +import type { PrebuiltRuleContent } from '../../content_model/prebuilt_rule_content'; +import type { TimestampOverrideObject } from '../../diffable_rule_model/diffable_field_types'; + +export const extractTimestampOverrideObject = ( + rule: RuleResponse | PrebuiltRuleContent +): TimestampOverrideObject | undefined => { + if (rule.timestamp_override == null) { + return undefined; + } + return { + field_name: rule.timestamp_override, + fallback_disabled: rule.timestamp_override_fallback_disabled ?? false, + }; +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_model/fields_diff.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_model/fields_diff.ts new file mode 100644 index 0000000000000..125f58a1fe8e1 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_model/fields_diff.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ThreeWayDiff, ThreeWayDiffAlgorithm } from './three_way_diff'; + +export type FieldsDiff = { + [Field in keyof TObject]: ThreeWayDiff; +}; + +export type FieldsDiffAlgorithmsFor = { + [Field in keyof TObject]: ThreeWayDiffAlgorithm; +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_model/rule_diff.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_model/rule_diff.ts new file mode 100644 index 0000000000000..c5a9312ea7fcd --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_model/rule_diff.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + DiffableCommonFields, + DiffableCustomQueryFields, + DiffableEqlFields, + DiffableMachineLearningFields, + DiffableNewTermsFields, + DiffableSavedQueryFields, + DiffableThreatMatchFields, + DiffableThresholdFields, +} from '../diffable_rule_model/diffable_rule'; + +import type { FieldsDiff } from './fields_diff'; +import type { ThreeWayDiff } from './three_way_diff'; + +export type CommonFieldsDiff = FieldsDiff; +export type CustomQueryFieldsDiff = FieldsDiff; +export type SavedQueryFieldsDiff = FieldsDiff; +export type EqlFieldsDiff = FieldsDiff; +export type ThreatMatchFieldsDiff = FieldsDiff; +export type ThresholdFieldsDiff = FieldsDiff; +export type MachineLearningFieldsDiff = FieldsDiff; +export type NewTermsFieldsDiff = FieldsDiff; + +/** + * It's an object which keys are the same as keys of DiffableRule, but values are + * three-way diffs calculated for their values. + * { + * name: ThreeWayDiff; + * tags: ThreeWayDiff; + * etc + * } + */ +export type RuleFieldsDiff = CommonFieldsDiff & + ( + | CustomQueryFieldsDiff + | SavedQueryFieldsDiff + | EqlFieldsDiff + | ThreatMatchFieldsDiff + | ThresholdFieldsDiff + | MachineLearningFieldsDiff + | NewTermsFieldsDiff + ); + +/** + * Three-way diff calculated for rule formatted as JSON text. + */ +export type RuleJsonDiff = ThreeWayDiff; + +export interface RuleDiff { + fields: RuleFieldsDiff; + json: RuleJsonDiff; + has_conflict: boolean; +} diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_model/three_way_diff.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_model/three_way_diff.ts new file mode 100644 index 0000000000000..22f267fb3b441 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_model/three_way_diff.ts @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ThreeWayDiffOutcome } from './three_way_diff_outcome'; +import type { ThreeWayMergeOutcome } from './three_way_merge_outcome'; + +/** + * Three versions of a value to pass to a diff algorithm. + */ +export interface ThreeVersionsOf { + /** + * Corresponds to the stock version of the currently installed prebuilt rule. + */ + base_version: TValue; + + /** + * Corresponds exactly to the currently installed prebuilt rule: + * - to the customized version (if it's customized) + * - to the stock version (if it's not customized) + */ + current_version: TValue; + + /** + * Corresponds to the "new" stock version that the user is trying to upgrade to. + */ + target_version: TValue; +} + +/** + * Represents a result of an abstract three-way diff/merge operation on a value + * (could be a whole rule JSON or a given rule field). + * + * Typical situations: + * + * 1. base=A, current=A, target=A => merged=A, conflict=false + * Stock rule, the value hasn't changed. + * + * 2. base=A, current=A, target=B => merged=B, conflict=false + * Stock rule, the value has changed. + * + * 3. base=A, current=B, target=A => merged=B, conflict=false + * Customized rule, the value hasn't changed. + * + * 4. base=A, current=B, target=B => merged=B, conflict=false + * Customized rule, the value has changed exactly the same way as in the user customization. + * + * 5. base=A, current=B, target=C => merged=D, conflict=false + * Customized rule, the value has changed, conflict between B and C resolved automatically. + * + * 6. base=A, current=B, target=C => merged=C, conflict=true + * Customized rule, the value has changed, conflict between B and C couldn't be resolved automatically. + */ +export interface ThreeWayDiff extends ThreeVersionsOf { + /** + * The result of an automatic three-way merge of three values: + * - base version + * - current version + * - target version + * + * Exact merge algorithm depends on the value: + * - one algo could be used for single-line strings and keywords (e.g. rule name) + * - another one could be used for multiline text (e.g. rule description) + * - another one could be used for arrays of keywords (e.g. rule tags) + * - another one could be used for the MITRE ATT&CK data structure + * - etc + * + * Merged version always has a value. We do our best to resolve conflicts automatically. + * If they can't be resolved automatically, merged version is equal to target version. + */ + merged_version: TValue; + + /** + * Tells which combination corresponds to the three input versions of the value for this specific diff. + */ + diff_outcome: ThreeWayDiffOutcome; + + /** + * The type of result of an automatic three-way merge of three values: + * - base version + * - current version + * - target version + */ + merge_outcome: ThreeWayMergeOutcome; + + /** + * Tells if the value has changed in the target version and the current version could be updated. + * True if: + * - base=A, current=A, target=B + * - base=A, current=B, target=C + */ + has_value_changed: boolean; + + /** + * True if: + * - current != target and we couldn't automatically resolve the conflict between them + * + * False if: + * - current == target (value won't change) + * - current != target && current == base (stock rule will get a new value) + * - current != target and we automatically resolved the conflict between them + */ + has_conflict: boolean; +} + +/** + * Given the three versions of a value, calculates a three-way diff for it. + */ +export type ThreeWayDiffAlgorithm = ( + versions: ThreeVersionsOf +) => ThreeWayDiff; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_model/three_way_diff_outcome.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_model/three_way_diff_outcome.ts new file mode 100644 index 0000000000000..db4d3d3a1f982 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_model/three_way_diff_outcome.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isEqual } from 'lodash'; + +/** + * Result of comparing three versions of a value against each other. + * Defines 5 typical combinations of 3 versions of a value. + */ +export enum ThreeWayDiffOutcome { + /** Stock rule, the value hasn't changed in the target version. */ + StockValueNoUpdate = 'BASE=A, CURRENT=A, TARGET=A', + + /** Stock rule, the value has changed in the target version. */ + StockValueCanUpdate = 'BASE=A, CURRENT=A, TARGET=B', + + /** Customized rule, the value hasn't changed in the target version comparing to the base one. */ + CustomizedValueNoUpdate = 'BASE=A, CURRENT=B, TARGET=A', + + /** Customized rule, the value has changed in the target version exactly the same way as in the user customization. */ + CustomizedValueSameUpdate = 'BASE=A, CURRENT=B, TARGET=B', + + /** Customized rule, the value has changed in the target version and is not equal to the current version. */ + CustomizedValueCanUpdate = 'BASE=A, CURRENT=B, TARGET=C', +} + +export const determineDiffOutcome = ( + baseVersion: TValue, + currentVersion: TValue, + targetVersion: TValue +): ThreeWayDiffOutcome => { + const baseEqlCurrent = isEqual(baseVersion, currentVersion); + const baseEqlTarget = isEqual(baseVersion, targetVersion); + const currentEqlTarget = isEqual(currentVersion, targetVersion); + + if (baseEqlCurrent) { + return currentEqlTarget + ? ThreeWayDiffOutcome.StockValueNoUpdate + : ThreeWayDiffOutcome.StockValueCanUpdate; + } + + if (baseEqlTarget) { + return ThreeWayDiffOutcome.CustomizedValueNoUpdate; + } + + return currentEqlTarget + ? ThreeWayDiffOutcome.CustomizedValueSameUpdate + : ThreeWayDiffOutcome.CustomizedValueCanUpdate; +}; + +export const determineIfValueChanged = (diffCase: ThreeWayDiffOutcome): boolean => { + return ( + diffCase === ThreeWayDiffOutcome.StockValueCanUpdate || + diffCase === ThreeWayDiffOutcome.CustomizedValueCanUpdate + ); +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_model/three_way_merge_outcome.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_model/three_way_merge_outcome.ts new file mode 100644 index 0000000000000..f3fbd56e1c430 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diff_model/three_way_merge_outcome.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Type of result of an automatic three-way merge of three values: + * - base version + * - current version + * - target version + */ +export enum ThreeWayMergeOutcome { + /** Took current version and returned as the merged one. */ + Current = 'CURRENT', + + /** Took target version and returned as the merged one. */ + Target = 'TARGET', + + /** Merged three versions successfully into a new one. */ + Merged = 'MERGED', + + /** Merged three versions with a conflict. */ + MergedWithConflict = 'MERGED_WITH_CONFLICT', +} diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diffable_rule_model/build_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diffable_rule_model/build_schema.ts new file mode 100644 index 0000000000000..ebbacc458143d --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diffable_rule_model/build_schema.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; +import { orUndefined } from '../../../rule_schema/model/build_rule_schemas'; + +interface RuleFields { + required: TRequired; + optional: TOptional; +} + +export const buildSchema = ( + fields: RuleFields +) => { + return t.intersection([ + t.exact(t.type(fields.required)), + t.exact(t.type(orUndefined(fields.optional))), + ]); +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diffable_rule_model/diffable_field_types.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diffable_rule_model/diffable_field_types.ts new file mode 100644 index 0000000000000..b0693d16cda4e --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diffable_rule_model/diffable_field_types.ts @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; +import { TimeDuration } from '@kbn/securitysolution-io-ts-types'; +import { + BuildingBlockType, + DataViewId, + IndexPatternArray, + KqlQueryLanguage, + RuleFilterArray, + RuleNameOverride as RuleNameOverrideFieldName, + RuleQuery, + TimelineTemplateId, + TimelineTemplateTitle, + TimestampOverride as TimestampOverrideFieldName, + TimestampOverrideFallbackDisabled, +} from '../../../rule_schema'; +import { saved_id } from '../../../schemas/common'; + +// ------------------------------------------------------------------------------------------------- +// Rule data source + +export enum DataSourceType { + 'index_patterns' = 'index_patterns', + 'data_view' = 'data_view', +} + +export type DataSourceIndexPatterns = t.TypeOf; +export const DataSourceIndexPatterns = t.exact( + t.type({ + type: t.literal(DataSourceType.index_patterns), + index_patterns: IndexPatternArray, + }) +); + +export type DataSourceDataView = t.TypeOf; +export const DataSourceDataView = t.exact( + t.type({ + type: t.literal(DataSourceType.data_view), + data_view_id: DataViewId, + }) +); + +export type RuleDataSource = t.TypeOf; +export const RuleDataSource = t.union([DataSourceIndexPatterns, DataSourceDataView]); + +// ------------------------------------------------------------------------------------------------- +// Rule data query + +export enum KqlQueryType { + 'inline_query' = 'inline_query', + 'saved_query' = 'saved_query', +} + +export type InlineKqlQuery = t.TypeOf; +export const InlineKqlQuery = t.exact( + t.type({ + type: t.literal(KqlQueryType.inline_query), + query: RuleQuery, + language: KqlQueryLanguage, + filters: RuleFilterArray, + }) +); + +export type SavedKqlQuery = t.TypeOf; +export const SavedKqlQuery = t.exact( + t.type({ + type: t.literal(KqlQueryType.saved_query), + saved_query_id: saved_id, + }) +); + +export type RuleKqlQuery = t.TypeOf; +export const RuleKqlQuery = t.union([InlineKqlQuery, SavedKqlQuery]); + +export type RuleEqlQuery = t.TypeOf; +export const RuleEqlQuery = t.exact( + t.type({ + query: RuleQuery, + language: t.literal('eql'), + filters: RuleFilterArray, + }) +); + +// ------------------------------------------------------------------------------------------------- +// Rule schedule + +export type RuleSchedule = t.TypeOf; +export const RuleSchedule = t.exact( + t.type({ + interval: TimeDuration({ allowedUnits: ['s', 'm', 'h'] }), + lookback: TimeDuration({ allowedUnits: ['s', 'm', 'h'] }), + }) +); + +// ------------------------------------------------------------------------------------------------- +// Rule name override + +export type RuleNameOverrideObject = t.TypeOf; +export const RuleNameOverrideObject = t.exact( + t.type({ + field_name: RuleNameOverrideFieldName, + }) +); + +// ------------------------------------------------------------------------------------------------- +// Timestamp override + +export type TimestampOverrideObject = t.TypeOf; +export const TimestampOverrideObject = t.exact( + t.type({ + field_name: TimestampOverrideFieldName, + fallback_disabled: TimestampOverrideFallbackDisabled, + }) +); + +// ------------------------------------------------------------------------------------------------- +// Reference to a timeline template + +export type TimelineTemplateReference = t.TypeOf; +export const TimelineTemplateReference = t.exact( + t.type({ + timeline_id: TimelineTemplateId, + timeline_title: TimelineTemplateTitle, + }) +); + +// ------------------------------------------------------------------------------------------------- +// Building block + +export type BuildingBlockObject = t.TypeOf; +export const BuildingBlockObject = t.exact( + t.type({ + type: BuildingBlockType, + }) +); diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diffable_rule_model/diffable_rule.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diffable_rule_model/diffable_rule.ts new file mode 100644 index 0000000000000..66077506927e3 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diffable_rule_model/diffable_rule.ts @@ -0,0 +1,244 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; + +import { + concurrent_searches, + items_per_search, + machine_learning_job_id, + RiskScore, + RiskScoreMapping, + RuleActionArray, + RuleActionThrottle, + Severity, + SeverityMapping, + threat_index, + threat_indicator_path, + threat_mapping, +} from '@kbn/securitysolution-io-ts-alerting-types'; + +import { + AlertSuppression, + EventCategoryOverride, + ExceptionListArray, + HistoryWindowStart, + InvestigationGuide, + MaxSignals, + NewTermsFields, + RelatedIntegrationArray, + RequiredFieldArray, + RuleAuthorArray, + RuleDescription, + RuleFalsePositiveArray, + RuleLicense, + RuleMetadata, + RuleName, + RuleReferenceArray, + RuleSignatureId, + RuleTagArray, + SetupGuide, + ThreatArray, + Threshold, + TiebreakerField, + TimestampField, +} from '../../../rule_schema'; + +import { anomaly_threshold } from '../../../schemas/common'; + +import { SemanticVersion } from '../content_model/semantic_version'; + +import { + BuildingBlockObject, + RuleEqlQuery, + InlineKqlQuery, + RuleKqlQuery, + RuleDataSource, + RuleNameOverrideObject, + RuleSchedule, + TimelineTemplateReference, + TimestampOverrideObject, +} from './diffable_field_types'; + +import { buildSchema } from './build_schema'; + +export type DiffableCommonFields = t.TypeOf; +export const DiffableCommonFields = buildSchema({ + required: { + // Technical fields + // NOTE: We might consider removing them from the schema and returning from the API + // not via the fields diff, but via dedicated properties in the response body. + rule_id: RuleSignatureId, + rule_content_version: SemanticVersion, + stack_version_min: SemanticVersion, + stack_version_max: SemanticVersion, + meta: RuleMetadata, + + // Main domain fields + name: RuleName, + tags: RuleTagArray, + description: RuleDescription, + severity: Severity, + severity_mapping: SeverityMapping, + risk_score: RiskScore, + risk_score_mapping: RiskScoreMapping, + + // About -> Advanced settings + references: RuleReferenceArray, + false_positives: RuleFalsePositiveArray, + threat: ThreatArray, + note: InvestigationGuide, + setup: SetupGuide, + related_integrations: RelatedIntegrationArray, + required_fields: RequiredFieldArray, + author: RuleAuthorArray, + license: RuleLicense, + + // Other domain fields + rule_schedule: RuleSchedule, // NOTE: new field + actions: RuleActionArray, + throttle: RuleActionThrottle, + exceptions_list: ExceptionListArray, + max_signals: MaxSignals, + }, + optional: { + rule_name_override: RuleNameOverrideObject, // NOTE: new field + timestamp_override: TimestampOverrideObject, // NOTE: new field + timeline_template: TimelineTemplateReference, // NOTE: new field + building_block: BuildingBlockObject, // NOTE: new field + }, +}); + +export type DiffableCustomQueryFields = t.TypeOf; +export const DiffableCustomQueryFields = buildSchema({ + required: { + type: t.literal('query'), + data_query: RuleKqlQuery, // NOTE: new field + }, + optional: { + data_source: RuleDataSource, // NOTE: new field + alert_suppression: AlertSuppression, + }, +}); + +export type DiffableSavedQueryFields = t.TypeOf; +export const DiffableSavedQueryFields = buildSchema({ + required: { + type: t.literal('saved_query'), + data_query: RuleKqlQuery, // NOTE: new field + }, + optional: { + data_source: RuleDataSource, // NOTE: new field + alert_suppression: AlertSuppression, + }, +}); + +export type DiffableEqlFields = t.TypeOf; +export const DiffableEqlFields = buildSchema({ + required: { + type: t.literal('eql'), + data_query: RuleEqlQuery, // NOTE: new field + }, + optional: { + data_source: RuleDataSource, // NOTE: new field + event_category_override: EventCategoryOverride, + timestamp_field: TimestampField, + tiebreaker_field: TiebreakerField, + }, +}); + +export type DiffableThreatMatchFields = t.TypeOf; +export const DiffableThreatMatchFields = buildSchema({ + required: { + type: t.literal('threat_match'), + data_query: RuleKqlQuery, // NOTE: new field + threat_query: InlineKqlQuery, // NOTE: new field + threat_index, + threat_mapping, + }, + optional: { + data_source: RuleDataSource, // NOTE: new field + threat_indicator_path, + concurrent_searches, // Should combine concurrent_searches and items_per_search? + items_per_search, + }, +}); + +export type DiffableThresholdFields = t.TypeOf; +export const DiffableThresholdFields = buildSchema({ + required: { + type: t.literal('threshold'), + data_query: RuleKqlQuery, // NOTE: new field + threshold: Threshold, + }, + optional: { + data_source: RuleDataSource, // NOTE: new field + }, +}); + +export type DiffableMachineLearningFields = t.TypeOf; +export const DiffableMachineLearningFields = buildSchema({ + required: { + type: t.literal('machine_learning'), + machine_learning_job_id, + anomaly_threshold, + }, + optional: {}, +}); + +export type DiffableNewTermsFields = t.TypeOf; +export const DiffableNewTermsFields = buildSchema({ + required: { + type: t.literal('new_terms'), + data_query: InlineKqlQuery, // NOTE: new field + new_terms_fields: NewTermsFields, + history_window_start: HistoryWindowStart, + }, + optional: { + data_source: RuleDataSource, // NOTE: new field + }, +}); + +/** + * Represents a normalized rule object that is suitable for passing to the diff algorithm. + * Every top-level field of a diffable rule can be compared separately on its own. + * + * It's important to do such normalization because: + * + * 1. We need to compare installed rules with prebuilt rule content. These objects have similar but not exactly + * the same interfaces. In order to compare them we need to convert them to a common interface. + * + * 2. It only makes sense to compare certain rule fields in combination with other fields. For example, + * we combine `index` and `data_view_id` fields into a `RuleDataSource` object, so that later we could + * calculate a diff for this whole object. If we don't combine them the app would successfully merge the + * following values independently from each other without a conflict: + * + * Base version: index=[logs-*], data_view_id=undefined + * Current version: index=[], data_view_id=some-data-view // user switched to a data view + * Target version: index=[logs-*, filebeat-*], data_view_id=undefined // Elastic added a new index pattern + * Merged version: index=[filebeat-*], data_view_id=some-data-view ??? + * + * Instead, semantically such change represents a conflict because the data source of the rule was changed + * in a potentially incompatible way, and the user might want to review the change and resolve it manually. + * The user must either pick index patterns or a data view, but not both at the same time. + * + * NOTE: Every top-level field in a DiffableRule MUST BE LOGICALLY INDEPENDENT from other + * top-level fields. + */ +export type DiffableRule = t.TypeOf; +export const DiffableRule = t.intersection([ + DiffableCommonFields, + t.union([ + DiffableCustomQueryFields, + DiffableSavedQueryFields, + DiffableEqlFields, + DiffableThreatMatchFields, + DiffableThresholdFields, + DiffableMachineLearningFields, + DiffableNewTermsFields, + ]), +]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diffable_rule_model/rule_fields.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diffable_rule_model/rule_fields.ts new file mode 100644 index 0000000000000..a9102e43aa56c --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/poc/diffable_rule_model/rule_fields.ts @@ -0,0 +1,393 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PrebuiltRuleToInstall } from '../../model/prebuilt_rule'; +import type { RuleResponse } from '../../../rule_schema'; + +/* eslint-disable @typescript-eslint/naming-convention */ + +const fieldsOfPrebuiltRuleToInstall = () => { + const rule = {} as unknown as PrebuiltRuleToInstall; + + // --------------------------------------------------------------------------- + // Common fields + const { + // ------------------------------------------------------------------------- + // Required fields + rule_id, // ✅ + version, // ✅ + // type, // ✅ + name, // ✅ + description, // ✅ + risk_score, // ✅ + severity, // ✅ + + // ------------------------------------------------------------------------- + // Optional fields + related_integrations, // ✅ + required_fields, // ✅ + setup, // ✅ + // Field overrides + rule_name_override, // ✅ + timestamp_override, // ✅ + timestamp_override_fallback_disabled, // ✅ + // Timeline template + timeline_id, // ✅ + timeline_title, // ✅ + // Attributes related to SavedObjectsClient.resolve API + outcome, // ✅ + alias_target_id, // ✅ + alias_purpose, // ✅ + // Misc attributes + license, // ✅ + note, // ✅ + building_block_type, // ✅ + output_index, // ✅ + namespace, // ✅ + meta, // ✅ + + // ------------------------------------------------------------------------- + // Defaultable fields + // Main attributes + tags, // ✅ + enabled, // ✅ + // Field overrides + risk_score_mapping, // ✅ + severity_mapping, // ✅ + // Rule schedule + interval, // ✅ + from, // ✅ + to, // ✅ + // Rule actions + actions, // ✅ + throttle, // ✅ + // Rule exceptions + exceptions_list, // ✅ + // Misc attributes + author, // ✅ + false_positives, // ✅ + references, // ✅ + // maxSignals not used in ML rules but probably should be used + max_signals, // ✅ + threat, // ✅ + + ...ruleWithoutAllCommonFields + } = rule; + + // --------------------------------------------------------------------------- + // Custom Query fields + if (ruleWithoutAllCommonFields.type === 'query') { + const { + type, // ✅ + index, // ✅ + data_view_id, // ✅ + query, // ✅ + language, // ✅ + filters, // ✅ + saved_id, // ✅ + response_actions, // ✅ + alert_suppression, // ✅ + ...rest_____________________ + } = ruleWithoutAllCommonFields; + } + + // --------------------------------------------------------------------------- + // Saved Query fields + if (ruleWithoutAllCommonFields.type === 'saved_query') { + const { + type, // ✅ + index, // ✅ + data_view_id, // ✅ + query, // ✅ + language, // ✅ + filters, // ✅ + saved_id, // ✅ + response_actions, // ✅ + alert_suppression, // ✅ + ...rest_____________________ + } = ruleWithoutAllCommonFields; + } + + // --------------------------------------------------------------------------- + // EQL fields + if (ruleWithoutAllCommonFields.type === 'eql') { + const { + type, // ✅ + index, // ✅ + data_view_id, // ✅ + query, // ✅ + language, // ✅ + filters, // ✅ + event_category_override, // ✅ + timestamp_field, // ✅ + tiebreaker_field, // ✅ + ...rest_____________________ + } = ruleWithoutAllCommonFields; + } + + // --------------------------------------------------------------------------- + // Indicator Match fields + if (ruleWithoutAllCommonFields.type === 'threat_match') { + const { + type, // ✅ + index, // ✅ + data_view_id, // ✅ + query, // ✅ + language, // ✅ + filters, // ✅ + saved_id, // ✅ + threat_query, // ✅ + threat_mapping, // ✅ + threat_index, // ✅ + threat_filters, // ✅ + threat_indicator_path, // ✅ + threat_language, // ✅ + concurrent_searches, // ✅ + items_per_search, // ✅ + ...rest_____________________ + } = ruleWithoutAllCommonFields; + } + + // --------------------------------------------------------------------------- + // Threshold fields + if (ruleWithoutAllCommonFields.type === 'threshold') { + const { + type, // ✅ + index, // ✅ + data_view_id, // ✅ + query, // ✅ + language, // ✅ + filters, // ✅ + saved_id, // ✅ + threshold, // ✅ + ...rest_____________________ + } = ruleWithoutAllCommonFields; + } + + // --------------------------------------------------------------------------- + // Machine Learning fields + if (ruleWithoutAllCommonFields.type === 'machine_learning') { + const { + type, // ✅ + machine_learning_job_id, // ✅ + anomaly_threshold, // ✅ + ...rest__________________________________ + } = ruleWithoutAllCommonFields; + } + + // --------------------------------------------------------------------------- + // New Terms fields + if (ruleWithoutAllCommonFields.type === 'new_terms') { + const { + type, // ✅ + index, // ✅ + data_view_id, // ✅ + query, // ✅ + language, // ✅ + filters, // ✅ + new_terms_fields, // ✅ + history_window_start, // ✅ + ...rest_____________________ + } = ruleWithoutAllCommonFields; + } +}; + +const fieldsOfRuleResponse = () => { + const rule = {} as unknown as RuleResponse; + + // --------------------------------------------------------------------------- + // Common fields + const { + // ------------------------------------------------------------------------- + // Response required and optional fields + id, // ✅ + rule_id, // ✅ + created_at, // ✅ + created_by, // ✅ + updated_at, // ✅ + updated_by, // ✅ + immutable, // ✅ + related_integrations, // ✅ + required_fields, // ✅ + setup, // ✅ + execution_summary, // ✅ + + // ------------------------------------------------------------------------- + // Required fields + // type, // ✅ + name, // ✅ + description, // ✅ + risk_score, // ✅ + severity, // ✅ + + // ------------------------------------------------------------------------- + // Optional fields + // Field overrides + rule_name_override, // ✅ + timestamp_override, // ✅ + timestamp_override_fallback_disabled, // ✅ + // Timeline template + timeline_id, // ✅ + timeline_title, // ✅ + // Attributes related to SavedObjectsClient.resolve API + outcome, // ✅ + alias_target_id, // ✅ + alias_purpose, // ✅ + // Misc attributes + license, // ✅ + note, // ✅ + building_block_type, // ✅ + output_index, // ✅ + namespace, // ✅ + meta, // ✅ + + // ------------------------------------------------------------------------- + // Defaultable fields + // Main attributes + version, // ✅ + tags, // ✅ + enabled, // ✅ + // Field overrides + risk_score_mapping, // ✅ + severity_mapping, // ✅ + // Rule schedule + interval, // ✅ + from, // ✅ + to, // ✅ + // Rule actions + actions, // ✅ + throttle, // ✅ + // Rule exceptions + exceptions_list, // ✅ + // Misc attributes + author, // ✅ + false_positives, // ✅ + references, // ✅ + // maxSignals not used in ML rules but probably should be used + max_signals, // ✅ + threat, // ✅ + + ...ruleWithoutAllCommonFields + } = rule; + + // --------------------------------------------------------------------------- + // Custom Query fields + if (ruleWithoutAllCommonFields.type === 'query') { + const { + type, // ✅ + index, // ✅ + data_view_id, // ✅ + query, // ✅ + language, // ✅ + filters, // ✅ + saved_id, // ✅ + response_actions, // ✅ + alert_suppression, // ✅ + ...rest_____________________ + } = ruleWithoutAllCommonFields; + } + + // --------------------------------------------------------------------------- + // Saved Query fields + if (ruleWithoutAllCommonFields.type === 'saved_query') { + const { + type, // ✅ + index, // ✅ + data_view_id, // ✅ + query, // ✅ + language, // ✅ + filters, // ✅ + saved_id, // ✅ + response_actions, // ✅ + alert_suppression, // ✅ + ...rest_____________________ + } = ruleWithoutAllCommonFields; + } + + // --------------------------------------------------------------------------- + // EQL fields + if (ruleWithoutAllCommonFields.type === 'eql') { + const { + type, // ✅ + index, // ✅ + data_view_id, // ✅ + query, // ✅ + language, // ✅ + filters, // ✅ + event_category_override, // ✅ + timestamp_field, // ✅ + tiebreaker_field, // ✅ + ...rest_____________________ + } = ruleWithoutAllCommonFields; + } + + // --------------------------------------------------------------------------- + // Indicator Match fields + if (ruleWithoutAllCommonFields.type === 'threat_match') { + const { + type, // ✅ + index, // ✅ + data_view_id, // ✅ + query, // ✅ + language, // ✅ + filters, // ✅ + saved_id, // ✅ + threat_query, // ✅ + threat_mapping, // ✅ + threat_index, // ✅ + threat_filters, // ✅ + threat_indicator_path, // ✅ + threat_language, // ✅ + concurrent_searches, // ✅ + items_per_search, // ✅ + ...rest_____________________ + } = ruleWithoutAllCommonFields; + } + + // --------------------------------------------------------------------------- + // Threshold fields + if (ruleWithoutAllCommonFields.type === 'threshold') { + const { + type, // ✅ + index, // ✅ + data_view_id, // ✅ + query, // ✅ + language, // ✅ + filters, // ✅ + saved_id, // ✅ + threshold, // ✅ + ...rest_____________________ + } = ruleWithoutAllCommonFields; + } + + // --------------------------------------------------------------------------- + // Machine Learning fields + if (ruleWithoutAllCommonFields.type === 'machine_learning') { + const { + type, // ✅ + machine_learning_job_id, // ✅ + anomaly_threshold, // ✅ + ...rest__________________________________ + } = ruleWithoutAllCommonFields; + } + + // --------------------------------------------------------------------------- + // New Terms fields + if (ruleWithoutAllCommonFields.type === 'new_terms') { + const { + type, // ✅ + index, // ✅ + data_view_id, // ✅ + query, // ✅ + language, // ✅ + filters, // ✅ + new_terms_fields, // ✅ + history_window_start, // ✅ + ...rest_____________________ + } = ruleWithoutAllCommonFields; + } +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/build_rule_schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/build_rule_schemas.ts index f7d52c682d191..c77ba322b1c79 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/build_rule_schemas.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/build_rule_schemas.ts @@ -21,6 +21,7 @@ export const buildRuleSchemas = ) => { return { + ...fields, create: buildCreateRuleSchema(fields.required, fields.optional, fields.defaultable), patch: buildPatchRuleSchema(fields.required, fields.optional, fields.defaultable), response: buildResponseRuleSchema(fields.required, fields.optional, fields.defaultable), @@ -59,10 +60,17 @@ const buildPatchRuleSchema = < ]); }; -type OrUndefined

= { +export type OrUndefined

= { [K in keyof P]: P[K] | t.UndefinedC; }; +export const orUndefined =

(props: P): OrUndefined

=> { + return Object.keys(props).reduce((acc, key) => { + acc[key] = t.union([props[key], t.undefined]); + return acc; + }, {}) as OrUndefined

; +}; + export const buildResponseRuleSchema = < Required extends t.Props, Optional extends t.Props, @@ -78,10 +86,7 @@ export const buildResponseRuleSchema = < // the conversion from internal schema to response schema TS will report an error. If we just used t.partial // instead, then optional fields can be accidentally omitted from the conversion - and any actual values // in those fields internally will be stripped in the response. - const optionalWithUndefined = Object.keys(optionalFields).reduce((acc, key) => { - acc[key] = t.union([optionalFields[key], t.undefined]); - return acc; - }, {}) as OrUndefined; + const optionalWithUndefined = orUndefined(optionalFields); return t.intersection([ t.exact(t.type(requiredFields)), t.exact(t.type(optionalWithUndefined)), diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/common_attributes/misc_attributes.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/common_attributes/misc_attributes.ts index 315a15190ec28..c4992da7f2702 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/common_attributes/misc_attributes.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/common_attributes/misc_attributes.ts @@ -47,7 +47,7 @@ export type RuleMetadata = t.TypeOf; export const RuleMetadata = t.object; // should be a more specific type? export type RuleLicense = t.TypeOf; -export const RuleLicense = t.string; // should be non-empty string? +export const RuleLicense = t.string; export type RuleAuthorArray = t.TypeOf; export const RuleAuthorArray = t.array(t.string); // should be non-empty strings? diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/rule_schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/rule_schemas.ts index 5d35811368b39..636c5ee5f6fa8 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/rule_schemas.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/rule_schemas.ts @@ -92,7 +92,7 @@ import { buildRuleSchemas } from './build_rule_schemas'; // ------------------------------------------------------------------------------------------------- // Base schema -const baseSchema = buildRuleSchemas({ +export const baseSchema = buildRuleSchemas({ required: { name: RuleName, description: RuleDescription, @@ -207,10 +207,22 @@ export const SharedResponseProps = t.intersection([ // ------------------------------------------------------------------------------------------------- // EQL rule schema +export enum QueryLanguage { + 'kuery' = 'kuery', + 'lucene' = 'lucene', + 'eql' = 'eql', +} + +export type KqlQueryLanguage = t.TypeOf; +export const KqlQueryLanguage = t.keyof({ kuery: null, lucene: null }); + +export type EqlQueryLanguage = t.TypeOf; +export const EqlQueryLanguage = t.literal('eql'); + const eqlSchema = buildRuleSchemas({ required: { type: t.literal('eql'), - language: t.literal('eql'), + language: EqlQueryLanguage, query: RuleQuery, }, optional: { @@ -257,12 +269,12 @@ const threatMatchSchema = buildRuleSchemas({ saved_id, threat_filters, threat_indicator_path, - threat_language: t.keyof({ kuery: null, lucene: null }), + threat_language: KqlQueryLanguage, concurrent_searches, items_per_search, }, defaultable: { - language: t.keyof({ kuery: null, lucene: null }), + language: KqlQueryLanguage, }, }); @@ -307,7 +319,7 @@ const querySchema = buildRuleSchemas({ }, defaultable: { query: RuleQuery, - language: t.keyof({ kuery: null, lucene: null }), + language: KqlQueryLanguage, }, }); @@ -345,7 +357,7 @@ const savedQuerySchema = buildRuleSchemas({ alert_suppression: AlertSuppression, }, defaultable: { - language: t.keyof({ kuery: null, lucene: null }), + language: KqlQueryLanguage, }, }); @@ -386,7 +398,7 @@ const thresholdSchema = buildRuleSchemas({ saved_id, }, defaultable: { - language: t.keyof({ kuery: null, lucene: null }), + language: KqlQueryLanguage, }, }); @@ -461,7 +473,7 @@ const newTermsSchema = buildRuleSchemas({ filters: RuleFilterArray, }, defaultable: { - language: t.keyof({ kuery: null, lucene: null }), + language: KqlQueryLanguage, }, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/get_prebuilt_rules_status/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/get_prebuilt_rules_status/route.ts new file mode 100644 index 0000000000000..a604282018898 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/get_prebuilt_rules_status/route.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import { transformError } from '@kbn/securitysolution-es-utils'; + +import { GET_PREBUILT_RULES_STATUS_URL } from '../../../../../../common/detection_engine/prebuilt_rules'; +import { GetPrebuiltRulesStatusRequestQuery } from '../../../../../../common/detection_engine/prebuilt_rules/poc/api/get_prebuilt_rules_status/request_schema'; +import type { + GetPrebuiltRulesStatusResponseBody, + PrebuiltRulesStatusStats, +} from '../../../../../../common/detection_engine/prebuilt_rules/poc/api/get_prebuilt_rules_status/response_schema'; + +import type { SecuritySolutionPluginRouter } from '../../../../../types'; +import { buildRouteValidation } from '../../../../../utils/build_validation/route_validation'; +import { buildSiemResponse } from '../../../routes/utils'; + +import { createCompositeRuleAssetsClient } from '../../logic/poc/saved_objects/rule_asset_composite_saved_objects_client'; +import { createComposite2RuleAssetsClient } from '../../logic/poc/saved_objects/rule_asset_composite2_saved_objects_client'; +import { createFlatRuleAssetsClient } from '../../logic/poc/saved_objects/rule_asset_flat_saved_objects_client'; +import { createPrebuiltRuleContentClient } from '../../logic/poc/prebuilt_rule_content_client'; +import { createPrebuiltRuleObjectsClient } from '../../logic/poc/prebuilt_rule_objects_client'; +import type { VersionBuckets } from '../../logic/poc/get_versions_to_install_and_upgrade'; +import { getVersionBuckets } from '../../logic/poc/get_versions_to_install_and_upgrade'; + +export const getPrebuiltRulesStatusRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.get( + { + path: GET_PREBUILT_RULES_STATUS_URL, + validate: { + query: buildRouteValidation(GetPrebuiltRulesStatusRequestQuery), + }, + options: { + tags: ['access:securitySolution'], + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + + try { + const ctx = await context.resolve(['core', 'alerting']); + const soClient = ctx.core.savedObjects.client; + const rulesClient = ctx.alerting.getRulesClient(); + const flatClient = createFlatRuleAssetsClient(soClient); + const compositeClient = createCompositeRuleAssetsClient(soClient); + const composite2Client = createComposite2RuleAssetsClient(soClient); + const ruleContentClient = createPrebuiltRuleContentClient( + flatClient, + compositeClient, + composite2Client + ); + const ruleObjectsClient = createPrebuiltRuleObjectsClient(rulesClient, soClient, logger); + + const [latestVersions, installedVersions] = await Promise.all([ + ruleContentClient.fetchLatestVersions(request.query.data_model), + ruleObjectsClient.fetchInstalledVersions(), + ]); + + const versionBuckets = getVersionBuckets({ + latestVersions, + installedVersions, + }); + + const stats = calculateRuleStats(versionBuckets); + + const body: GetPrebuiltRulesStatusResponseBody = { + status_code: 200, + message: 'OK', + attributes: { + stats, + }, + }; + + return response.ok({ body }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; + +const calculateRuleStats = (buckets: VersionBuckets): PrebuiltRulesStatusStats => { + const { latestVersions, installedVersions, latestVersionsToInstall, installedVersionsToUpgrade } = + buckets; + + return { + num_prebuilt_rules_total: latestVersions.length, + num_prebuilt_rules_installed: installedVersions.length, + num_prebuilt_rules_to_install: latestVersionsToInstall.length, + num_prebuilt_rules_to_upgrade: installedVersionsToUpgrade.length, + rule_ids_to_install: latestVersionsToInstall.map((r) => r.rule_id), + rule_ids_to_upgrade: installedVersionsToUpgrade.map((r) => r.rule_id), + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/install_test_assets/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/install_test_assets/route.ts new file mode 100644 index 0000000000000..c935e89fadf5e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/install_test_assets/route.ts @@ -0,0 +1,206 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; +import moment from 'moment'; +import { transformError } from '@kbn/securitysolution-es-utils'; +import { PositiveIntegerGreaterThanZero } from '@kbn/securitysolution-io-ts-types'; + +import type { PrebuiltRuleToInstall } from '../../../../../../common/detection_engine/prebuilt_rules'; +import { INSTALL_TEST_ASSETS_URL } from '../../../../../../common/detection_engine/prebuilt_rules'; +import type { PrebuiltRuleContent } from '../../../../../../common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_content'; +import { getSemanticVersion } from '../../../../../../common/detection_engine/prebuilt_rules/poc/content_model/semantic_version'; + +import type { SecuritySolutionPluginRouter } from '../../../../../types'; +import { buildRouteValidation } from '../../../../../utils/build_validation/route_validation'; +import { buildSiemResponse } from '../../../routes/utils'; + +import { ruleAssetSavedObjectsClientFactory } from '../../logic/rule_asset/rule_asset_saved_objects_client'; +import { getLatestPrebuiltRules } from '../../logic/get_latest_prebuilt_rules'; +import type { RuleAssetFlatAttributes } from '../../logic/poc/saved_objects/rule_asset_flat_saved_objects_type'; +import type { RuleAssetCompositeAttributes } from '../../logic/poc/saved_objects/rule_asset_composite_saved_objects_type'; +import { createCompositeRuleAssetsClient } from '../../logic/poc/saved_objects/rule_asset_composite_saved_objects_client'; +import { createComposite2RuleAssetsClient } from '../../logic/poc/saved_objects/rule_asset_composite2_saved_objects_client'; +import { createFlatRuleAssetsClient } from '../../logic/poc/saved_objects/rule_asset_flat_saved_objects_client'; +import type { + RuleAssetComposite2Attributes, + RuleVersionInfo, +} from '../../logic/poc/saved_objects/rule_asset_composite2_saved_objects_type'; + +type RequestBody = t.TypeOf; +const RequestBody = t.exact( + t.type({ + num_versions_per_rule: PositiveIntegerGreaterThanZero, + }) +); + +export const installTestPrebuiltRuleAssetsRoute = (router: SecuritySolutionPluginRouter) => { + router.post( + { + path: INSTALL_TEST_ASSETS_URL, + validate: { + body: buildRouteValidation(RequestBody), + }, + options: { + tags: ['access:securitySolution'], + timeout: { + // FUNFACT: If we do not add a very long timeout what will happen + // is that Chrome which receive a 408 error and then do a retry. + // This retry can cause lots of connections to happen. Using a very + // long timeout will ensure that Chrome does not do retries and saturate the connections. + idleSocket: moment.duration('1', 'hour').asMilliseconds(), + }, + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + + try { + const ctx = await context.resolve(['core']); + const soClient = ctx.core.savedObjects.client; + const ruleAssetsClient = ruleAssetSavedObjectsClientFactory(soClient); + const flatRuleAssetsClient = createFlatRuleAssetsClient(soClient); + const compositeRuleAssetsClient = createCompositeRuleAssetsClient(soClient); + const composite2RuleAssetsClient = createComposite2RuleAssetsClient(soClient); + + const filesystemRules = await getLatestPrebuiltRules(ruleAssetsClient); + + const versionedRuleAssets = generateVersionedRuleAssets( + Array.from(filesystemRules.values()), + request.body.num_versions_per_rule + ); + + await flatRuleAssetsClient.bulkDeleteAll(); + await flatRuleAssetsClient.bulkCreate(versionedRuleAssets.flat); + + await compositeRuleAssetsClient.bulkDeleteAll(); + await compositeRuleAssetsClient.bulkCreate(versionedRuleAssets.composite); + + await composite2RuleAssetsClient.bulkDeleteAll(); + await composite2RuleAssetsClient.bulkCreate(versionedRuleAssets.composite2); + + return response.ok({ + body: { + num_flat_assets: versionedRuleAssets.flat.length, + num_composite_assets: versionedRuleAssets.composite.length, + }, + }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; + +const generateVersionedRuleAssets = ( + rules: PrebuiltRuleToInstall[], + numberOfVersionsPerRule: number +) => { + const flat: RuleAssetFlatAttributes[] = []; + const composite: RuleAssetCompositeAttributes[] = []; + const composite2: RuleAssetComposite2Attributes[] = []; + + rules.forEach((rule) => { + flat.push(...generateFlatRuleAssets(rule, numberOfVersionsPerRule)); + composite.push(generateCompositeRuleAsset(rule, numberOfVersionsPerRule)); + composite2.push(generateComposite2RuleAsset(rule, numberOfVersionsPerRule)); + }); + + return { flat, composite, composite2 }; +}; + +const generateFlatRuleAssets = ( + rule: PrebuiltRuleToInstall, + numberOfVersionsPerRule: number +): RuleAssetFlatAttributes[] => { + const { name: ruleName, rule_id: ruleId, version: ruleVersion, ...restOfRuleAttributes } = rule; + const result: RuleAssetFlatAttributes[] = []; + + for (let i = 0; i < numberOfVersionsPerRule; i++) { + const semanticVersion = getSemanticVersion(ruleVersion, i); + result.push({ + name: `${ruleName} v${semanticVersion}`, + rule_id: ruleId, + rule_content_version: semanticVersion, + stack_version_min: '8.5.0', + stack_version_max: '8.7.0', + ...restOfRuleAttributes, + }); + } + + return result; +}; + +const generateCompositeRuleAsset = ( + rule: PrebuiltRuleToInstall, + numberOfVersionsPerRule: number +): RuleAssetCompositeAttributes => { + const { name: ruleName, rule_id: ruleId, version: ruleVersion, ...restOfRuleAttributes } = rule; + const result: RuleAssetCompositeAttributes = { + rule_id: ruleId, + versions: [], + }; + + for (let i = 0; i < numberOfVersionsPerRule; i++) { + const semanticVersion = getSemanticVersion(ruleVersion, i); + result.versions.push({ + name: `${ruleName} v${semanticVersion}`, + rule_content_version: semanticVersion, + stack_version_min: '8.5.0', + stack_version_max: '8.7.0', + ...restOfRuleAttributes, + }); + } + + return result; +}; + +const generateComposite2RuleAsset = ( + rule: PrebuiltRuleToInstall, + numberOfVersionsPerRule: number +): RuleAssetComposite2Attributes => { + const { name: ruleName, rule_id: ruleId, version: ruleVersion, ...restOfRuleAttributes } = rule; + const result: RuleAssetComposite2Attributes = { + rule_id: ruleId, + versions: [], + content: {}, + }; + + for (let i = 0; i < numberOfVersionsPerRule; i++) { + const semanticVersion = getSemanticVersion(ruleVersion, i); + const versionKey = getVersionKey(semanticVersion); + const contentKey = getContentKey(ruleId, versionKey); + + const versionInfo: RuleVersionInfo = { + rule_content_version: semanticVersion, + stack_version_min: '8.5.0', + stack_version_max: '8.7.0', + }; + + const content: PrebuiltRuleContent = { + name: `${ruleName} v${semanticVersion}`, + rule_id: ruleId, + rule_content_version: semanticVersion, + stack_version_min: '8.5.0', + stack_version_max: '8.7.0', + ...restOfRuleAttributes, + }; + + result.versions.push(versionInfo); + result.content[contentKey] = content; + } + + return result; +}; + +const getVersionKey = (version: string): string => version.replaceAll('.', '_'); + +const getContentKey = (ruleId: string, versionKey: string): string => `${ruleId}__v${versionKey}`; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/register_routes.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/register_routes.ts index 39e822af3e147..08f7c117be6d3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/register_routes.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/register_routes.ts @@ -5,18 +5,33 @@ * 2.0. */ +import type { Logger } from '@kbn/core/server'; import type { ConfigType } from '../../../../config'; import type { SetupPlugins } from '../../../../plugin_contract'; import type { SecuritySolutionPluginRouter } from '../../../../types'; import { getPrebuiltRulesAndTimelinesStatusRoute } from './get_prebuilt_rules_and_timelines_status/route'; +import { getPrebuiltRulesStatusRoute } from './get_prebuilt_rules_status/route'; import { installPrebuiltRulesAndTimelinesRoute } from './install_prebuilt_rules_and_timelines/route'; +import { installTestPrebuiltRuleAssetsRoute } from './install_test_assets/route'; +import { reviewRuleInstallationRoute } from './review_rule_installation/route'; +import { reviewRuleUpgradeRoute } from './review_rule_upgrade/route'; export const registerPrebuiltRulesRoutes = ( router: SecuritySolutionPluginRouter, config: ConfigType, + logger: Logger, security: SetupPlugins['security'] ) => { + // Legacy endpoints that we're going to deprecate getPrebuiltRulesAndTimelinesStatusRoute(router, config, security); installPrebuiltRulesAndTimelinesRoute(router); + + // Endpoint for testing the rule upgrade and installation workflows + installTestPrebuiltRuleAssetsRoute(router); + + // New endpoints for the rule upgrade and installation workflows + getPrebuiltRulesStatusRoute(router, logger); + reviewRuleInstallationRoute(router, logger); + reviewRuleUpgradeRoute(router, logger); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_installation/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_installation/route.ts new file mode 100644 index 0000000000000..414441534341a --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_installation/route.ts @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import { transformError } from '@kbn/securitysolution-es-utils'; + +import { REVIEW_RULE_INSTALLATION_URL } from '../../../../../../common/detection_engine/prebuilt_rules'; +import { ReviewRuleInstallationRequestBody } from '../../../../../../common/detection_engine/prebuilt_rules/poc/api/review_rule_installation/request_schema'; +import type { + ReviewRuleInstallationResponseBody, + RuleInstallationInfoForReview, + RuleInstallationStatsForReview, +} from '../../../../../../common/detection_engine/prebuilt_rules/poc/api/review_rule_installation/response_schema'; +import type { PrebuiltRuleContent } from '../../../../../../common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_content'; +import { convertRuleToDiffable } from '../../../../../../common/detection_engine/prebuilt_rules/poc/diff_algorithm/normalization/convert_rule_to_diffable'; + +import type { SecuritySolutionPluginRouter } from '../../../../../types'; +import { buildRouteValidation } from '../../../../../utils/build_validation/route_validation'; +import { buildSiemResponse } from '../../../routes/utils'; +import { getVersionBuckets } from '../../logic/poc/get_versions_to_install_and_upgrade'; + +import { createPrebuiltRuleContentClient } from '../../logic/poc/prebuilt_rule_content_client'; +import { createPrebuiltRuleObjectsClient } from '../../logic/poc/prebuilt_rule_objects_client'; +import { createComposite2RuleAssetsClient } from '../../logic/poc/saved_objects/rule_asset_composite2_saved_objects_client'; +import { createCompositeRuleAssetsClient } from '../../logic/poc/saved_objects/rule_asset_composite_saved_objects_client'; +import { createFlatRuleAssetsClient } from '../../logic/poc/saved_objects/rule_asset_flat_saved_objects_client'; + +export const reviewRuleInstallationRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.post( + { + path: REVIEW_RULE_INSTALLATION_URL, + validate: { + body: buildRouteValidation(ReviewRuleInstallationRequestBody), + }, + options: { + tags: ['access:securitySolution'], + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + + try { + const ctx = await context.resolve(['core', 'alerting']); + const soClient = ctx.core.savedObjects.client; + const rulesClient = ctx.alerting.getRulesClient(); + const flatClient = createFlatRuleAssetsClient(soClient); + const compositeClient = createCompositeRuleAssetsClient(soClient); + const composite2Client = createComposite2RuleAssetsClient(soClient); + const ruleContentClient = createPrebuiltRuleContentClient( + flatClient, + compositeClient, + composite2Client + ); + const ruleObjectsClient = createPrebuiltRuleObjectsClient(rulesClient, soClient, logger); + + const [latestVersions, installedVersions] = await Promise.all([ + ruleContentClient.fetchLatestVersions(request.body.data_model), + ruleObjectsClient.fetchInstalledVersions(), + ]); + + const versionBuckets = getVersionBuckets({ + latestVersions, + installedVersions, + }); + + const rulesToInstall = await ruleContentClient.fetchRulesByVersionInfo( + request.body.data_model, + versionBuckets.latestVersionsToInstall + ); + + const body: ReviewRuleInstallationResponseBody = { + status_code: 200, + message: 'OK', + attributes: { + stats: calculateRuleStats(rulesToInstall), + rules: calculateRuleInfos(rulesToInstall), + }, + }; + + return response.ok({ body }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; + +const getAggregatedTags = (rules: PrebuiltRuleContent[]): string[] => { + const set = new Set(); + + rules.forEach((rule) => { + (rule.tags ?? []).forEach((tag) => { + set.add(tag); + }); + }); + + return Array.from(set.values()); +}; + +const calculateRuleStats = ( + rulesToInstall: PrebuiltRuleContent[] +): RuleInstallationStatsForReview => { + const tagsOfRulesToInstall = getAggregatedTags(rulesToInstall); + return { + num_rules_to_install: rulesToInstall.length, + tags: tagsOfRulesToInstall, + }; +}; + +const calculateRuleInfos = ( + rulesToInstall: PrebuiltRuleContent[] +): RuleInstallationInfoForReview[] => { + return rulesToInstall.map((rule) => convertRuleToDiffable(rule)); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/route.ts new file mode 100644 index 0000000000000..c786559f7d636 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/route.ts @@ -0,0 +1,179 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { pickBy } from 'lodash'; +import type { Logger } from '@kbn/core/server'; +import { transformError } from '@kbn/securitysolution-es-utils'; + +import { REVIEW_RULE_UPGRADE_URL } from '../../../../../../common/detection_engine/prebuilt_rules'; +import { ReviewRuleUpgradeRequestBody } from '../../../../../../common/detection_engine/prebuilt_rules/poc/api/review_rule_upgrade/request_schema'; +import type { + ReviewRuleUpgradeResponseBody, + RuleUpgradeInfoForReview, + RuleUpgradeStatsForReview, +} from '../../../../../../common/detection_engine/prebuilt_rules/poc/api/review_rule_upgrade/response_schema'; +import type { PrebuiltRuleContent } from '../../../../../../common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_content'; +import type { PrebuiltRuleVersionInfo } from '../../../../../../common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_version_info'; +import type { + CalculateRuleDiffArgs, + CalculateRuleDiffResult, +} from '../../../../../../common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculate_rule_diff'; +import { calculateRuleDiff } from '../../../../../../common/detection_engine/prebuilt_rules/poc/diff_algorithm/calculate_rule_diff'; +import type { ThreeWayDiff } from '../../../../../../common/detection_engine/prebuilt_rules/poc/diff_model/three_way_diff'; +import type { RuleResponse } from '../../../../../../common/detection_engine/rule_schema'; + +import type { SecuritySolutionPluginRouter } from '../../../../../types'; +import { buildRouteValidation } from '../../../../../utils/build_validation/route_validation'; +import { buildSiemResponse } from '../../../routes/utils'; + +import { createPrebuiltRuleContentClient } from '../../logic/poc/prebuilt_rule_content_client'; +import { createPrebuiltRuleObjectsClient } from '../../logic/poc/prebuilt_rule_objects_client'; +import { createCompositeRuleAssetsClient } from '../../logic/poc/saved_objects/rule_asset_composite_saved_objects_client'; +import { createComposite2RuleAssetsClient } from '../../logic/poc/saved_objects/rule_asset_composite2_saved_objects_client'; +import { createFlatRuleAssetsClient } from '../../logic/poc/saved_objects/rule_asset_flat_saved_objects_client'; +import { getVersionBuckets } from '../../logic/poc/get_versions_to_install_and_upgrade'; + +export const reviewRuleUpgradeRoute = (router: SecuritySolutionPluginRouter, logger: Logger) => { + router.post( + { + path: REVIEW_RULE_UPGRADE_URL, + validate: { + body: buildRouteValidation(ReviewRuleUpgradeRequestBody), + }, + options: { + tags: ['access:securitySolution'], + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + + try { + const ctx = await context.resolve(['core', 'alerting']); + const soClient = ctx.core.savedObjects.client; + const rulesClient = ctx.alerting.getRulesClient(); + const flatClient = createFlatRuleAssetsClient(soClient); + const compositeClient = createCompositeRuleAssetsClient(soClient); + const composite2Client = createComposite2RuleAssetsClient(soClient); + const ruleContentClient = createPrebuiltRuleContentClient( + flatClient, + compositeClient, + composite2Client + ); + const ruleObjectsClient = createPrebuiltRuleObjectsClient(rulesClient, soClient, logger); + + const [latestVersions, { installedVersions, installedRules }] = await Promise.all([ + ruleContentClient.fetchLatestVersions(request.body.data_model), + ruleObjectsClient.fetchInstalledRules(), + ]); + + const versionBuckets = getVersionBuckets({ + latestVersions, + installedVersions, + }); + + const [baseRules, latestRules] = await Promise.all([ + ruleContentClient.fetchRulesByVersionInfo( + request.body.data_model, + versionBuckets.installedVersionsToUpgrade + ), + ruleContentClient.fetchRulesByVersionInfo( + request.body.data_model, + versionBuckets.latestVersionsToUpgrade + ), + ]); + + const ruleDiffCalculationArgs = getRuleDiffCalculationArgs( + versionBuckets.installedVersionsToUpgrade, + installedRules, + baseRules, + latestRules + ); + const ruleDiffCalculationResults = ruleDiffCalculationArgs.map((args) => { + return calculateRuleDiff(args); + }); + + const body: ReviewRuleUpgradeResponseBody = { + status_code: 200, + message: 'OK', + attributes: { + stats: calculateRuleStats(ruleDiffCalculationResults), + rules: calculateRuleInfos(ruleDiffCalculationResults), + }, + }; + + return response.ok({ body }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; + +const getRuleDiffCalculationArgs = ( + installedVersionsToUpgrade: PrebuiltRuleVersionInfo[], + installedRules: RuleResponse[], + baseRules: PrebuiltRuleContent[], + latestRules: PrebuiltRuleContent[] +): CalculateRuleDiffArgs[] => { + const installedRulesMap = new Map(installedRules.map((r) => [r.rule_id, r])); + const baseRulesMap = new Map(baseRules.map((r) => [r.rule_id, r])); + const latestRulesMap = new Map(latestRules.map((r) => [r.rule_id, r])); + + const result: CalculateRuleDiffArgs[] = []; + + installedVersionsToUpgrade.forEach((versionToUpgrade) => { + const ruleId = versionToUpgrade.rule_id; + const installedRule = installedRulesMap.get(ruleId); + const baseRule = baseRulesMap.get(ruleId); + const latestRule = latestRulesMap.get(ruleId); + + if (installedRule != null && baseRule != null && latestRule != null) { + result.push({ + currentVersion: installedRule, + baseVersion: baseRule, + targetVersion: latestRule, + }); + } + }); + + return result; +}; + +const calculateRuleStats = (results: CalculateRuleDiffResult[]): RuleUpgradeStatsForReview => { + return { + num_rules_to_upgrade: results.length, + num_stock_rules_to_upgrade: results.length, + num_customized_rules_to_upgrade: 0, + tags: [], + fields: [], + }; +}; + +const calculateRuleInfos = (results: CalculateRuleDiffResult[]): RuleUpgradeInfoForReview[] => { + return results.map((result) => { + const { ruleDiff, ruleVersions } = result; + const installedCurrentVersion = ruleVersions.input.current; + const diffableCurrentVersion = ruleVersions.output.current; + + return { + id: installedCurrentVersion.id, + rule_id: installedCurrentVersion.rule_id, + rule: diffableCurrentVersion, + diff: { + fields: pickBy>( + ruleDiff.fields, + (fieldDiff) => fieldDiff.has_value_changed || fieldDiff.has_conflict + ), + has_conflict: ruleDiff.has_conflict, + }, + }; + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/get_versions_to_install_and_upgrade.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/get_versions_to_install_and_upgrade.ts new file mode 100644 index 0000000000000..0714ed1d75d4c --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/get_versions_to_install_and_upgrade.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import semver from 'semver'; +import type { PrebuiltRuleVersionInfo } from '../../../../../../common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_version_info'; + +export interface GetVersionBucketsArgs { + latestVersions: PrebuiltRuleVersionInfo[]; + installedVersions: PrebuiltRuleVersionInfo[]; +} + +export interface VersionBuckets { + latestVersions: PrebuiltRuleVersionInfo[]; + installedVersions: PrebuiltRuleVersionInfo[]; + latestVersionsToInstall: PrebuiltRuleVersionInfo[]; + latestVersionsToUpgrade: PrebuiltRuleVersionInfo[]; + installedVersionsToUpgrade: PrebuiltRuleVersionInfo[]; +} + +export const getVersionBuckets = (args: GetVersionBucketsArgs): VersionBuckets => { + const { latestVersions, installedVersions } = args; + + const installedVersionsMap = new Map(installedVersions.map((item) => [item.rule_id, item])); + + const latestVersionsToInstall: PrebuiltRuleVersionInfo[] = []; + const latestVersionsToUpgrade: PrebuiltRuleVersionInfo[] = []; + const installedVersionsToUpgrade: PrebuiltRuleVersionInfo[] = []; + + latestVersions.forEach((latestVersion) => { + const installedVersion = installedVersionsMap.get(latestVersion.rule_id); + + if (installedVersion == null) { + // If this rule is not installed + latestVersionsToInstall.push(latestVersion); + } + + if ( + installedVersion != null && + semver.lt(installedVersion.rule_content_version, latestVersion.rule_content_version) + ) { + // If this rule is installed but outdated + latestVersionsToUpgrade.push(latestVersion); + installedVersionsToUpgrade.push(installedVersion); + } + }); + + return { + latestVersions, + installedVersions, + latestVersionsToInstall, + latestVersionsToUpgrade, + installedVersionsToUpgrade, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/prebuilt_rule_content_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/prebuilt_rule_content_client.ts new file mode 100644 index 0000000000000..f9d1cd07fb934 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/prebuilt_rule_content_client.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { assertUnreachable } from '../../../../../../common/utility_types'; +import { withSecuritySpan } from '../../../../../utils/with_security_span'; +import type { PrebuiltRuleContentDataModel } from '../../../../../../common/detection_engine/prebuilt_rules/poc/api/get_prebuilt_rules_status/request_schema'; +import type { PrebuiltRuleContent } from '../../../../../../common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_content'; +import type { PrebuiltRuleVersionInfo } from '../../../../../../common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_version_info'; +import type { ICompositeRuleAssetsClient } from './saved_objects/rule_asset_composite_saved_objects_client'; +import type { IComposite2RuleAssetsClient } from './saved_objects/rule_asset_composite2_saved_objects_client'; +import type { IFlatRuleAssetsClient } from './saved_objects/rule_asset_flat_saved_objects_client'; + +export interface IPrebuiltRuleContentClient { + fetchLatestVersions(dataModel: PrebuiltRuleContentDataModel): Promise; + + fetchRulesByVersionInfo( + dataModel: PrebuiltRuleContentDataModel, + versions: PrebuiltRuleVersionInfo[] + ): Promise; +} + +export const createPrebuiltRuleContentClient = ( + flatClient: IFlatRuleAssetsClient, + compositeClient: ICompositeRuleAssetsClient, + composite2Client: IComposite2RuleAssetsClient +): IPrebuiltRuleContentClient => { + const pickClient = (dataModel: PrebuiltRuleContentDataModel) => { + switch (dataModel) { + case 'flat': + return flatClient; + case 'composite': + return compositeClient; + case 'composite2': + return composite2Client; + default: + return assertUnreachable(dataModel); + } + }; + + return { + fetchLatestVersions: ( + dataModel: PrebuiltRuleContentDataModel + ): Promise => { + return withSecuritySpan('IPrebuiltRuleContentClient.fetchLatestVersions', async () => { + const client = pickClient(dataModel); + const items = await client.fetchLatestVersions(); + return items; + }); + }, + + fetchRulesByVersionInfo: ( + dataModel: PrebuiltRuleContentDataModel, + versions: PrebuiltRuleVersionInfo[] + ): Promise => { + return withSecuritySpan('IPrebuiltRuleContentClient.fetchRulesByVersionInfo', async () => { + const client = pickClient(dataModel); + const items = await client.fetchRulesByIdAndVersion({ + rules: versions.map((v) => ({ id: v.rule_id, version: v.rule_content_version })), + }); + return items; + }); + }, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/prebuilt_rule_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/prebuilt_rule_objects_client.ts new file mode 100644 index 0000000000000..75c08d7c0306a --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/prebuilt_rule_objects_client.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract, Logger } from '@kbn/core/server'; +import type { RulesClient } from '@kbn/alerting-plugin/server'; + +import type { PrebuiltRuleVersionInfo } from '../../../../../../common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_version_info'; +import { convertLegacyVersionToSemantic } from '../../../../../../common/detection_engine/prebuilt_rules/poc/content_model/semantic_version'; +import type { RuleResponse } from '../../../../../../common/detection_engine/rule_schema'; + +import { withSecuritySpan } from '../../../../../utils/with_security_span'; +import { getExistingPrepackagedRules } from '../../../rule_management/logic/search/get_existing_prepackaged_rules'; +import { internalRuleToAPIResponse } from '../../../rule_management/normalization/rule_converters'; + +export interface IPrebuiltRuleObjectsClient { + fetchInstalledVersions(): Promise; + fetchInstalledRules(): Promise; +} + +export interface FetchInstalledRulesResult { + installedVersions: PrebuiltRuleVersionInfo[]; + installedRules: RuleResponse[]; +} + +export const createPrebuiltRuleObjectsClient = ( + rulesClient: RulesClient, + savedObjectsClient: SavedObjectsClientContract, + logger: Logger +): IPrebuiltRuleObjectsClient => { + const fetchInstalledVersions = (): Promise => { + return withSecuritySpan('IPrebuiltRuleObjectsClient.fetchInstalledVersions', async () => { + const rulesData = await getExistingPrepackagedRules({ rulesClient }); + const rules = rulesData.map((rule) => internalRuleToAPIResponse(rule)); + const versions = rules.map((rule) => convertRuleToVersionInfo(rule)); + return versions; + }); + }; + + const fetchInstalledRules = (): Promise => { + return withSecuritySpan('IPrebuiltRuleObjectsClient.fetchInstalledRules', async () => { + const rulesData = await getExistingPrepackagedRules({ rulesClient }); + const rules = rulesData.map((rule) => internalRuleToAPIResponse(rule)); + const versions = rules.map((rule) => convertRuleToVersionInfo(rule)); + return { + installedVersions: versions, + installedRules: rules, + }; + }); + }; + + return { + fetchInstalledVersions, + fetchInstalledRules, + }; +}; + +const convertRuleToVersionInfo = (rule: RuleResponse): PrebuiltRuleVersionInfo => { + const versionInfo: PrebuiltRuleVersionInfo = { + rule_id: rule.rule_id, + rule_content_version: convertLegacyVersionToSemantic(rule.version), + stack_version_min: '', + stack_version_max: '', + }; + return versionInfo; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_composite2_saved_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_composite2_saved_objects_client.ts new file mode 100644 index 0000000000000..763ee37e45587 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_composite2_saved_objects_client.ts @@ -0,0 +1,185 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { chunk } from 'lodash'; +import semver from 'semver'; + +import type { + SavedObjectsBulkCreateObject, + SavedObjectsBulkDeleteObject, + SavedObjectsClientContract, + SavedObjectsFindResult, +} from '@kbn/core/server'; + +import { withSecuritySpan } from '../../../../../../utils/with_security_span'; +import type { PrebuiltRuleContent } from '../../../../../../../common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_content'; +import type { PrebuiltRuleVersionInfo } from '../../../../../../../common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_version_info'; +import type { SemanticVersion } from '../../../../../../../common/detection_engine/prebuilt_rules/poc/content_model/semantic_version'; +import type { RuleSignatureId } from '../../../../../../../common/detection_engine/rule_schema'; +import { RULE_ASSET_COMPOSITE2_SO_TYPE } from './rule_asset_composite2_saved_objects_type'; +import type { RuleAssetComposite2Attributes } from './rule_asset_composite2_saved_objects_type'; + +const MAX_RULE_ASSETS_PER_REQUEST = 100; + +export interface IComposite2RuleAssetsClient { + bulkDeleteAll(): Promise; + + bulkCreate(rules: RuleAssetComposite2Attributes[]): Promise; + + fetchLatestVersions(): Promise; + + fetchRulesByIdAndVersion(args: FetchSpecificRulesArgs): Promise; +} + +export interface FetchSpecificRulesArgs { + rules: Array<{ id: RuleSignatureId; version: SemanticVersion }>; +} + +type FindSelector = (result: SavedObjectsFindResult) => T; + +export const createComposite2RuleAssetsClient = ( + savedObjectsClient: SavedObjectsClientContract +): IComposite2RuleAssetsClient => { + const fetchAll = async (selector: FindSelector) => { + const finder = savedObjectsClient.createPointInTimeFinder({ + perPage: MAX_RULE_ASSETS_PER_REQUEST, + type: RULE_ASSET_COMPOSITE2_SO_TYPE, + }); + + const result: T[] = []; + + for await (const response of finder.find()) { + const selectedValues = response.saved_objects.map((so) => selector(so)); + result.push(...selectedValues); + } + + await finder.close(); + + return result; + }; + + return { + bulkDeleteAll: (): Promise => { + return withSecuritySpan('IComposite2RuleAssetsClient.bulkDeleteAll', async () => { + const allIds = await fetchAll((so) => so.id); + const allObjects: SavedObjectsBulkDeleteObject[] = allIds.map((id) => { + return { type: RULE_ASSET_COMPOSITE2_SO_TYPE, id }; + }); + + await savedObjectsClient.bulkDelete(allObjects, { + refresh: false, + force: true, + }); + }); + }, + + bulkCreate: (rules: RuleAssetComposite2Attributes[]): Promise => { + return withSecuritySpan('IComposite2RuleAssetsClient.bulkCreate', async () => { + const objects: Array> = + rules.map((rule) => ({ + id: rule.rule_id, + type: RULE_ASSET_COMPOSITE2_SO_TYPE, + attributes: rule, + })); + + const chunks = chunk(objects, MAX_RULE_ASSETS_PER_REQUEST); + + for (const chunkOfObjects of chunks) { + await savedObjectsClient.bulkCreate(chunkOfObjects, { + refresh: false, + overwrite: true, + }); + } + }); + }, + + fetchLatestVersions: (): Promise => { + return withSecuritySpan('IComposite2RuleAssetsClient.fetchLatestVersions', async () => { + const findResult = await savedObjectsClient.find({ + type: RULE_ASSET_COMPOSITE2_SO_TYPE, + fields: [ + `${RULE_ASSET_COMPOSITE2_SO_TYPE}.rule_id`, + `${RULE_ASSET_COMPOSITE2_SO_TYPE}.versions.rule_content_version`, + `${RULE_ASSET_COMPOSITE2_SO_TYPE}.versions.stack_version_min`, + `${RULE_ASSET_COMPOSITE2_SO_TYPE}.versions.stack_version_max`, + ], + perPage: 10000, + }); + + return findResult.saved_objects.map((so) => { + const latestVersion = findLatestVersion(so.attributes.versions); + const versionInfo: PrebuiltRuleVersionInfo = { + rule_id: so.attributes.rule_id, + rule_content_version: latestVersion.rule_content_version, + stack_version_min: latestVersion.stack_version_min, + stack_version_max: latestVersion.stack_version_max, + }; + return versionInfo; + }); + }); + }, + + fetchRulesByIdAndVersion: (args: FetchSpecificRulesArgs): Promise => { + return withSecuritySpan('IComposite2RuleAssetsClient.fetchRulesByIdAndVersion', async () => { + const { rules } = args; + + if (rules.length === 0) { + // NOTE: without early return it would build incorrect filter and fetch all existing saved objects + return []; + } + + const attr = `${RULE_ASSET_COMPOSITE2_SO_TYPE}.attributes`; + const filter = rules + .map(({ id: ruleId, version: ruleVersion }) => { + const versionKey = getVersionKey(ruleVersion); + const contentKey = getContentKey(ruleId, versionKey); + return `(${attr}.rule_id: ${ruleId} AND ${attr}.content.${contentKey}.name: *)`; + }) + .join(' OR '); + + const contentKeyFields = rules.map(({ id: ruleId, version: ruleVersion }) => { + const versionKey = getVersionKey(ruleVersion); + const contentKey = getContentKey(ruleId, versionKey); + return `${RULE_ASSET_COMPOSITE2_SO_TYPE}.content.${contentKey}`; + }); + + const fields = [`${RULE_ASSET_COMPOSITE2_SO_TYPE}.rule_id`, ...contentKeyFields]; + + const findResult = await savedObjectsClient.find({ + type: RULE_ASSET_COMPOSITE2_SO_TYPE, + filter, + fields, + perPage: 10000, + }); + + // TODO: Validate that returned objects match the schema of PrebuiltRuleContent + return findResult.saved_objects.map((so) => { + const contentOfRequestedVersion = Object.values(so.attributes.content)[0]; + const ruleContent: PrebuiltRuleContent = { + ...contentOfRequestedVersion, + rule_id: so.attributes.rule_id, + }; + return ruleContent; + }); + }); + }, + }; +}; + +function findLatestVersion(versions: RuleAssetComposite2Attributes['versions']) { + let latestVersion = versions[0]; + versions.slice(1).forEach((version) => { + if (semver.lte(latestVersion.rule_content_version, version.rule_content_version)) + latestVersion = version; + }); + + return latestVersion; +} + +const getVersionKey = (version: string): string => version.replaceAll('.', '_'); + +const getContentKey = (ruleId: string, versionKey: string): string => `${ruleId}__v${versionKey}`; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_composite2_saved_objects_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_composite2_saved_objects_type.ts new file mode 100644 index 0000000000000..69ac309ad5e12 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_composite2_saved_objects_type.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObject, SavedObjectsType } from '@kbn/core/server'; +import type { PrebuiltRuleContent } from '../../../../../../../common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_content'; + +export const RULE_ASSET_COMPOSITE2_SO_TYPE = 'security-rule-composite2'; + +export type RuleAssetComposite2SavedObject = SavedObject; + +export interface RuleAssetComposite2Attributes { + rule_id: string; + versions: RuleVersionInfo[]; + content: Record; +} + +export interface RuleVersionInfo { + rule_content_version: string; + stack_version_min: string; + stack_version_max: string; +} + +const ruleAssetComposite2Mappings: SavedObjectsType['mappings'] = { + dynamic: 'strict', + properties: { + rule_id: { + type: 'keyword', + }, + versions: { + type: 'nested', + properties: { + rule_content_version: { + type: 'version', + }, + stack_version_min: { + type: 'version', + }, + stack_version_max: { + type: 'version', + }, + }, + }, + content: { + type: 'flattened', + }, + }, +}; + +export const ruleAssetComposite2Type: SavedObjectsType = { + name: RULE_ASSET_COMPOSITE2_SO_TYPE, + mappings: ruleAssetComposite2Mappings, + hidden: false, + management: { + importableAndExportable: true, + visibleInManagement: false, + }, + namespaceType: 'agnostic', +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_composite_saved_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_composite_saved_objects_client.ts new file mode 100644 index 0000000000000..5f492b45f019f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_composite_saved_objects_client.ts @@ -0,0 +1,176 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { chunk } from 'lodash'; +import semver from 'semver'; + +import type { + SavedObjectsBulkCreateObject, + SavedObjectsBulkDeleteObject, + SavedObjectsClientContract, + SavedObjectsFindResult, +} from '@kbn/core/server'; + +import { withSecuritySpan } from '../../../../../../utils/with_security_span'; +import type { PrebuiltRuleContent } from '../../../../../../../common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_content'; +import type { PrebuiltRuleVersionInfo } from '../../../../../../../common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_version_info'; +import type { SemanticVersion } from '../../../../../../../common/detection_engine/prebuilt_rules/poc/content_model/semantic_version'; +import type { RuleSignatureId } from '../../../../../../../common/detection_engine/rule_schema'; +import { RULE_ASSET_COMPOSITE_SO_TYPE } from './rule_asset_composite_saved_objects_type'; +import type { RuleAssetCompositeAttributes } from './rule_asset_composite_saved_objects_type'; + +const MAX_RULE_ASSETS_PER_REQUEST = 100; + +export interface ICompositeRuleAssetsClient { + bulkDeleteAll(): Promise; + + bulkCreate(rules: RuleAssetCompositeAttributes[]): Promise; + + fetchLatestVersions(): Promise; + + fetchRulesByIdAndVersion(args: FetchSpecificRulesArgs): Promise; +} + +export interface FetchSpecificRulesArgs { + rules: Array<{ id: RuleSignatureId; version: SemanticVersion }>; +} + +type FindSelector = (result: SavedObjectsFindResult) => T; + +export const createCompositeRuleAssetsClient = ( + savedObjectsClient: SavedObjectsClientContract +): ICompositeRuleAssetsClient => { + const fetchAll = async (selector: FindSelector) => { + const finder = savedObjectsClient.createPointInTimeFinder({ + perPage: MAX_RULE_ASSETS_PER_REQUEST, + type: RULE_ASSET_COMPOSITE_SO_TYPE, + }); + + const result: T[] = []; + + for await (const response of finder.find()) { + const selectedValues = response.saved_objects.map((so) => selector(so)); + result.push(...selectedValues); + } + + await finder.close(); + + return result; + }; + + return { + bulkDeleteAll: (): Promise => { + return withSecuritySpan('ICompositeRuleAssetsClient.bulkDeleteAll', async () => { + const allIds = await fetchAll((so) => so.id); + const allObjects: SavedObjectsBulkDeleteObject[] = allIds.map((id) => { + return { type: RULE_ASSET_COMPOSITE_SO_TYPE, id }; + }); + + await savedObjectsClient.bulkDelete(allObjects, { + refresh: false, + force: true, + }); + }); + }, + + bulkCreate: (rules: RuleAssetCompositeAttributes[]): Promise => { + return withSecuritySpan('ICompositeRuleAssetsClient.bulkCreate', async () => { + const objects: Array> = + rules.map((rule) => ({ + id: rule.rule_id, + type: RULE_ASSET_COMPOSITE_SO_TYPE, + attributes: rule, + })); + + const chunks = chunk(objects, MAX_RULE_ASSETS_PER_REQUEST); + + for (const chunkOfObjects of chunks) { + await savedObjectsClient.bulkCreate(chunkOfObjects, { + refresh: false, + overwrite: true, + }); + } + }); + }, + + fetchLatestVersions: (): Promise => { + return withSecuritySpan('ICompositeRuleAssetsClient.fetchLatestVersions', async () => { + const findResult = await savedObjectsClient.find({ + type: RULE_ASSET_COMPOSITE_SO_TYPE, + fields: [ + `${RULE_ASSET_COMPOSITE_SO_TYPE}.rule_id`, + `${RULE_ASSET_COMPOSITE_SO_TYPE}.versions.name`, + `${RULE_ASSET_COMPOSITE_SO_TYPE}.versions.rule_content_version`, + `${RULE_ASSET_COMPOSITE_SO_TYPE}.versions.stack_version_min`, + `${RULE_ASSET_COMPOSITE_SO_TYPE}.versions.stack_version_max`, + ], + perPage: 10000, + }); + + return findResult.saved_objects.map((so) => { + const latestVersion = findLatestVersion(so.attributes.versions); + const versionInfo: PrebuiltRuleVersionInfo = { + rule_id: so.attributes.rule_id, + rule_content_version: latestVersion.rule_content_version, + stack_version_min: latestVersion.stack_version_min, + stack_version_max: latestVersion.stack_version_max, + }; + return versionInfo; + }); + }); + }, + + fetchRulesByIdAndVersion: (args: FetchSpecificRulesArgs): Promise => { + return withSecuritySpan('ICompositeRuleAssetsClient.fetchRulesByIdAndVersion', async () => { + const { rules } = args; + + if (rules.length === 0) { + // NOTE: without early return it would build incorrect filter and fetch all existing saved objects + return []; + } + + const attr = `${RULE_ASSET_COMPOSITE_SO_TYPE}.attributes`; + const filter = rules + .map( + (rule) => + `(${attr}.rule_id: ${rule.id} AND ${attr}.versions:{ rule_content_version: ${rule.version} })` + ) + .join(' OR '); + + const findResult = await savedObjectsClient.find({ + type: RULE_ASSET_COMPOSITE_SO_TYPE, + filter, + perPage: 10000, + }); + + // TODO: Validate that returned objects match the schema of PrebuiltRuleContent + return findResult.saved_objects.map((so) => { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const versionNum = rules.find((r) => r.id === so.attributes.rule_id)!.version; + const versionObj = so.attributes.versions.find( + (v) => v.rule_content_version === versionNum + ); + const specificVersion = { + rule_id: so.attributes.rule_id, + ...versionObj, + }; + return specificVersion as PrebuiltRuleContent; + }); + }); + }, + }; +}; + +function findLatestVersion(versions: RuleAssetCompositeAttributes['versions']) { + let latestVersion = versions[0]; + versions.slice(1).forEach((version) => { + if (semver.lte(latestVersion.rule_content_version, version.rule_content_version)) + latestVersion = version; + }); + + return latestVersion; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_composite_saved_objects_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_composite_saved_objects_type.ts new file mode 100644 index 0000000000000..aecfdcc5e0654 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_composite_saved_objects_type.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObject, SavedObjectsType } from '@kbn/core/server'; + +export const RULE_ASSET_COMPOSITE_SO_TYPE = 'security-rule-composite'; + +export type RuleAssetCompositeSavedObject = SavedObject; + +export interface RuleAssetCompositeAttributes { + rule_id: string; + versions: HistoricalRuleVersion[]; +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export interface HistoricalRuleVersion extends Record { + name: string; + rule_content_version: string; + stack_version_min: string; + stack_version_max: string; +} + +const ruleAssetCompositeMappings: SavedObjectsType['mappings'] = { + dynamic: false, + properties: { + rule_id: { + type: 'keyword', + }, + versions: { + type: 'nested', + properties: { + name: { + type: 'keyword', + }, + rule_content_version: { + type: 'version', + }, + stack_version_min: { + type: 'version', + }, + stack_version_max: { + type: 'version', + }, + }, + }, + }, +}; + +export const ruleAssetCompositeType: SavedObjectsType = { + name: RULE_ASSET_COMPOSITE_SO_TYPE, + mappings: ruleAssetCompositeMappings, + hidden: false, + management: { + importableAndExportable: true, + visibleInManagement: false, + }, + namespaceType: 'agnostic', +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_flat_saved_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_flat_saved_objects_client.ts new file mode 100644 index 0000000000000..2b9f0fe840026 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_flat_saved_objects_client.ts @@ -0,0 +1,257 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { chunk } from 'lodash'; + +import type { + SavedObjectsBulkCreateObject, + SavedObjectsBulkDeleteObject, + SavedObjectsClientContract, + SavedObjectsFindResult, +} from '@kbn/core/server'; + +import { withSecuritySpan } from '../../../../../../utils/with_security_span'; +import type { PrebuiltRuleContent } from '../../../../../../../common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_content'; +import type { PrebuiltRuleVersionInfo } from '../../../../../../../common/detection_engine/prebuilt_rules/poc/content_model/prebuilt_rule_version_info'; +import type { SemanticVersion } from '../../../../../../../common/detection_engine/prebuilt_rules/poc/content_model/semantic_version'; +import type { RuleSignatureId } from '../../../../../../../common/detection_engine/rule_schema'; +import { RULE_ASSET_FLAT_SO_TYPE } from './rule_asset_flat_saved_objects_type'; +import type { RuleAssetFlatAttributes } from './rule_asset_flat_saved_objects_type'; + +const MAX_RULE_ASSETS_PER_REQUEST = 500; + +export interface IFlatRuleAssetsClient { + bulkDeleteAll(): Promise; + + bulkCreate(rules: RuleAssetFlatAttributes[]): Promise; + + fetchLatestVersions(): Promise; + + fetchRulesByIdAndVersion(args: FetchSpecificRulesArgs): Promise; +} + +export interface FetchSpecificRulesArgs { + rules: Array<{ id: RuleSignatureId; version: SemanticVersion }>; +} + +type FindSelector = (result: SavedObjectsFindResult) => T; + +export const createFlatRuleAssetsClient = ( + savedObjectsClient: SavedObjectsClientContract +): IFlatRuleAssetsClient => { + const fetchAll = async (selector: FindSelector) => { + const finder = savedObjectsClient.createPointInTimeFinder({ + perPage: MAX_RULE_ASSETS_PER_REQUEST, + type: RULE_ASSET_FLAT_SO_TYPE, + }); + + const result: T[] = []; + + for await (const response of finder.find()) { + const selectedValues = response.saved_objects.map((so) => selector(so)); + result.push(...selectedValues); + } + + await finder.close(); + + return result; + }; + + return { + bulkDeleteAll: (): Promise => { + return withSecuritySpan('IFlatRuleAssetsClient.bulkDeleteAll', async () => { + const allIds = await fetchAll((so) => so.id); + const allObjects: SavedObjectsBulkDeleteObject[] = allIds.map((id) => { + return { type: RULE_ASSET_FLAT_SO_TYPE, id }; + }); + + await savedObjectsClient.bulkDelete(allObjects, { + refresh: false, + force: true, + }); + }); + }, + + bulkCreate: (rules: RuleAssetFlatAttributes[]): Promise => { + return withSecuritySpan('IFlatRuleAssetsClient.bulkCreate', async () => { + const objects: Array> = rules.map( + (rule) => ({ + id: `${rule.rule_id}:${rule.rule_content_version}`, + type: RULE_ASSET_FLAT_SO_TYPE, + attributes: rule, + }) + ); + + const chunks = chunk(objects, MAX_RULE_ASSETS_PER_REQUEST); + + for (const chunkOfObjects of chunks) { + await savedObjectsClient.bulkCreate(chunkOfObjects, { + refresh: false, + overwrite: true, + }); + } + }); + }, + + fetchLatestVersions: (): Promise => { + return withSecuritySpan('IFlatRuleAssetsClient.fetchLatestVersions', async () => { + /* + GET .kibana/_search + { + "size": 0, + "query": { + "term": { "type": "security-rule-flat" } + }, + "aggs": { + "rules": { + "terms": { + "field": "security-rule-flat.rule_id", + "size": 10000 + }, + "aggs": { + "latest_version": { + "top_hits": { + "size": 1, + "sort": [ + { + "security-rule-flat.rule_content_version": { + "order": "desc" + } + } + ], + "_source": { + "includes": [ + "security-rule-flat.rule_id", + "security-rule-flat.rule_content_version", + "security-rule-flat.stack_version_min", + "security-rule-flat.stack_version_max" + ] + } + } + } + } + } + } + } + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const findResult = await savedObjectsClient.find({ + type: RULE_ASSET_FLAT_SO_TYPE, + aggs: { + rules: { + terms: { + field: `${RULE_ASSET_FLAT_SO_TYPE}.attributes.rule_id`, + size: 10000, + }, + aggs: { + latest_version: { + top_hits: { + size: 1, + sort: [ + { + [`${RULE_ASSET_FLAT_SO_TYPE}.rule_content_version`]: { + order: 'desc', + }, + }, + ], + _source: [ + `${RULE_ASSET_FLAT_SO_TYPE}.rule_id`, + `${RULE_ASSET_FLAT_SO_TYPE}.rule_content_version`, + `${RULE_ASSET_FLAT_SO_TYPE}.stack_version_min`, + `${RULE_ASSET_FLAT_SO_TYPE}.stack_version_max`, + ], + }, + }, + }, + }, + }, + }); + + /* + "aggregations": { + "rules": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": "000047bb-b27a-47ec-8b62-ef1a5d2c9e19", + "doc_count": 10, + "latest_version": { + "hits": { + "total": { + "value": 10, + "relation": "eq" + }, + "max_score": null, + "hits": [ + { + "_index": ".kibana_8.7.0_001", + "_id": "security-rule-flat:000047bb-b27a-47ec-8b62-ef1a5d2c9e19:102.0.9", + "_score": null, + "_source": { + "security-rule-flat": { + "rule_id": "000047bb-b27a-47ec-8b62-ef1a5d2c9e19", + "rule_content_version": "102.0.9", + "stack_version_min": "8.5.0", + "stack_version_max": "8.7.0" + } + }, + "sort": [ + "102.0.9" + ] + } + ] + } + } + }, + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const buckets: any[] = findResult.aggregations?.rules?.buckets ?? []; + + return buckets.map((bucket) => { + const hit = bucket.latest_version.hits.hits[0]; + const soAttributes = hit._source[RULE_ASSET_FLAT_SO_TYPE]; + const versionInfo: PrebuiltRuleVersionInfo = { + rule_id: soAttributes.rule_id, + rule_content_version: soAttributes.rule_content_version, + stack_version_min: soAttributes.stack_version_min, + stack_version_max: soAttributes.stack_version_max, + }; + return versionInfo; + }); + }); + }, + + fetchRulesByIdAndVersion: (args: FetchSpecificRulesArgs): Promise => { + return withSecuritySpan('IFlatRuleAssetsClient.fetchRulesByIdAndVersion', async () => { + const { rules } = args; + + if (rules.length === 0) { + // NOTE: without early return it would build incorrect filter and fetch all existing saved objects + return []; + } + + const attr = `${RULE_ASSET_FLAT_SO_TYPE}.attributes`; + const filter = rules + .map( + (rule) => + `(${attr}.rule_id: ${rule.id} AND ${attr}.rule_content_version: ${rule.version})` + ) + .join(' OR '); + + const findResult = await savedObjectsClient.find({ + type: RULE_ASSET_FLAT_SO_TYPE, + filter, + perPage: 10000, + }); + + // TODO: Validate that returned objects match the schema of PrebuiltRuleContent + return findResult.saved_objects.map((so) => so.attributes as PrebuiltRuleContent); + }); + }, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_flat_saved_objects_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_flat_saved_objects_type.ts new file mode 100644 index 0000000000000..06f4e61dfe5ad --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_flat_saved_objects_type.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObject, SavedObjectsType } from '@kbn/core/server'; + +export const RULE_ASSET_FLAT_SO_TYPE = 'security-rule-flat'; + +export type RuleAssetFlatSavedObject = SavedObject; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export interface RuleAssetFlatAttributes extends Record { + name: string; + rule_id: string; + rule_content_version: string; + stack_version_min: string; + stack_version_max: string; +} + +const ruleAssetFlatMappings: SavedObjectsType['mappings'] = { + dynamic: false, + properties: { + name: { + type: 'keyword', + }, + rule_id: { + type: 'keyword', + }, + rule_content_version: { + type: 'version', + }, + stack_version_min: { + type: 'version', + }, + stack_version_max: { + type: 'version', + }, + }, +}; + +export const ruleAssetFlatType: SavedObjectsType = { + name: RULE_ASSET_FLAT_SO_TYPE, + mappings: ruleAssetFlatMappings, + hidden: false, + management: { + importableAndExportable: true, + visibleInManagement: false, + }, + namespaceType: 'agnostic', +}; diff --git a/x-pack/plugins/security_solution/server/routes/index.ts b/x-pack/plugins/security_solution/server/routes/index.ts index 38809aff316a0..da8e5bf36f81c 100644 --- a/x-pack/plugins/security_solution/server/routes/index.ts +++ b/x-pack/plugins/security_solution/server/routes/index.ts @@ -91,7 +91,7 @@ export const initRoutes = ( ) => { registerFleetIntegrationsRoutes(router, logger); registerLegacyRuleActionsRoutes(router, logger); - registerPrebuiltRulesRoutes(router, config, security); + registerPrebuiltRulesRoutes(router, config, logger, security); registerRuleExceptionsRoutes(router); registerManageExceptionsRoutes(router); registerRuleManagementRoutes(router, config, ml, logger); diff --git a/x-pack/plugins/security_solution/server/saved_objects.ts b/x-pack/plugins/security_solution/server/saved_objects.ts index bc1d86add3767..c48704a9cbf02 100644 --- a/x-pack/plugins/security_solution/server/saved_objects.ts +++ b/x-pack/plugins/security_solution/server/saved_objects.ts @@ -12,6 +12,9 @@ import { noteType, pinnedEventType, timelineType } from './lib/timeline/saved_ob import { legacyType as legacyRuleActionsType } from './lib/detection_engine/rule_actions_legacy'; import { ruleExecutionType } from './lib/detection_engine/rule_monitoring'; import { ruleAssetType } from './lib/detection_engine/prebuilt_rules/logic/rule_asset/rule_asset_saved_object_mappings'; +import { ruleAssetCompositeType } from './lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_composite_saved_objects_type'; +import { ruleAssetComposite2Type } from './lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_composite2_saved_objects_type'; +import { ruleAssetFlatType } from './lib/detection_engine/prebuilt_rules/logic/poc/saved_objects/rule_asset_flat_saved_objects_type'; import { type as signalsMigrationType } from './lib/detection_engine/migrations/saved_objects'; import { exceptionsArtifactType, @@ -24,6 +27,9 @@ const types = [ legacyRuleActionsType, ruleExecutionType, ruleAssetType, + ruleAssetCompositeType, + ruleAssetComposite2Type, + ruleAssetFlatType, timelineType, exceptionsArtifactType, manifestType, diff --git a/yarn.lock b/yarn.lock index 45558eda05538..0aa5e1162b70a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -20413,6 +20413,11 @@ node-cache@^5.1.0: dependencies: clone "2.x" +node-diff3@^3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/node-diff3/-/node-diff3-3.1.2.tgz#49df8d821dc9cbab87bfd6182171d90169613a97" + integrity sha512-wUd9TWy059I8mZdH6G3LPNlAEfxDvXtn/RcyFrbqL3v34WlDxn+Mh4HDhOwWuaMk/ROVepe5tTpnGHbve6Db2g== + node-dir@^0.1.10: version "0.1.17" resolved "https://registry.yarnpkg.com/node-dir/-/node-dir-0.1.17.tgz#5f5665d93351335caabef8f1c554516cf5f1e4e5"