From 05d3dfa4471904fb2b494b6af8a6cdb81fe869dc Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Wed, 6 Mar 2024 13:05:10 +0100 Subject: [PATCH] [Security Solution] Set socket timeout for potentially long running Rule Management API endpoints (#177329) **Fixes: https://github.com/elastic/kibana/issues/177277** ## Summary This PR set a reasonably high (1 hour) socket timeout for potentially long running Rule Management API endpoints. It's important to note this fix only mitigates closing TCP connection risks. Proxies have own TCP connection timeout though it's higher than default node.js 2 minutes. ## Details When performing operations on a large number of rules and/or in a resource limited or suffering from performance degradation environment endpoints may take more time than default node.js socket timeout which is 2 minutes. According to the [HTTP spec](https://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html#sec8.2.4) browser should retry if the connection was closed by the server. Taking into account API endpoint's handler isn't terminated after closing a TCP connection a retry attempt will spawn a new request processing in parallel. Under some circumstance it can lead to creating multiple rules with the same `rule_id` and for example end up creating more rules than expected like described here https://github.com/elastic/kibana/issues/176207. --- .../install_prebuilt_rules_and_timelines_route.ts | 8 ++------ .../perform_rule_installation_route.ts | 4 ++++ .../perform_rule_upgrade_route.ts | 4 ++++ .../review_rule_installation_route.ts | 4 ++++ .../review_rule_upgrade_route.ts | 4 ++++ .../detection_engine/prebuilt_rules/constants.ts | 8 ++++++++ .../api/rules/bulk_actions/route.ts | 4 ++-- .../api/rules/bulk_create_rules/route.ts | 5 ++++- .../api/rules/bulk_delete_rules/route.ts | 4 ++++ .../api/rules/bulk_patch_rules/route.ts | 4 ++++ .../api/rules/bulk_update_rules/route.ts | 4 ++++ .../api/rules/export_rules/route.ts | 4 ++++ .../api/rules/import_rules/route.ts | 4 ++++ .../rule_management/api/timeouts.ts | 15 +++++++++++++++ 14 files changed, 67 insertions(+), 9 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/constants.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/timeouts.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/install_prebuilt_rules_and_timelines/install_prebuilt_rules_and_timelines_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/install_prebuilt_rules_and_timelines/install_prebuilt_rules_and_timelines_route.ts index 546256889ecfb..20f1bc962232e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/install_prebuilt_rules_and_timelines/install_prebuilt_rules_and_timelines_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/install_prebuilt_rules_and_timelines/install_prebuilt_rules_and_timelines_route.ts @@ -8,7 +8,6 @@ import type { RulesClient } from '@kbn/alerting-plugin/server'; import type { ExceptionListClient } from '@kbn/lists-plugin/server'; import { transformError } from '@kbn/securitysolution-es-utils'; -import moment from 'moment'; import { InstallPrebuiltRulesAndTimelinesResponse, PREBUILT_RULES_URL, @@ -19,6 +18,7 @@ import type { } from '../../../../../types'; import { buildSiemResponse } from '../../../routes/utils'; import { getExistingPrepackagedRules } from '../../../rule_management/logic/search/get_existing_prepackaged_rules'; +import { PREBUILT_RULES_OPERATION_SOCKET_TIMEOUT_MS } from '../../constants'; import { ensureLatestRulesPackageInstalled } from '../../logic/ensure_latest_rules_package_installed'; import { getRulesToInstall } from '../../logic/get_rules_to_install'; import { getRulesToUpdate } from '../../logic/get_rules_to_update'; @@ -36,11 +36,7 @@ export const installPrebuiltRulesAndTimelinesRoute = (router: SecuritySolutionPl options: { tags: ['access:securitySolution'], timeout: { - // FUNFACT: If we do not add a very long timeout what will happen - // is that Chrome which receive a 408 error and then do a retry. - // This retry can cause lots of connections to happen. Using a very - // long timeout will ensure that Chrome does not do retries and saturate the connections. - idleSocket: moment.duration('1', 'hour').asMilliseconds(), + idleSocket: PREBUILT_RULES_OPERATION_SOCKET_TIMEOUT_MS, }, }, }) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/perform_rule_installation/perform_rule_installation_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/perform_rule_installation/perform_rule_installation_route.ts index dd6437ccc8e29..d76c6d93fc852 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/perform_rule_installation/perform_rule_installation_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/perform_rule_installation/perform_rule_installation_route.ts @@ -28,6 +28,7 @@ import { createPrebuiltRuleObjectsClient } from '../../logic/rule_objects/prebui import { fetchRuleVersionsTriad } from '../../logic/rule_versions/fetch_rule_versions_triad'; import { getVersionBuckets } from '../../model/rule_versions/get_version_buckets'; import { performTimelinesInstallation } from '../../logic/perform_timelines_installation'; +import { PREBUILT_RULES_OPERATION_SOCKET_TIMEOUT_MS } from '../../constants'; export const performRuleInstallationRoute = (router: SecuritySolutionPluginRouter) => { router.versioned @@ -36,6 +37,9 @@ export const performRuleInstallationRoute = (router: SecuritySolutionPluginRoute path: PERFORM_RULE_INSTALLATION_URL, options: { tags: ['access:securitySolution'], + timeout: { + idleSocket: PREBUILT_RULES_OPERATION_SOCKET_TIMEOUT_MS, + }, }, }) .addVersion( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/perform_rule_upgrade/perform_rule_upgrade_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/perform_rule_upgrade/perform_rule_upgrade_route.ts index 3009495e32297..09342a9150a50 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/perform_rule_upgrade/perform_rule_upgrade_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/perform_rule_upgrade/perform_rule_upgrade_route.ts @@ -30,6 +30,7 @@ import { upgradePrebuiltRules } from '../../logic/rule_objects/upgrade_prebuilt_ import { fetchRuleVersionsTriad } from '../../logic/rule_versions/fetch_rule_versions_triad'; import type { PrebuiltRuleAsset } from '../../model/rule_assets/prebuilt_rule_asset'; import { getVersionBuckets } from '../../model/rule_versions/get_version_buckets'; +import { PREBUILT_RULES_OPERATION_SOCKET_TIMEOUT_MS } from '../../constants'; export const performRuleUpgradeRoute = (router: SecuritySolutionPluginRouter) => { router.versioned @@ -38,6 +39,9 @@ export const performRuleUpgradeRoute = (router: SecuritySolutionPluginRouter) => path: PERFORM_RULE_UPGRADE_URL, options: { tags: ['access:securitySolution'], + timeout: { + idleSocket: PREBUILT_RULES_OPERATION_SOCKET_TIMEOUT_MS, + }, }, }) .addVersion( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_installation/review_rule_installation_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_installation/review_rule_installation_route.ts index c10ca16fe0874..0b30c9bab4782 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_installation/review_rule_installation_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_installation/review_rule_installation_route.ts @@ -19,6 +19,7 @@ import { fetchRuleVersionsTriad } from '../../logic/rule_versions/fetch_rule_ver import type { PrebuiltRuleAsset } from '../../model/rule_assets/prebuilt_rule_asset'; import { getVersionBuckets } from '../../model/rule_versions/get_version_buckets'; import { convertPrebuiltRuleAssetToRuleResponse } from '../../../rule_management/normalization/rule_converters'; +import { PREBUILT_RULES_OPERATION_SOCKET_TIMEOUT_MS } from '../../constants'; export const reviewRuleInstallationRoute = (router: SecuritySolutionPluginRouter) => { router.versioned @@ -27,6 +28,9 @@ export const reviewRuleInstallationRoute = (router: SecuritySolutionPluginRouter path: REVIEW_RULE_INSTALLATION_URL, options: { tags: ['access:securitySolution'], + timeout: { + idleSocket: PREBUILT_RULES_OPERATION_SOCKET_TIMEOUT_MS, + }, }, }) .addVersion( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/review_rule_upgrade_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/review_rule_upgrade_route.ts index 1fcb36b592d56..2270c6fea7396 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/review_rule_upgrade_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/review_rule_upgrade_route.ts @@ -25,6 +25,7 @@ import { createPrebuiltRuleObjectsClient } from '../../logic/rule_objects/prebui import { fetchRuleVersionsTriad } from '../../logic/rule_versions/fetch_rule_versions_triad'; import { getVersionBuckets } from '../../model/rule_versions/get_version_buckets'; import { convertPrebuiltRuleAssetToRuleResponse } from '../../../rule_management/normalization/rule_converters'; +import { PREBUILT_RULES_OPERATION_SOCKET_TIMEOUT_MS } from '../../constants'; export const reviewRuleUpgradeRoute = (router: SecuritySolutionPluginRouter) => { router.versioned @@ -33,6 +34,9 @@ export const reviewRuleUpgradeRoute = (router: SecuritySolutionPluginRouter) => path: REVIEW_RULE_UPGRADE_URL, options: { tags: ['access:securitySolution'], + timeout: { + idleSocket: PREBUILT_RULES_OPERATION_SOCKET_TIMEOUT_MS, + }, }, }) .addVersion( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/constants.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/constants.ts new file mode 100644 index 0000000000000..1024b3e0726cd --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/constants.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const PREBUILT_RULES_OPERATION_SOCKET_TIMEOUT_MS = 1800000 as const; // 30 minutes diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_actions/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_actions/route.ts index d09c11d1289bd..ef06f8f004c98 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_actions/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_actions/route.ts @@ -6,7 +6,6 @@ */ import { truncate } from 'lodash'; -import moment from 'moment'; import { BadRequestError, transformError } from '@kbn/securitysolution-es-utils'; import type { IKibanaResponse, KibanaResponseFactory, Logger } from '@kbn/core/server'; @@ -58,6 +57,7 @@ import { validateBulkDuplicateRule, dryRunValidateBulkEditRule, } from '../../../logic/bulk_actions/validations'; +import { RULE_MANAGEMENT_BULK_ACTION_SOCKET_TIMEOUT_MS } from '../../timeouts'; const MAX_RULES_TO_PROCESS_TOTAL = 10000; const MAX_ERROR_MESSAGE_LENGTH = 1000; @@ -242,7 +242,7 @@ export const performBulkActionRoute = ( options: { tags: ['access:securitySolution', routeLimitedConcurrencyTag(MAX_ROUTE_CONCURRENCY)], timeout: { - idleSocket: moment.duration(15, 'minutes').asMilliseconds(), + idleSocket: RULE_MANAGEMENT_BULK_ACTION_SOCKET_TIMEOUT_MS, }, }, }) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_create_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_create_rules/route.ts index 7ab03f2ac7dcd..abd14b7fbb4a9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_create_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_create_rules/route.ts @@ -26,7 +26,7 @@ import { transformValidateBulkError } from '../../../utils/validate'; import { buildRouteValidationWithZod } from '../../../../../../utils/build_validation/route_validation'; import { validateRuleDefaultExceptionList } from '../../../logic/exceptions/validate_rule_default_exception_list'; import { validateRulesWithDuplicatedDefaultExceptionsList } from '../../../logic/exceptions/validate_rules_with_duplicated_default_exceptions_list'; - +import { RULE_MANAGEMENT_BULK_ACTION_SOCKET_TIMEOUT_MS } from '../../timeouts'; import { transformBulkError, createBulkErrorObject, @@ -48,6 +48,9 @@ export const bulkCreateRulesRoute = ( path: DETECTION_ENGINE_RULES_BULK_CREATE, options: { tags: ['access:securitySolution'], + timeout: { + idleSocket: RULE_MANAGEMENT_BULK_ACTION_SOCKET_TIMEOUT_MS, + }, }, }) .addVersion( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_delete_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_delete_rules/route.ts index 639f279215801..98c938f43a7bb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_delete_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_delete_rules/route.ts @@ -29,6 +29,7 @@ import { readRules } from '../../../logic/crud/read_rules'; import { getIdBulkError } from '../../../utils/utils'; import { transformValidateBulkError } from '../../../utils/validate'; import { getDeprecatedBulkEndpointHeader, logDeprecatedBulkEndpoint } from '../../deprecation'; +import { RULE_MANAGEMENT_BULK_ACTION_SOCKET_TIMEOUT_MS } from '../../timeouts'; type Handler = RequestHandler< unknown, @@ -106,6 +107,9 @@ export const bulkDeleteRulesRoute = (router: SecuritySolutionPluginRouter, logge path: DETECTION_ENGINE_RULES_BULK_DELETE, options: { tags: ['access:securitySolution'], + timeout: { + idleSocket: RULE_MANAGEMENT_BULK_ACTION_SOCKET_TIMEOUT_MS, + }, }, }; const versionConfig = { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_patch_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_patch_rules/route.ts index 7d7c4a44fa1e1..5d6d74f230ce5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_patch_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_patch_rules/route.ts @@ -27,6 +27,7 @@ import { readRules } from '../../../logic/crud/read_rules'; import { getDeprecatedBulkEndpointHeader, logDeprecatedBulkEndpoint } from '../../deprecation'; import { validateRuleDefaultExceptionList } from '../../../logic/exceptions/validate_rule_default_exception_list'; import { validateRulesWithDuplicatedDefaultExceptionsList } from '../../../logic/exceptions/validate_rules_with_duplicated_default_exceptions_list'; +import { RULE_MANAGEMENT_BULK_ACTION_SOCKET_TIMEOUT_MS } from '../../timeouts'; /** * @deprecated since version 8.2.0. Use the detection_engine/rules/_bulk_action API instead @@ -42,6 +43,9 @@ export const bulkPatchRulesRoute = ( path: DETECTION_ENGINE_RULES_BULK_UPDATE, options: { tags: ['access:securitySolution'], + timeout: { + idleSocket: RULE_MANAGEMENT_BULK_ACTION_SOCKET_TIMEOUT_MS, + }, }, }) .addVersion( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_update_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_update_rules/route.ts index dd9fec81f34a6..7bec95bf90da9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_update_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_update_rules/route.ts @@ -32,6 +32,7 @@ import { readRules } from '../../../logic/crud/read_rules'; import { getDeprecatedBulkEndpointHeader, logDeprecatedBulkEndpoint } from '../../deprecation'; import { validateRuleDefaultExceptionList } from '../../../logic/exceptions/validate_rule_default_exception_list'; import { validateRulesWithDuplicatedDefaultExceptionsList } from '../../../logic/exceptions/validate_rules_with_duplicated_default_exceptions_list'; +import { RULE_MANAGEMENT_BULK_ACTION_SOCKET_TIMEOUT_MS } from '../../timeouts'; /** * @deprecated since version 8.2.0. Use the detection_engine/rules/_bulk_action API instead @@ -47,6 +48,9 @@ export const bulkUpdateRulesRoute = ( path: DETECTION_ENGINE_RULES_BULK_UPDATE, options: { tags: ['access:securitySolution'], + timeout: { + idleSocket: RULE_MANAGEMENT_BULK_ACTION_SOCKET_TIMEOUT_MS, + }, }, }) .addVersion( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/export_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/export_rules/route.ts index 5b190a1065018..301bb62452f28 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/export_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/export_rules/route.ts @@ -21,6 +21,7 @@ import { getNonPackagedRulesCount } from '../../../logic/search/get_existing_pre import { getExportByObjectIds } from '../../../logic/export/get_export_by_object_ids'; import { getExportAll } from '../../../logic/export/get_export_all'; import { buildSiemResponse } from '../../../../routes/utils'; +import { RULE_MANAGEMENT_IMPORT_EXPORT_SOCKET_TIMEOUT_MS } from '../../timeouts'; export const exportRulesRoute = ( router: SecuritySolutionPluginRouter, @@ -33,6 +34,9 @@ export const exportRulesRoute = ( path: `${DETECTION_ENGINE_RULES_URL}/_export`, options: { tags: ['access:securitySolution'], + timeout: { + idleSocket: RULE_MANAGEMENT_IMPORT_EXPORT_SOCKET_TIMEOUT_MS, + }, }, }) .addVersion( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts index 5867d099fb8de..0e1013f452d66 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts @@ -33,6 +33,7 @@ import { getTupleDuplicateErrorsAndUniqueRules, migrateLegacyActionsIds, } from '../../../utils/utils'; +import { RULE_MANAGEMENT_IMPORT_EXPORT_SOCKET_TIMEOUT_MS } from '../../timeouts'; const CHUNK_PARSED_OBJECT_SIZE = 50; @@ -51,6 +52,9 @@ export const importRulesRoute = ( maxBytes: config.maxRuleImportPayloadBytes, output: 'stream', }, + timeout: { + idleSocket: RULE_MANAGEMENT_IMPORT_EXPORT_SOCKET_TIMEOUT_MS, + }, }, }) .addVersion( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/timeouts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/timeouts.ts new file mode 100644 index 0000000000000..24bf61ddc2773 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/timeouts.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * 1 hour = 3600000 ms = 60 minutes * 60 seconds * 1000 ms + */ +export const RULE_MANAGEMENT_BULK_ACTION_SOCKET_TIMEOUT_MS = 3600000 as const; +/** + * 1 hour = 3600000 ms = 60 minutes * 60 seconds * 1000 ms + */ +export const RULE_MANAGEMENT_IMPORT_EXPORT_SOCKET_TIMEOUT_MS = 3600000 as const;