diff --git a/.eslintrc.js b/.eslintrc.js index 730c9599f23f9..e2d02c33288a7 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1995,9 +1995,6 @@ module.exports = { // logsShared depends on o11y/private plugins, but platform plugins depend on it 'x-pack/plugins/observability_solution/logs_shared/**', - // this plugin depends on visTypeTimeseries plugin (for TSVB viz) which is platform/private ATM - 'x-pack/plugins/observability_solution/infra/**', - // TODO @kibana/operations 'scripts/create_observability_rules.js', // is importing "@kbn/observability-alerting-test-data" (observability/private) 'src/cli_setup/**', // is importing "@kbn/interactive-setup-plugin" (platform/private) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ef265cf7c569a..f0d509e283b2a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -324,7 +324,7 @@ packages/kbn-custom-icons @elastic/obs-ux-logs-team packages/kbn-custom-integrations @elastic/obs-ux-logs-team packages/kbn-cypress-config @elastic/kibana-operations packages/kbn-data-service @elastic/kibana-visualizations @elastic/kibana-data-discovery -packages/kbn-data-stream-adapter @elastic/security-threat-hunting-explore +packages/kbn-data-stream-adapter @elastic/security-threat-hunting packages/kbn-data-view-utils @elastic/kibana-data-discovery packages/kbn-datemath @elastic/kibana-data-discovery packages/kbn-dev-cli-errors @elastic/kibana-operations @@ -380,6 +380,7 @@ packages/kbn-i18n @elastic/kibana-core packages/kbn-i18n-react @elastic/kibana-core packages/kbn-import-locator @elastic/kibana-operations packages/kbn-import-resolver @elastic/kibana-operations +packages/kbn-index-adapter @elastic/security-threat-hunting packages/kbn-interpreter @elastic/kibana-visualizations packages/kbn-investigation-shared @elastic/obs-ux-management-team packages/kbn-io-ts-utils @elastic/obs-knowledge-team diff --git a/api_docs/kbn_elastic_assistant_common.devdocs.json b/api_docs/kbn_elastic_assistant_common.devdocs.json index 3a8972f18fd3a..fbd6ee4384bb0 100644 --- a/api_docs/kbn_elastic_assistant_common.devdocs.json +++ b/api_docs/kbn_elastic_assistant_common.devdocs.json @@ -997,7 +997,7 @@ "\nInterface for features available to the elastic assistant" ], "signature": [ - "{ readonly assistantKnowledgeBaseByDefault: boolean; readonly assistantModelEvaluation: boolean; }" + "{ readonly assistantModelEvaluation: boolean; }" ], "path": "x-pack/packages/kbn-elastic-assistant-common/impl/capabilities/index.ts", "deprecated": false, @@ -2772,7 +2772,7 @@ "label": "GetCapabilitiesResponse", "description": [], "signature": [ - "{ assistantKnowledgeBaseByDefault: boolean; assistantModelEvaluation: boolean; }" + "{ assistantModelEvaluation: boolean; }" ], "path": "x-pack/packages/kbn-elastic-assistant-common/impl/schemas/capabilities/get_capabilities_route.gen.ts", "deprecated": false, @@ -4767,7 +4767,7 @@ "\nDefault features available to the elastic assistant" ], "signature": [ - "{ readonly assistantKnowledgeBaseByDefault: true; readonly assistantModelEvaluation: false; }" + "{ readonly assistantModelEvaluation: false; }" ], "path": "x-pack/packages/kbn-elastic-assistant-common/impl/capabilities/index.ts", "deprecated": false, @@ -5232,7 +5232,7 @@ "label": "GetCapabilitiesResponse", "description": [], "signature": [ - "Zod.ZodObject<{ assistantKnowledgeBaseByDefault: Zod.ZodBoolean; assistantModelEvaluation: Zod.ZodBoolean; }, \"strip\", Zod.ZodTypeAny, { assistantKnowledgeBaseByDefault: boolean; assistantModelEvaluation: boolean; }, { assistantKnowledgeBaseByDefault: boolean; assistantModelEvaluation: boolean; }>" + "Zod.ZodObject<{ assistantModelEvaluation: Zod.ZodBoolean; }, \"strip\", Zod.ZodTypeAny, { assistantModelEvaluation: boolean; }, { assistantModelEvaluation: boolean; }>" ], "path": "x-pack/packages/kbn-elastic-assistant-common/impl/schemas/capabilities/get_capabilities_route.gen.ts", "deprecated": false, @@ -6201,4 +6201,4 @@ } ] } -} \ No newline at end of file +} diff --git a/api_docs/security_solution.devdocs.json b/api_docs/security_solution.devdocs.json index 608baf2669dfc..94910aa77b079 100644 --- a/api_docs/security_solution.devdocs.json +++ b/api_docs/security_solution.devdocs.json @@ -420,7 +420,7 @@ "\nExperimental flag needed to enable the link" ], "signature": [ - "\"assistantKnowledgeBaseByDefault\" | \"assistantModelEvaluation\" | \"excludePoliciesInFilterEnabled\" | \"kubernetesEnabled\" | \"donutChartEmbeddablesEnabled\" | \"previewTelemetryUrlEnabled\" | \"extendedRuleExecutionLoggingEnabled\" | \"socTrendsEnabled\" | \"responseActionUploadEnabled\" | \"automatedProcessActionsEnabled\" | \"responseActionsSentinelOneV1Enabled\" | \"responseActionsSentinelOneV2Enabled\" | \"responseActionsSentinelOneGetFileEnabled\" | \"responseActionsSentinelOneKillProcessEnabled\" | \"responseActionsSentinelOneProcessesEnabled\" | \"responseActionsCrowdstrikeManualHostIsolationEnabled\" | \"endpointManagementSpaceAwarenessEnabled\" | \"securitySolutionNotesDisabled\" | \"entityAlertPreviewDisabled\" | \"newUserDetailsFlyoutManagedUser\" | \"riskScoringPersistence\" | \"riskScoringRoutesEnabled\" | \"esqlRulesDisabled\" | \"protectionUpdatesEnabled\" | \"disableTimelineSaveTour\" | \"riskEnginePrivilegesRouteEnabled\" | \"sentinelOneDataInAnalyzerEnabled\" | \"sentinelOneManualHostActionsEnabled\" | \"crowdstrikeDataInAnalyzerEnabled\" | \"responseActionsTelemetryEnabled\" | \"jamfDataInAnalyzerEnabled\" | \"timelineEsqlTabDisabled\" | \"analyzerDatePickersAndSourcererDisabled\" | \"graphVisualizationInFlyoutEnabled\" | \"prebuiltRulesCustomizationEnabled\" | \"malwareOnWriteScanOptionAvailable\" | \"unifiedManifestEnabled\" | \"valueListItemsModalEnabled\" | \"filterProcessDescendantsForEventFiltersEnabled\" | \"dataIngestionHubEnabled\" | \"entityStoreDisabled\" | \"siemMigrationsEnabled\" | undefined" + "\"assistantModelEvaluation\" | \"excludePoliciesInFilterEnabled\" | \"kubernetesEnabled\" | \"donutChartEmbeddablesEnabled\" | \"previewTelemetryUrlEnabled\" | \"extendedRuleExecutionLoggingEnabled\" | \"socTrendsEnabled\" | \"responseActionUploadEnabled\" | \"automatedProcessActionsEnabled\" | \"responseActionsSentinelOneV1Enabled\" | \"responseActionsSentinelOneV2Enabled\" | \"responseActionsSentinelOneGetFileEnabled\" | \"responseActionsSentinelOneKillProcessEnabled\" | \"responseActionsSentinelOneProcessesEnabled\" | \"responseActionsCrowdstrikeManualHostIsolationEnabled\" | \"endpointManagementSpaceAwarenessEnabled\" | \"securitySolutionNotesDisabled\" | \"entityAlertPreviewDisabled\" | \"newUserDetailsFlyoutManagedUser\" | \"riskScoringPersistence\" | \"riskScoringRoutesEnabled\" | \"esqlRulesDisabled\" | \"protectionUpdatesEnabled\" | \"disableTimelineSaveTour\" | \"riskEnginePrivilegesRouteEnabled\" | \"sentinelOneDataInAnalyzerEnabled\" | \"sentinelOneManualHostActionsEnabled\" | \"crowdstrikeDataInAnalyzerEnabled\" | \"responseActionsTelemetryEnabled\" | \"jamfDataInAnalyzerEnabled\" | \"timelineEsqlTabDisabled\" | \"analyzerDatePickersAndSourcererDisabled\" | \"graphVisualizationInFlyoutEnabled\" | \"prebuiltRulesCustomizationEnabled\" | \"malwareOnWriteScanOptionAvailable\" | \"unifiedManifestEnabled\" | \"valueListItemsModalEnabled\" | \"filterProcessDescendantsForEventFiltersEnabled\" | \"dataIngestionHubEnabled\" | \"entityStoreDisabled\" | \"siemMigrationsEnabled\" | undefined" ], "path": "x-pack/plugins/security_solution/public/common/links/types.ts", "deprecated": false, @@ -500,7 +500,7 @@ "\nExperimental flag needed to disable the link. Opposite of experimentalKey" ], "signature": [ - "\"assistantKnowledgeBaseByDefault\" | \"assistantModelEvaluation\" | \"excludePoliciesInFilterEnabled\" | \"kubernetesEnabled\" | \"donutChartEmbeddablesEnabled\" | \"previewTelemetryUrlEnabled\" | \"extendedRuleExecutionLoggingEnabled\" | \"socTrendsEnabled\" | \"responseActionUploadEnabled\" | \"automatedProcessActionsEnabled\" | \"responseActionsSentinelOneV1Enabled\" | \"responseActionsSentinelOneV2Enabled\" | \"responseActionsSentinelOneGetFileEnabled\" | \"responseActionsSentinelOneKillProcessEnabled\" | \"responseActionsSentinelOneProcessesEnabled\" | \"responseActionsCrowdstrikeManualHostIsolationEnabled\" | \"endpointManagementSpaceAwarenessEnabled\" | \"securitySolutionNotesDisabled\" | \"entityAlertPreviewDisabled\" | \"newUserDetailsFlyoutManagedUser\" | \"riskScoringPersistence\" | \"riskScoringRoutesEnabled\" | \"esqlRulesDisabled\" | \"protectionUpdatesEnabled\" | \"disableTimelineSaveTour\" | \"riskEnginePrivilegesRouteEnabled\" | \"sentinelOneDataInAnalyzerEnabled\" | \"sentinelOneManualHostActionsEnabled\" | \"crowdstrikeDataInAnalyzerEnabled\" | \"responseActionsTelemetryEnabled\" | \"jamfDataInAnalyzerEnabled\" | \"timelineEsqlTabDisabled\" | \"analyzerDatePickersAndSourcererDisabled\" | \"graphVisualizationInFlyoutEnabled\" | \"prebuiltRulesCustomizationEnabled\" | \"malwareOnWriteScanOptionAvailable\" | \"unifiedManifestEnabled\" | \"valueListItemsModalEnabled\" | \"filterProcessDescendantsForEventFiltersEnabled\" | \"dataIngestionHubEnabled\" | \"entityStoreDisabled\" | \"siemMigrationsEnabled\" | undefined" + "\"assistantModelEvaluation\" | \"excludePoliciesInFilterEnabled\" | \"kubernetesEnabled\" | \"donutChartEmbeddablesEnabled\" | \"previewTelemetryUrlEnabled\" | \"extendedRuleExecutionLoggingEnabled\" | \"socTrendsEnabled\" | \"responseActionUploadEnabled\" | \"automatedProcessActionsEnabled\" | \"responseActionsSentinelOneV1Enabled\" | \"responseActionsSentinelOneV2Enabled\" | \"responseActionsSentinelOneGetFileEnabled\" | \"responseActionsSentinelOneKillProcessEnabled\" | \"responseActionsSentinelOneProcessesEnabled\" | \"responseActionsCrowdstrikeManualHostIsolationEnabled\" | \"endpointManagementSpaceAwarenessEnabled\" | \"securitySolutionNotesDisabled\" | \"entityAlertPreviewDisabled\" | \"newUserDetailsFlyoutManagedUser\" | \"riskScoringPersistence\" | \"riskScoringRoutesEnabled\" | \"esqlRulesDisabled\" | \"protectionUpdatesEnabled\" | \"disableTimelineSaveTour\" | \"riskEnginePrivilegesRouteEnabled\" | \"sentinelOneDataInAnalyzerEnabled\" | \"sentinelOneManualHostActionsEnabled\" | \"crowdstrikeDataInAnalyzerEnabled\" | \"responseActionsTelemetryEnabled\" | \"jamfDataInAnalyzerEnabled\" | \"timelineEsqlTabDisabled\" | \"analyzerDatePickersAndSourcererDisabled\" | \"graphVisualizationInFlyoutEnabled\" | \"prebuiltRulesCustomizationEnabled\" | \"malwareOnWriteScanOptionAvailable\" | \"unifiedManifestEnabled\" | \"valueListItemsModalEnabled\" | \"filterProcessDescendantsForEventFiltersEnabled\" | \"dataIngestionHubEnabled\" | \"entityStoreDisabled\" | \"siemMigrationsEnabled\" | undefined" ], "path": "x-pack/plugins/security_solution/public/common/links/types.ts", "deprecated": false, @@ -1791,7 +1791,7 @@ "label": "experimentalFeatures", "description": [], "signature": [ - "{ readonly excludePoliciesInFilterEnabled: boolean; readonly kubernetesEnabled: boolean; readonly donutChartEmbeddablesEnabled: boolean; readonly previewTelemetryUrlEnabled: boolean; readonly extendedRuleExecutionLoggingEnabled: boolean; readonly socTrendsEnabled: boolean; readonly responseActionUploadEnabled: boolean; readonly automatedProcessActionsEnabled: boolean; readonly responseActionsSentinelOneV1Enabled: boolean; readonly responseActionsSentinelOneV2Enabled: boolean; readonly responseActionsSentinelOneGetFileEnabled: boolean; readonly responseActionsSentinelOneKillProcessEnabled: boolean; readonly responseActionsSentinelOneProcessesEnabled: boolean; readonly responseActionsCrowdstrikeManualHostIsolationEnabled: boolean; readonly endpointManagementSpaceAwarenessEnabled: boolean; readonly securitySolutionNotesDisabled: boolean; readonly entityAlertPreviewDisabled: boolean; readonly assistantModelEvaluation: boolean; readonly assistantKnowledgeBaseByDefault: boolean; readonly newUserDetailsFlyoutManagedUser: boolean; readonly riskScoringPersistence: boolean; readonly riskScoringRoutesEnabled: boolean; readonly esqlRulesDisabled: boolean; readonly protectionUpdatesEnabled: boolean; readonly disableTimelineSaveTour: boolean; readonly riskEnginePrivilegesRouteEnabled: boolean; readonly sentinelOneDataInAnalyzerEnabled: boolean; readonly sentinelOneManualHostActionsEnabled: boolean; readonly crowdstrikeDataInAnalyzerEnabled: boolean; readonly responseActionsTelemetryEnabled: boolean; readonly jamfDataInAnalyzerEnabled: boolean; readonly timelineEsqlTabDisabled: boolean; readonly analyzerDatePickersAndSourcererDisabled: boolean; readonly graphVisualizationInFlyoutEnabled: boolean; readonly prebuiltRulesCustomizationEnabled: boolean; readonly malwareOnWriteScanOptionAvailable: boolean; readonly unifiedManifestEnabled: boolean; readonly valueListItemsModalEnabled: boolean; readonly filterProcessDescendantsForEventFiltersEnabled: boolean; readonly dataIngestionHubEnabled: boolean; readonly entityStoreDisabled: boolean; readonly siemMigrationsEnabled: boolean; }" + "{ readonly excludePoliciesInFilterEnabled: boolean; readonly kubernetesEnabled: boolean; readonly donutChartEmbeddablesEnabled: boolean; readonly previewTelemetryUrlEnabled: boolean; readonly extendedRuleExecutionLoggingEnabled: boolean; readonly socTrendsEnabled: boolean; readonly responseActionUploadEnabled: boolean; readonly automatedProcessActionsEnabled: boolean; readonly responseActionsSentinelOneV1Enabled: boolean; readonly responseActionsSentinelOneV2Enabled: boolean; readonly responseActionsSentinelOneGetFileEnabled: boolean; readonly responseActionsSentinelOneKillProcessEnabled: boolean; readonly responseActionsSentinelOneProcessesEnabled: boolean; readonly responseActionsCrowdstrikeManualHostIsolationEnabled: boolean; readonly endpointManagementSpaceAwarenessEnabled: boolean; readonly securitySolutionNotesDisabled: boolean; readonly entityAlertPreviewDisabled: boolean; readonly assistantModelEvaluation: boolean; readonly newUserDetailsFlyoutManagedUser: boolean; readonly riskScoringPersistence: boolean; readonly riskScoringRoutesEnabled: boolean; readonly esqlRulesDisabled: boolean; readonly protectionUpdatesEnabled: boolean; readonly disableTimelineSaveTour: boolean; readonly riskEnginePrivilegesRouteEnabled: boolean; readonly sentinelOneDataInAnalyzerEnabled: boolean; readonly sentinelOneManualHostActionsEnabled: boolean; readonly crowdstrikeDataInAnalyzerEnabled: boolean; readonly responseActionsTelemetryEnabled: boolean; readonly jamfDataInAnalyzerEnabled: boolean; readonly timelineEsqlTabDisabled: boolean; readonly analyzerDatePickersAndSourcererDisabled: boolean; readonly graphVisualizationInFlyoutEnabled: boolean; readonly prebuiltRulesCustomizationEnabled: boolean; readonly malwareOnWriteScanOptionAvailable: boolean; readonly unifiedManifestEnabled: boolean; readonly valueListItemsModalEnabled: boolean; readonly filterProcessDescendantsForEventFiltersEnabled: boolean; readonly dataIngestionHubEnabled: boolean; readonly entityStoreDisabled: boolean; readonly siemMigrationsEnabled: boolean; }" ], "path": "x-pack/plugins/security_solution/public/types.ts", "deprecated": false, @@ -3039,7 +3039,7 @@ "\nThe security solution generic experimental features" ], "signature": [ - "{ readonly excludePoliciesInFilterEnabled: boolean; readonly kubernetesEnabled: boolean; readonly donutChartEmbeddablesEnabled: boolean; readonly previewTelemetryUrlEnabled: boolean; readonly extendedRuleExecutionLoggingEnabled: boolean; readonly socTrendsEnabled: boolean; readonly responseActionUploadEnabled: boolean; readonly automatedProcessActionsEnabled: boolean; readonly responseActionsSentinelOneV1Enabled: boolean; readonly responseActionsSentinelOneV2Enabled: boolean; readonly responseActionsSentinelOneGetFileEnabled: boolean; readonly responseActionsSentinelOneKillProcessEnabled: boolean; readonly responseActionsSentinelOneProcessesEnabled: boolean; readonly responseActionsCrowdstrikeManualHostIsolationEnabled: boolean; readonly endpointManagementSpaceAwarenessEnabled: boolean; readonly securitySolutionNotesDisabled: boolean; readonly entityAlertPreviewDisabled: boolean; readonly assistantModelEvaluation: boolean; readonly assistantKnowledgeBaseByDefault: boolean; readonly newUserDetailsFlyoutManagedUser: boolean; readonly riskScoringPersistence: boolean; readonly riskScoringRoutesEnabled: boolean; readonly esqlRulesDisabled: boolean; readonly protectionUpdatesEnabled: boolean; readonly disableTimelineSaveTour: boolean; readonly riskEnginePrivilegesRouteEnabled: boolean; readonly sentinelOneDataInAnalyzerEnabled: boolean; readonly sentinelOneManualHostActionsEnabled: boolean; readonly crowdstrikeDataInAnalyzerEnabled: boolean; readonly responseActionsTelemetryEnabled: boolean; readonly jamfDataInAnalyzerEnabled: boolean; readonly timelineEsqlTabDisabled: boolean; readonly analyzerDatePickersAndSourcererDisabled: boolean; readonly graphVisualizationInFlyoutEnabled: boolean; readonly prebuiltRulesCustomizationEnabled: boolean; readonly malwareOnWriteScanOptionAvailable: boolean; readonly unifiedManifestEnabled: boolean; readonly valueListItemsModalEnabled: boolean; readonly filterProcessDescendantsForEventFiltersEnabled: boolean; readonly dataIngestionHubEnabled: boolean; readonly entityStoreDisabled: boolean; readonly siemMigrationsEnabled: boolean; }" + "{ readonly excludePoliciesInFilterEnabled: boolean; readonly kubernetesEnabled: boolean; readonly donutChartEmbeddablesEnabled: boolean; readonly previewTelemetryUrlEnabled: boolean; readonly extendedRuleExecutionLoggingEnabled: boolean; readonly socTrendsEnabled: boolean; readonly responseActionUploadEnabled: boolean; readonly automatedProcessActionsEnabled: boolean; readonly responseActionsSentinelOneV1Enabled: boolean; readonly responseActionsSentinelOneV2Enabled: boolean; readonly responseActionsSentinelOneGetFileEnabled: boolean; readonly responseActionsSentinelOneKillProcessEnabled: boolean; readonly responseActionsSentinelOneProcessesEnabled: boolean; readonly responseActionsCrowdstrikeManualHostIsolationEnabled: boolean; readonly endpointManagementSpaceAwarenessEnabled: boolean; readonly securitySolutionNotesDisabled: boolean; readonly entityAlertPreviewDisabled: boolean; readonly assistantModelEvaluation: boolean; readonly newUserDetailsFlyoutManagedUser: boolean; readonly riskScoringPersistence: boolean; readonly riskScoringRoutesEnabled: boolean; readonly esqlRulesDisabled: boolean; readonly protectionUpdatesEnabled: boolean; readonly disableTimelineSaveTour: boolean; readonly riskEnginePrivilegesRouteEnabled: boolean; readonly sentinelOneDataInAnalyzerEnabled: boolean; readonly sentinelOneManualHostActionsEnabled: boolean; readonly crowdstrikeDataInAnalyzerEnabled: boolean; readonly responseActionsTelemetryEnabled: boolean; readonly jamfDataInAnalyzerEnabled: boolean; readonly timelineEsqlTabDisabled: boolean; readonly analyzerDatePickersAndSourcererDisabled: boolean; readonly graphVisualizationInFlyoutEnabled: boolean; readonly prebuiltRulesCustomizationEnabled: boolean; readonly malwareOnWriteScanOptionAvailable: boolean; readonly unifiedManifestEnabled: boolean; readonly valueListItemsModalEnabled: boolean; readonly filterProcessDescendantsForEventFiltersEnabled: boolean; readonly dataIngestionHubEnabled: boolean; readonly entityStoreDisabled: boolean; readonly siemMigrationsEnabled: boolean; }" ], "path": "x-pack/plugins/security_solution/server/plugin_contract.ts", "deprecated": false, @@ -3212,7 +3212,7 @@ "label": "ExperimentalFeatures", "description": [], "signature": [ - "{ readonly excludePoliciesInFilterEnabled: boolean; readonly kubernetesEnabled: boolean; readonly donutChartEmbeddablesEnabled: boolean; readonly previewTelemetryUrlEnabled: boolean; readonly extendedRuleExecutionLoggingEnabled: boolean; readonly socTrendsEnabled: boolean; readonly responseActionUploadEnabled: boolean; readonly automatedProcessActionsEnabled: boolean; readonly responseActionsSentinelOneV1Enabled: boolean; readonly responseActionsSentinelOneV2Enabled: boolean; readonly responseActionsSentinelOneGetFileEnabled: boolean; readonly responseActionsSentinelOneKillProcessEnabled: boolean; readonly responseActionsSentinelOneProcessesEnabled: boolean; readonly responseActionsCrowdstrikeManualHostIsolationEnabled: boolean; readonly endpointManagementSpaceAwarenessEnabled: boolean; readonly securitySolutionNotesDisabled: boolean; readonly entityAlertPreviewDisabled: boolean; readonly assistantModelEvaluation: boolean; readonly assistantKnowledgeBaseByDefault: boolean; readonly newUserDetailsFlyoutManagedUser: boolean; readonly riskScoringPersistence: boolean; readonly riskScoringRoutesEnabled: boolean; readonly esqlRulesDisabled: boolean; readonly protectionUpdatesEnabled: boolean; readonly disableTimelineSaveTour: boolean; readonly riskEnginePrivilegesRouteEnabled: boolean; readonly sentinelOneDataInAnalyzerEnabled: boolean; readonly sentinelOneManualHostActionsEnabled: boolean; readonly crowdstrikeDataInAnalyzerEnabled: boolean; readonly responseActionsTelemetryEnabled: boolean; readonly jamfDataInAnalyzerEnabled: boolean; readonly timelineEsqlTabDisabled: boolean; readonly analyzerDatePickersAndSourcererDisabled: boolean; readonly graphVisualizationInFlyoutEnabled: boolean; readonly prebuiltRulesCustomizationEnabled: boolean; readonly malwareOnWriteScanOptionAvailable: boolean; readonly unifiedManifestEnabled: boolean; readonly valueListItemsModalEnabled: boolean; readonly filterProcessDescendantsForEventFiltersEnabled: boolean; readonly dataIngestionHubEnabled: boolean; readonly entityStoreDisabled: boolean; readonly siemMigrationsEnabled: boolean; }" + "{ readonly excludePoliciesInFilterEnabled: boolean; readonly kubernetesEnabled: boolean; readonly donutChartEmbeddablesEnabled: boolean; readonly previewTelemetryUrlEnabled: boolean; readonly extendedRuleExecutionLoggingEnabled: boolean; readonly socTrendsEnabled: boolean; readonly responseActionUploadEnabled: boolean; readonly automatedProcessActionsEnabled: boolean; readonly responseActionsSentinelOneV1Enabled: boolean; readonly responseActionsSentinelOneV2Enabled: boolean; readonly responseActionsSentinelOneGetFileEnabled: boolean; readonly responseActionsSentinelOneKillProcessEnabled: boolean; readonly responseActionsSentinelOneProcessesEnabled: boolean; readonly responseActionsCrowdstrikeManualHostIsolationEnabled: boolean; readonly endpointManagementSpaceAwarenessEnabled: boolean; readonly securitySolutionNotesDisabled: boolean; readonly entityAlertPreviewDisabled: boolean; readonly assistantModelEvaluation: boolean; readonly newUserDetailsFlyoutManagedUser: boolean; readonly riskScoringPersistence: boolean; readonly riskScoringRoutesEnabled: boolean; readonly esqlRulesDisabled: boolean; readonly protectionUpdatesEnabled: boolean; readonly disableTimelineSaveTour: boolean; readonly riskEnginePrivilegesRouteEnabled: boolean; readonly sentinelOneDataInAnalyzerEnabled: boolean; readonly sentinelOneManualHostActionsEnabled: boolean; readonly crowdstrikeDataInAnalyzerEnabled: boolean; readonly responseActionsTelemetryEnabled: boolean; readonly jamfDataInAnalyzerEnabled: boolean; readonly timelineEsqlTabDisabled: boolean; readonly analyzerDatePickersAndSourcererDisabled: boolean; readonly graphVisualizationInFlyoutEnabled: boolean; readonly prebuiltRulesCustomizationEnabled: boolean; readonly malwareOnWriteScanOptionAvailable: boolean; readonly unifiedManifestEnabled: boolean; readonly valueListItemsModalEnabled: boolean; readonly filterProcessDescendantsForEventFiltersEnabled: boolean; readonly dataIngestionHubEnabled: boolean; readonly entityStoreDisabled: boolean; readonly siemMigrationsEnabled: boolean; }" ], "path": "x-pack/plugins/security_solution/common/experimental_features.ts", "deprecated": false, @@ -3278,7 +3278,7 @@ "\nA list of allowed values that can be used in `xpack.securitySolution.enableExperimental`.\nThis object is then used to validate and parse the value entered." ], "signature": [ - "{ readonly excludePoliciesInFilterEnabled: false; readonly kubernetesEnabled: true; readonly donutChartEmbeddablesEnabled: false; readonly previewTelemetryUrlEnabled: false; readonly extendedRuleExecutionLoggingEnabled: false; readonly socTrendsEnabled: false; readonly responseActionUploadEnabled: true; readonly automatedProcessActionsEnabled: true; readonly responseActionsSentinelOneV1Enabled: true; readonly responseActionsSentinelOneV2Enabled: true; readonly responseActionsSentinelOneGetFileEnabled: true; readonly responseActionsSentinelOneKillProcessEnabled: true; readonly responseActionsSentinelOneProcessesEnabled: true; readonly responseActionsCrowdstrikeManualHostIsolationEnabled: true; readonly endpointManagementSpaceAwarenessEnabled: false; readonly securitySolutionNotesDisabled: false; readonly entityAlertPreviewDisabled: false; readonly assistantModelEvaluation: false; readonly assistantKnowledgeBaseByDefault: true; readonly newUserDetailsFlyoutManagedUser: false; readonly riskScoringPersistence: true; readonly riskScoringRoutesEnabled: true; readonly esqlRulesDisabled: false; readonly protectionUpdatesEnabled: true; readonly disableTimelineSaveTour: false; readonly riskEnginePrivilegesRouteEnabled: true; readonly sentinelOneDataInAnalyzerEnabled: true; readonly sentinelOneManualHostActionsEnabled: true; readonly crowdstrikeDataInAnalyzerEnabled: true; readonly responseActionsTelemetryEnabled: false; readonly jamfDataInAnalyzerEnabled: true; readonly timelineEsqlTabDisabled: false; readonly analyzerDatePickersAndSourcererDisabled: false; readonly graphVisualizationInFlyoutEnabled: false; readonly prebuiltRulesCustomizationEnabled: false; readonly malwareOnWriteScanOptionAvailable: true; readonly unifiedManifestEnabled: true; readonly valueListItemsModalEnabled: true; readonly filterProcessDescendantsForEventFiltersEnabled: true; readonly dataIngestionHubEnabled: false; readonly entityStoreDisabled: false; readonly siemMigrationsEnabled: false; }" + "{ readonly excludePoliciesInFilterEnabled: false; readonly kubernetesEnabled: true; readonly donutChartEmbeddablesEnabled: false; readonly previewTelemetryUrlEnabled: false; readonly extendedRuleExecutionLoggingEnabled: false; readonly socTrendsEnabled: false; readonly responseActionUploadEnabled: true; readonly automatedProcessActionsEnabled: true; readonly responseActionsSentinelOneV1Enabled: true; readonly responseActionsSentinelOneV2Enabled: true; readonly responseActionsSentinelOneGetFileEnabled: true; readonly responseActionsSentinelOneKillProcessEnabled: true; readonly responseActionsSentinelOneProcessesEnabled: true; readonly responseActionsCrowdstrikeManualHostIsolationEnabled: true; readonly endpointManagementSpaceAwarenessEnabled: false; readonly securitySolutionNotesDisabled: false; readonly entityAlertPreviewDisabled: false; readonly assistantModelEvaluation: false; readonly newUserDetailsFlyoutManagedUser: false; readonly riskScoringPersistence: true; readonly riskScoringRoutesEnabled: true; readonly esqlRulesDisabled: false; readonly protectionUpdatesEnabled: true; readonly disableTimelineSaveTour: false; readonly riskEnginePrivilegesRouteEnabled: true; readonly sentinelOneDataInAnalyzerEnabled: true; readonly sentinelOneManualHostActionsEnabled: true; readonly crowdstrikeDataInAnalyzerEnabled: true; readonly responseActionsTelemetryEnabled: false; readonly jamfDataInAnalyzerEnabled: true; readonly timelineEsqlTabDisabled: false; readonly analyzerDatePickersAndSourcererDisabled: false; readonly graphVisualizationInFlyoutEnabled: false; readonly prebuiltRulesCustomizationEnabled: false; readonly malwareOnWriteScanOptionAvailable: true; readonly unifiedManifestEnabled: true; readonly valueListItemsModalEnabled: true; readonly filterProcessDescendantsForEventFiltersEnabled: true; readonly dataIngestionHubEnabled: false; readonly entityStoreDisabled: false; readonly siemMigrationsEnabled: false; }" ], "path": "x-pack/plugins/security_solution/common/experimental_features.ts", "deprecated": false, @@ -3287,4 +3287,4 @@ } ] } -} \ No newline at end of file +} diff --git a/docs/management/action-types.asciidoc b/docs/management/action-types.asciidoc index 1357af980d278..e3b01ba0afcf7 100644 --- a/docs/management/action-types.asciidoc +++ b/docs/management/action-types.asciidoc @@ -8,6 +8,12 @@ Actions are instantiations of a connector that are linked to rules and run as ba [cols="2"] |=== +// ifeval::["featureAIConnector"=="true"] +// a| <> + +// | Send a request to {infer}. +// endif::[] + a| <> | Send a request to {bedrock}. @@ -28,10 +34,6 @@ a| <> | Send a request to {gemini}. -a| <> - -| Send a request to {infer}. - a| <> | Send email from your server. diff --git a/docs/management/connectors/action-types/inference.asciidoc b/docs/management/connectors/action-types/inference.asciidoc index ea8a0be675e18..d47374e9b4cdd 100644 --- a/docs/management/connectors/action-types/inference.asciidoc +++ b/docs/management/connectors/action-types/inference.asciidoc @@ -1,13 +1,14 @@ [[inference-action-type]] -== {infer-cap} connector and action +== AI connector and action ++++ -{infer-cap} +AI ++++ :frontmatter-description: Add a connector that can send requests to {inference}. :frontmatter-tags-products: [kibana] :frontmatter-tags-content-type: [how-to] :frontmatter-tags-user-goals: [configure] +coming::[] The {infer} connector uses the {es} client to send requests to an {infer} service. The connector uses the <> to send the request. @@ -16,6 +17,7 @@ The connector uses the <> to send the r [[define-inference-ui]] === Create connectors in {kib} +// TBD After you set the `xpack.stack_connectors.enableExperimental` to include `inferenceConnectorOn`, You can create connectors in *{stack-manage-app} > {connectors-ui}*. For example: [role="screenshot"] diff --git a/docs/management/connectors/index.asciidoc b/docs/management/connectors/index.asciidoc index c5233ad4f4934..c5c1ce4600c5d 100644 --- a/docs/management/connectors/index.asciidoc +++ b/docs/management/connectors/index.asciidoc @@ -4,7 +4,9 @@ include::action-types/crowdstrike.asciidoc[leveloffset=+1] include::action-types/d3security.asciidoc[leveloffset=+1] include::action-types/email.asciidoc[leveloffset=+1] include::action-types/gemini.asciidoc[leveloffset=+1] -include::action-types/inference.asciidoc[leveloffset=+1] +// ifeval::["featureAIConnector"=="true"] +// include::action-types/inference.asciidoc[leveloffset=+1] +// endif::[] include::action-types/resilient.asciidoc[leveloffset=+1] include::action-types/index.asciidoc[leveloffset=+1] include::action-types/jira.asciidoc[leveloffset=+1] diff --git a/docs/settings/alert-action-settings.asciidoc b/docs/settings/alert-action-settings.asciidoc index e0fa3f0aab860..0f4987822dc32 100644 --- a/docs/settings/alert-action-settings.asciidoc +++ b/docs/settings/alert-action-settings.asciidoc @@ -269,7 +269,6 @@ A configuration URL that varies by connector: -- * For an <>, specifies the {bedrock} request URL. * For an <>, specifies the {gemini} request URL. -* For an <>, specifies the Elastic {inference} request. * For a <>, specifies the OpenAI request URL. * For a <>, specifies the {ibm-r} instance URL. * For a <>, specifies the Jira instance URL. @@ -277,7 +276,9 @@ A configuration URL that varies by connector: * For a <>, specifies the PagerDuty event URL. Defaults to `https://events.pagerduty.com/v2/enqueue`. * For a <>, <>, or <> specifies the ServiceNow instance URL. * For a <>, specifies the {swimlane} instance URL. - +// ifeval::["featureAIConnector"=="true"] +// * For an <>, specifies the Elastic {inference} request. +// endif::[] NOTE: If you are using the `xpack.actions.allowedHosts` setting, make sure the hostname in the URL is added to the allowed hosts. -- diff --git a/oas_docs/examples/get_connector_types_generativeai_response.yaml b/oas_docs/examples/get_connector_types_generativeai_response.yaml index 8299da3558150..a97199e0a3927 100644 --- a/oas_docs/examples/get_connector_types_generativeai_response.yaml +++ b/oas_docs/examples/get_connector_types_generativeai_response.yaml @@ -31,14 +31,3 @@ value: supported_feature_ids: - generativeAIForSecurity is_system_action_type: false - - id: .inference - name: Inference API - enabled: true - enabled_in_config: true - enabled_in_license: true - minimum_license_required: enterprise - supported_feature_ids: - - generativeAIForSecurity - - generativeAIForObservability - - generativeAIForSearchPlayground - is_system_action_type: false diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml index 5f18154db449d..95c201de052c2 100644 --- a/oas_docs/output/kibana.serverless.yaml +++ b/oas_docs/output/kibana.serverless.yaml @@ -39976,17 +39976,20 @@ components: type: object properties: unit: - enum: - - s - - m - - h - type: string + $ref: >- + #/components/schemas/Security_Detections_API_AlertSuppressionDurationUnit value: minimum: 1 type: integer required: - value - unit + Security_Detections_API_AlertSuppressionDurationUnit: + enum: + - s + - m + - h + type: string Security_Detections_API_AlertSuppressionGroupBy: items: type: string diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml index 133dede5fcd0c..8b4953b4149a2 100644 --- a/oas_docs/output/kibana.yaml +++ b/oas_docs/output/kibana.yaml @@ -48220,17 +48220,20 @@ components: type: object properties: unit: - enum: - - s - - m - - h - type: string + $ref: >- + #/components/schemas/Security_Detections_API_AlertSuppressionDurationUnit value: minimum: 1 type: integer required: - value - unit + Security_Detections_API_AlertSuppressionDurationUnit: + enum: + - s + - m + - h + type: string Security_Detections_API_AlertSuppressionGroupBy: items: type: string diff --git a/oas_docs/overlays/connectors.overlays.yaml b/oas_docs/overlays/connectors.overlays.yaml index 022946e893be2..816542a450e3a 100644 --- a/oas_docs/overlays/connectors.overlays.yaml +++ b/oas_docs/overlays/connectors.overlays.yaml @@ -140,6 +140,8 @@ actions: default: {} description: The connector configuration details. oneOf: + # AI (.inference) TBD + # - $ref: '../../x-pack/plugins/actions/docs/openapi/components/schemas/inference_config.yaml' # Bedrock (.bedrock) - $ref: '../../x-pack/plugins/actions/docs/openapi/components/schemas/bedrock_config.yaml' # Crowdstrike (.crowdstrike) @@ -194,6 +196,8 @@ actions: additionalProperties: {} default: {} oneOf: + # AI (.inference) + # - $ref: '../../x-pack/plugins/actions/docs/openapi/components/schemas/inference_secrets.yaml' # Bedrock (.bedrock) - $ref: '../../x-pack/plugins/actions/docs/openapi/components/schemas/bedrock_secrets.yaml' # Crowdstrike (.crowdstrike) @@ -253,6 +257,8 @@ actions: default: {} description: The connector configuration details. oneOf: + # AI (.inference) + # - $ref: '../../x-pack/plugins/actions/docs/openapi/components/schemas/inference_config.yaml' # Bedrock (.bedrock) - $ref: '../../x-pack/plugins/actions/docs/openapi/components/schemas/bedrock_config.yaml' # Crowdstrike (.crowdstrike) @@ -307,6 +313,8 @@ actions: additionalProperties: {} default: {} oneOf: + # AI (.inference) + # - $ref: '../../x-pack/plugins/actions/docs/openapi/components/schemas/inference_secrets.yaml' # Bedrock (.bedrock) - $ref: '../../x-pack/plugins/actions/docs/openapi/components/schemas/bedrock_secrets.yaml' # Crowdstrike (.crowdstrike) diff --git a/package.json b/package.json index 87905c955d2d8..48e377c9a6237 100644 --- a/package.json +++ b/package.json @@ -567,6 +567,7 @@ "@kbn/i18n-react": "link:packages/kbn-i18n-react", "@kbn/iframe-embedded-plugin": "link:x-pack/test/functional_embedded/plugins/iframe_embedded", "@kbn/image-embeddable-plugin": "link:src/plugins/image_embeddable", + "@kbn/index-adapter": "link:packages/kbn-index-adapter", "@kbn/index-lifecycle-management-common-shared": "link:x-pack/packages/index-lifecycle-management/index_lifecycle_management_common_shared", "@kbn/index-lifecycle-management-plugin": "link:x-pack/plugins/index_lifecycle_management", "@kbn/index-management-plugin": "link:x-pack/plugins/index_management", diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts index db95dcf4155bc..6da1decaab9ab 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts @@ -58,6 +58,8 @@ export class K8sEntity extends Serializable { 'entity.definition_id': `builtin_${entityTypeWithSchema}`, 'entity.identity_fields': identityFields, 'entity.display_name': getDisplayName({ identityFields, fields }), + 'entity.definition_version': '1.0.0', + 'entity.schema_version': '1.0', }); } } diff --git a/packages/kbn-data-stream-adapter/index.ts b/packages/kbn-data-stream-adapter/index.ts index 4fd7c7ebd1572..f03a384dca1ff 100644 --- a/packages/kbn-data-stream-adapter/index.ts +++ b/packages/kbn-data-stream-adapter/index.ts @@ -9,13 +9,13 @@ export { DataStreamAdapter } from './src/data_stream_adapter'; export { DataStreamSpacesAdapter } from './src/data_stream_spaces_adapter'; -export { retryTransientEsErrors } from './src/retry_transient_es_errors'; -export { ecsFieldMap, type EcsFieldMap } from './src/field_maps/ecs_field_map'; +export { retryTransientEsErrors, ecsFieldMap } from '@kbn/index-adapter'; export type { - DataStreamAdapterParams, SetComponentTemplateParams, SetIndexTemplateParams, InstallParams, -} from './src/data_stream_adapter'; -export * from './src/field_maps/types'; + EcsFieldMap, +} from '@kbn/index-adapter'; + +export * from '@kbn/index-adapter/src/field_maps/types'; diff --git a/packages/kbn-data-stream-adapter/kibana.jsonc b/packages/kbn-data-stream-adapter/kibana.jsonc index 99cbb458a8517..43317dca0b91e 100644 --- a/packages/kbn-data-stream-adapter/kibana.jsonc +++ b/packages/kbn-data-stream-adapter/kibana.jsonc @@ -1,5 +1,6 @@ { - "type": "shared-common", + "type": "shared-server", "id": "@kbn/data-stream-adapter", - "owner": "@elastic/security-threat-hunting-explore" + "owner": "@elastic/security-threat-hunting", + "visibility": "shared" } diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.test.ts b/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.test.ts index 97ca06b04ac83..e2141d4afb740 100644 --- a/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.test.ts +++ b/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.test.ts @@ -136,10 +136,11 @@ describe('createOrUpdateDataStream', () => { it(`should create data stream if not exists`, async () => { esClient.indices.getDataStream.mockResolvedValueOnce({ data_streams: [] }); - await createDataStream({ + await createOrUpdateDataStream({ esClient, logger, name, + totalFieldsLimit, }); expect(esClient.indices.createDataStream).toHaveBeenCalledWith({ name }); diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts b/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts index 791c99c6e3809..2b0fba3fb0ac0 100644 --- a/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts +++ b/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts @@ -11,7 +11,7 @@ import type { IndicesDataStream } from '@elastic/elasticsearch/lib/api/types'; import type { IndicesSimulateIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { Logger, ElasticsearchClient } from '@kbn/core/server'; import { get } from 'lodash'; -import { retryTransientEsErrors } from './retry_transient_es_errors'; +import { retryTransientEsErrors } from '@kbn/index-adapter'; interface UpdateIndexMappingsOpts { logger: Logger; @@ -168,7 +168,7 @@ export async function createDataStream({ esClient, name, }: CreateDataStreamParams): Promise { - logger.info(`Creating data stream - ${name}`); + logger.debug(`Checking data stream exists - ${name}`); // check if data stream exists let dataStreamExists = false; @@ -189,6 +189,7 @@ export async function createDataStream({ if (dataStreamExists) { return; } + logger.info(`Installing data stream - ${name}`); try { await retryTransientEsErrors(() => esClient.indices.createDataStream({ name }), { logger }); diff --git a/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts b/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts index 6843c181b2638..f54ed81312d75 100644 --- a/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts +++ b/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts @@ -7,145 +7,22 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import type { - ClusterPutComponentTemplateRequest, - IndicesIndexSettings, - IndicesPutIndexTemplateIndexTemplateMapping, - IndicesPutIndexTemplateRequest, -} from '@elastic/elasticsearch/lib/api/types'; -import type { Logger, ElasticsearchClient } from '@kbn/core/server'; -import type { Subject } from 'rxjs'; -import type { FieldMap } from './field_maps/types'; -import { createOrUpdateComponentTemplate } from './create_or_update_component_template'; +import { IndexAdapter, SetIndexTemplateParams, type InstallParams } from '@kbn/index-adapter'; import { createOrUpdateDataStream } from './create_or_update_data_stream'; -import { createOrUpdateIndexTemplate } from './create_or_update_index_template'; -import { InstallShutdownError, installWithTimeout } from './install_with_timeout'; -import { getComponentTemplate, getIndexTemplate } from './resource_installer_utils'; - -export interface DataStreamAdapterParams { - kibanaVersion: string; - totalFieldsLimit?: number; -} -export interface SetComponentTemplateParams { - name: string; - fieldMap: FieldMap; - settings?: IndicesIndexSettings; - dynamic?: 'strict' | boolean; -} -export interface SetIndexTemplateParams { - name: string; - componentTemplateRefs?: string[]; - namespace?: string; - template?: IndicesPutIndexTemplateIndexTemplateMapping; - hidden?: boolean; -} - -export interface GetInstallFnParams { - logger: Logger; - pluginStop$: Subject; - tasksTimeoutMs?: number; -} -export interface InstallParams { - logger: Logger; - esClient: ElasticsearchClient | Promise; - pluginStop$: Subject; - tasksTimeoutMs?: number; -} - -const DEFAULT_FIELDS_LIMIT = 2500; - -export class DataStreamAdapter { - protected readonly kibanaVersion: string; - protected readonly totalFieldsLimit: number; - protected componentTemplates: ClusterPutComponentTemplateRequest[] = []; - protected indexTemplates: IndicesPutIndexTemplateRequest[] = []; - protected installed: boolean; - - constructor(protected readonly name: string, options: DataStreamAdapterParams) { - this.installed = false; - this.kibanaVersion = options.kibanaVersion; - this.totalFieldsLimit = options.totalFieldsLimit ?? DEFAULT_FIELDS_LIMIT; - } - - public setComponentTemplate(params: SetComponentTemplateParams) { - if (this.installed) { - throw new Error('Cannot set component template after install'); - } - this.componentTemplates.push(getComponentTemplate(params)); - } +export class DataStreamAdapter extends IndexAdapter { public setIndexTemplate(params: SetIndexTemplateParams) { - if (this.installed) { - throw new Error('Cannot set index template after install'); - } - this.indexTemplates.push( - getIndexTemplate({ - ...params, - indexPatterns: [this.name], - kibanaVersion: this.kibanaVersion, - totalFieldsLimit: this.totalFieldsLimit, - }) - ); - } - - protected getInstallFn({ logger, pluginStop$, tasksTimeoutMs }: GetInstallFnParams) { - return async (promise: Promise, description?: string): Promise => { - try { - await installWithTimeout({ - installFn: () => promise, - description, - timeoutMs: tasksTimeoutMs, - pluginStop$, - }); - } catch (err) { - if (err instanceof InstallShutdownError) { - logger.info(err.message); - } else { - throw err; - } - } - }; + super.setIndexTemplate({ ...params, isDataStream: true }); } - public async install({ - logger, - esClient: esClientToResolve, - pluginStop$, - tasksTimeoutMs, - }: InstallParams) { + public async install(params: InstallParams) { this.installed = true; + const { logger, pluginStop$, tasksTimeoutMs } = params; + const esClient = await params.esClient; - const esClient = await esClientToResolve; - const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs }); - - // Install component templates in parallel - await Promise.all( - this.componentTemplates.map((componentTemplate) => - installFn( - createOrUpdateComponentTemplate({ - template: componentTemplate, - esClient, - logger, - totalFieldsLimit: this.totalFieldsLimit, - }), - `${componentTemplate.name} component template` - ) - ) - ); + await this.installTemplates(params); - // Install index templates in parallel - await Promise.all( - this.indexTemplates.map((indexTemplate) => - installFn( - createOrUpdateIndexTemplate({ - template: indexTemplate, - esClient, - logger, - }), - `${indexTemplate.name} index template` - ) - ) - ); + const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs }); // create data stream when everything is ready await installFn( diff --git a/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts b/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts index 9ea3c1a4a311f..df131920b7bf9 100644 --- a/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts +++ b/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts @@ -7,59 +7,26 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { createOrUpdateComponentTemplate } from './create_or_update_component_template'; -import { createDataStream, updateDataStreams } from './create_or_update_data_stream'; -import { createOrUpdateIndexTemplate } from './create_or_update_index_template'; import { - DataStreamAdapter, - type DataStreamAdapterParams, + IndexPatternAdapter, + type SetIndexTemplateParams, type InstallParams, -} from './data_stream_adapter'; - -export class DataStreamSpacesAdapter extends DataStreamAdapter { - private installedSpaceDataStreamName: Map>; - private _installSpace?: (spaceId: string) => Promise; + type InstallIndex, +} from '@kbn/index-adapter'; +import { createDataStream, updateDataStreams } from './create_or_update_data_stream'; - constructor(private readonly prefix: string, options: DataStreamAdapterParams) { - super(`${prefix}-*`, options); // make indexTemplate `indexPatterns` match all data stream space names - this.installedSpaceDataStreamName = new Map(); +export class DataStreamSpacesAdapter extends IndexPatternAdapter { + public setIndexTemplate(params: SetIndexTemplateParams) { + super.setIndexTemplate({ ...params, isDataStream: true }); } - public async install({ - logger, - esClient: esClientToResolve, - pluginStop$, - tasksTimeoutMs, - }: InstallParams) { - this.installed = true; + protected async _install(params: InstallParams): Promise { + const { logger, pluginStop$, tasksTimeoutMs } = params; - const esClient = await esClientToResolve; - const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs }); + await this.installTemplates(params); - // Install component templates in parallel - await Promise.all( - this.componentTemplates.map((componentTemplate) => - installFn( - createOrUpdateComponentTemplate({ - template: componentTemplate, - esClient, - logger, - totalFieldsLimit: this.totalFieldsLimit, - }), - `create or update ${componentTemplate.name} component template` - ) - ) - ); - - // Install index templates in parallel - await Promise.all( - this.indexTemplates.map((indexTemplate) => - installFn( - createOrUpdateIndexTemplate({ template: indexTemplate, esClient, logger }), - `create or update ${indexTemplate.name} index template` - ) - ) - ); + const esClient = await params.esClient; + const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs }); // Update existing space data streams await installFn( @@ -72,31 +39,21 @@ export class DataStreamSpacesAdapter extends DataStreamAdapter { `update space data streams` ); - // define function to install data stream for spaces on demand - this._installSpace = async (spaceId: string) => { - const existingInstallPromise = this.installedSpaceDataStreamName.get(spaceId); - if (existingInstallPromise) { - return existingInstallPromise; - } - const name = `${this.prefix}-${spaceId}`; - const installPromise = installFn( - createDataStream({ name, esClient, logger }), - `create ${name} data stream` - ).then(() => name); - - this.installedSpaceDataStreamName.set(spaceId, installPromise); - return installPromise; - }; + // define function to install data stream on demand + return async (name: string) => + installFn(createDataStream({ name, esClient, logger }), `create ${name} data stream`); } + /** + * Method to create the data stream for a given space ID. + * It resolves with the full data stream name. + */ public async installSpace(spaceId: string): Promise { - if (!this._installSpace) { - throw new Error('Cannot installSpace before install'); - } - return this._installSpace(spaceId); + await this.createIndex(spaceId); + return this.getIndexName(spaceId); } public async getInstalledSpaceName(spaceId: string): Promise { - return this.installedSpaceDataStreamName.get(spaceId); + return this.getInstalledIndexName(spaceId); } } diff --git a/packages/kbn-data-stream-adapter/tsconfig.json b/packages/kbn-data-stream-adapter/tsconfig.json index 7eded8e71bef4..8c8bcce97fe74 100644 --- a/packages/kbn-data-stream-adapter/tsconfig.json +++ b/packages/kbn-data-stream-adapter/tsconfig.json @@ -5,18 +5,14 @@ "types": [ "jest", "node", - "react", - "@emotion/react/types/css-prop", - "@testing-library/jest-dom", - "@testing-library/react" ] }, - "include": ["**/*.ts", "**/*.tsx"], + "include": ["**/*.ts"], "kbn_references": [ "@kbn/core", - "@kbn/std", - "@kbn/safer-lodash-set", - "@kbn/logging-mocks", + "@kbn/index-adapter", + ], + "exclude": [ + "target/**/*" ], - "exclude": ["target/**/*"] } diff --git a/packages/kbn-index-adapter/README.md b/packages/kbn-index-adapter/README.md new file mode 100644 index 0000000000000..e3eb455c2e2cc --- /dev/null +++ b/packages/kbn-index-adapter/README.md @@ -0,0 +1,59 @@ +# @kbn/index-adapter + +Utility library for Elasticsearch index management. + +## IndexAdapter + +Manage single index. Example: + +``` +// Setup +const indexAdapter = new IndexAdapter('my-awesome-index', { kibanaVersion: '8.12.1' }); + +indexAdapter.setComponentTemplate({ + name: 'awesome-component-template', + fieldMap: { + 'awesome.field1: { type: 'keyword', required: true }, + 'awesome.nested.field2: { type: 'number', required: false }, + // ... + }, +}); + +indexAdapter.setIndexTemplate({ + name: 'awesome-index-template', + componentTemplateRefs: ['awesome-component-template', 'ecs-component-template'], +}); + +// Start +await indexAdapter.install({ logger, esClient, pluginStop$ }); // Installs templates and the 'my-awesome-index' index, or updates existing. +``` + + +## IndexPatternAdapter + +Manage index patterns. Example: + +``` +// Setup +const indexPatternAdapter = new IndexPatternAdapter('my-awesome-index', { kibanaVersion: '8.12.1' }); + +indexPatternAdapter.setComponentTemplate({ + name: 'awesome-component-template', + fieldMap: { + 'awesome.field1: { type: 'keyword', required: true }, + 'awesome.nested.field2: { type: 'number', required: false }, + // ... + }, +}); + +indexPatternAdapter.setIndexTemplate({ + name: 'awesome-index-template', + componentTemplateRefs: ['awesome-component-template', 'ecs-component-template'], +}); + +// Start +indexPatternAdapter.install({ logger, esClient, pluginStop$ }); // Installs/updates templates for the index pattern 'my-awesome-index-*', and updates mappings of all specific indices + +// Create a specific index on the fly +await indexPatternAdapter.installIndex('12345'); // creates 'my-awesome-index-12345' index if it does not exist. +``` diff --git a/packages/kbn-index-adapter/index.ts b/packages/kbn-index-adapter/index.ts new file mode 100644 index 0000000000000..6956792135282 --- /dev/null +++ b/packages/kbn-index-adapter/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +export { IndexAdapter } from './src/index_adapter'; +export { IndexPatternAdapter, type InstallIndex } from './src/index_pattern_adapter'; +export { retryTransientEsErrors } from './src/retry_transient_es_errors'; +export { ecsFieldMap, type EcsFieldMap } from './src/field_maps/ecs_field_map'; +export { createOrUpdateIndexTemplate } from './src/create_or_update_index_template'; +export { createOrUpdateComponentTemplate } from './src/create_or_update_component_template'; + +export type { + SetComponentTemplateParams, + SetIndexTemplateParams, + IndexAdapterParams, + InstallParams, +} from './src/index_adapter'; +export * from './src/field_maps/types'; diff --git a/packages/kbn-index-adapter/jest.config.js b/packages/kbn-index-adapter/jest.config.js new file mode 100644 index 0000000000000..bf08ec1526382 --- /dev/null +++ b/packages/kbn-index-adapter/jest.config.js @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../..', + roots: ['/packages/kbn-index-adapter'], +}; diff --git a/packages/kbn-index-adapter/kibana.jsonc b/packages/kbn-index-adapter/kibana.jsonc new file mode 100644 index 0000000000000..575d95f5a3e39 --- /dev/null +++ b/packages/kbn-index-adapter/kibana.jsonc @@ -0,0 +1,6 @@ +{ + "type": "shared-server", + "id": "@kbn/index-adapter", + "owner": "@elastic/security-threat-hunting", + "visibility": "shared" +} diff --git a/packages/kbn-index-adapter/package.json b/packages/kbn-index-adapter/package.json new file mode 100644 index 0000000000000..70b79abe1b571 --- /dev/null +++ b/packages/kbn-index-adapter/package.json @@ -0,0 +1,7 @@ +{ + "name": "@kbn/index-adapter", + "version": "1.0.0", + "description": "Utility library for Elasticsearch index management", + "license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0", + "private": true +} \ No newline at end of file diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_component_template.test.ts b/packages/kbn-index-adapter/src/create_or_update_component_template.test.ts similarity index 100% rename from packages/kbn-data-stream-adapter/src/create_or_update_component_template.test.ts rename to packages/kbn-index-adapter/src/create_or_update_component_template.test.ts diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_component_template.ts b/packages/kbn-index-adapter/src/create_or_update_component_template.ts similarity index 100% rename from packages/kbn-data-stream-adapter/src/create_or_update_component_template.ts rename to packages/kbn-index-adapter/src/create_or_update_component_template.ts diff --git a/packages/kbn-index-adapter/src/create_or_update_index.test.ts b/packages/kbn-index-adapter/src/create_or_update_index.test.ts new file mode 100644 index 0000000000000..6c32b183e1fda --- /dev/null +++ b/packages/kbn-index-adapter/src/create_or_update_index.test.ts @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { updateIndices, createIndex, createOrUpdateIndex } from './create_or_update_index'; + +const logger = loggingSystemMock.createLogger(); +const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + +esClient.indices.putMapping.mockResolvedValue({ acknowledged: true }); +esClient.indices.putSettings.mockResolvedValue({ acknowledged: true }); + +const simulateIndexTemplateResponse = { template: { mappings: {}, settings: {}, aliases: {} } }; +esClient.indices.simulateIndexTemplate.mockResolvedValue(simulateIndexTemplateResponse); + +const name = 'test_index_name'; +const totalFieldsLimit = 1000; + +describe('updateIndices', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it(`should update indices`, async () => { + const indexName = 'test_index_name-default'; + esClient.indices.get.mockResolvedValueOnce({ [indexName]: {} }); + + await updateIndices({ + esClient, + logger, + name, + totalFieldsLimit, + }); + + expect(esClient.indices.get).toHaveBeenCalledWith({ + index: name, + expand_wildcards: 'all', + }); + + expect(esClient.indices.putSettings).toHaveBeenCalledWith({ + index: indexName, + body: { 'index.mapping.total_fields.limit': totalFieldsLimit }, + }); + expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledWith({ + name: indexName, + }); + expect(esClient.indices.putMapping).toHaveBeenCalledWith({ + index: indexName, + body: simulateIndexTemplateResponse.template.mappings, + }); + }); + + it(`should update multiple indices`, async () => { + const indexName1 = 'test_index_name-1'; + const indexName2 = 'test_index_name-2'; + esClient.indices.get.mockResolvedValueOnce({ [indexName1]: {}, [indexName2]: {} }); + + await updateIndices({ + esClient, + logger, + name, + totalFieldsLimit, + }); + + expect(esClient.indices.putSettings).toHaveBeenCalledTimes(2); + expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes(2); + expect(esClient.indices.putMapping).toHaveBeenCalledTimes(2); + }); + + it(`should not update indices when not exist`, async () => { + esClient.indices.get.mockResolvedValueOnce({}); + + await updateIndices({ + esClient, + logger, + name, + totalFieldsLimit, + }); + + expect(esClient.indices.putSettings).not.toHaveBeenCalled(); + expect(esClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); + expect(esClient.indices.putMapping).not.toHaveBeenCalled(); + }); +}); + +describe('createIndex', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it(`should create index`, async () => { + esClient.indices.exists.mockResolvedValueOnce(false); + + await createIndex({ + esClient, + logger, + name, + }); + + expect(esClient.indices.exists).toHaveBeenCalledWith({ index: name, expand_wildcards: 'all' }); + expect(esClient.indices.create).toHaveBeenCalledWith({ index: name }); + }); + + it(`should not create index if already exists`, async () => { + esClient.indices.exists.mockResolvedValueOnce(true); + + await createIndex({ + esClient, + logger, + name, + }); + + expect(esClient.indices.exists).toHaveBeenCalledWith({ index: name, expand_wildcards: 'all' }); + expect(esClient.indices.create).not.toHaveBeenCalled(); + }); +}); + +describe('createOrUpdateIndex', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it(`should create index if not exists`, async () => { + esClient.indices.exists.mockResolvedValueOnce(false); + + await createOrUpdateIndex({ + esClient, + logger, + name, + totalFieldsLimit, + }); + + expect(esClient.indices.create).toHaveBeenCalledWith({ index: name }); + }); + + it(`should update index if already exists`, async () => { + esClient.indices.exists.mockResolvedValueOnce(true); + + await createOrUpdateIndex({ + esClient, + logger, + name, + totalFieldsLimit, + }); + + expect(esClient.indices.exists).toHaveBeenCalledWith({ index: name, expand_wildcards: 'all' }); + + expect(esClient.indices.putSettings).toHaveBeenCalledWith({ + index: name, + body: { 'index.mapping.total_fields.limit': totalFieldsLimit }, + }); + expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledWith({ + name, + }); + expect(esClient.indices.putMapping).toHaveBeenCalledWith({ + index: name, + body: simulateIndexTemplateResponse.template.mappings, + }); + }); +}); diff --git a/packages/kbn-index-adapter/src/create_or_update_index.ts b/packages/kbn-index-adapter/src/create_or_update_index.ts new file mode 100644 index 0000000000000..ff825c61305b7 --- /dev/null +++ b/packages/kbn-index-adapter/src/create_or_update_index.ts @@ -0,0 +1,237 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import type { IndexName } from '@elastic/elasticsearch/lib/api/types'; +import type { IndicesSimulateIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { Logger, ElasticsearchClient } from '@kbn/core/server'; +import { get } from 'lodash'; +import { retryTransientEsErrors } from './retry_transient_es_errors'; + +interface UpdateIndexMappingsOpts { + logger: Logger; + esClient: ElasticsearchClient; + indexNames: string[]; + totalFieldsLimit: number; +} + +interface UpdateIndexOpts { + logger: Logger; + esClient: ElasticsearchClient; + indexName: string; + totalFieldsLimit: number; +} + +const updateTotalFieldLimitSetting = async ({ + logger, + esClient, + indexName, + totalFieldsLimit, +}: UpdateIndexOpts) => { + logger.debug(`Updating total field limit setting for ${indexName} data stream.`); + + try { + const body = { 'index.mapping.total_fields.limit': totalFieldsLimit }; + await retryTransientEsErrors(() => esClient.indices.putSettings({ index: indexName, body }), { + logger, + }); + } catch (err) { + logger.error( + `Failed to PUT index.mapping.total_fields.limit settings for ${indexName}: ${err.message}` + ); + throw err; + } +}; + +// This will update the mappings but *not* the settings. This +// is due to the fact settings can be classed as dynamic and static, and static +// updates will fail on an index that isn't closed. New settings *will* be applied as part +// of the ILM policy rollovers. More info: https://github.com/elastic/kibana/pull/113389#issuecomment-940152654 +const updateMapping = async ({ logger, esClient, indexName }: UpdateIndexOpts) => { + logger.debug(`Updating mappings for ${indexName} data stream.`); + + let simulatedIndexMapping: IndicesSimulateIndexTemplateResponse; + try { + simulatedIndexMapping = await retryTransientEsErrors( + () => esClient.indices.simulateIndexTemplate({ name: indexName }), + { logger } + ); + } catch (err) { + logger.error( + `Ignored PUT mappings for ${indexName}; error generating simulated mappings: ${err.message}` + ); + return; + } + + const simulatedMapping = get(simulatedIndexMapping, ['template', 'mappings']); + + if (simulatedMapping == null) { + logger.error(`Ignored PUT mappings for ${indexName}; simulated mappings were empty`); + return; + } + + try { + await retryTransientEsErrors( + () => esClient.indices.putMapping({ index: indexName, body: simulatedMapping }), + { logger } + ); + } catch (err) { + logger.error(`Failed to PUT mapping for ${indexName}: ${err.message}`); + throw err; + } +}; +/** + * Updates the data stream mapping and total field limit setting + */ +const updateIndexMappings = async ({ + logger, + esClient, + totalFieldsLimit, + indexNames, +}: UpdateIndexMappingsOpts) => { + // Update total field limit setting of found indices + // Other index setting changes are not updated at this time + await Promise.all( + indexNames.map((indexName) => + updateTotalFieldLimitSetting({ logger, esClient, totalFieldsLimit, indexName }) + ) + ); + // Update mappings of the found indices. + await Promise.all( + indexNames.map((indexName) => updateMapping({ logger, esClient, totalFieldsLimit, indexName })) + ); +}; + +export interface CreateOrUpdateIndexParams { + name: string; + logger: Logger; + esClient: ElasticsearchClient; + totalFieldsLimit: number; +} + +export async function createOrUpdateIndex({ + logger, + esClient, + name, + totalFieldsLimit, +}: CreateOrUpdateIndexParams): Promise { + logger.info(`Creating index - ${name}`); + + // check if index exists + let indexExists = false; + try { + indexExists = await retryTransientEsErrors( + () => esClient.indices.exists({ index: name, expand_wildcards: 'all' }), + { logger } + ); + } catch (error) { + if (error?.statusCode !== 404) { + logger.error(`Error fetching index for ${name} - ${error.message}`); + throw error; + } + } + + // if a index exists, update the underlying mapping + if (indexExists) { + await updateIndexMappings({ + logger, + esClient, + indexNames: [name], + totalFieldsLimit, + }); + } else { + try { + await retryTransientEsErrors(() => esClient.indices.create({ index: name }), { logger }); + } catch (error) { + if (error?.meta?.body?.error?.type !== 'resource_already_exists_exception') { + logger.error(`Error creating index ${name} - ${error.message}`); + throw error; + } + } + } +} + +export interface CreateIndexParams { + name: string; + logger: Logger; + esClient: ElasticsearchClient; +} + +export async function createIndex({ logger, esClient, name }: CreateIndexParams): Promise { + logger.debug(`Checking existence of index - ${name}`); + + // check if index exists + let indexExists = false; + try { + indexExists = await retryTransientEsErrors( + () => esClient.indices.exists({ index: name, expand_wildcards: 'all' }), + { + logger, + } + ); + } catch (error) { + if (error?.statusCode !== 404) { + logger.error(`Error fetching index for ${name} - ${error.message}`); + throw error; + } + } + + // return if index already created + if (indexExists) { + return; + } + + logger.info(`Creating index - ${name}`); + try { + await retryTransientEsErrors(() => esClient.indices.create({ index: name }), { logger }); + } catch (error) { + if (error?.meta?.body?.error?.type !== 'resource_already_exists_exception') { + logger.error(`Error creating index ${name} - ${error.message}`); + throw error; + } + } +} + +export interface CreateOrUpdateSpacesIndexParams { + name: string; + logger: Logger; + esClient: ElasticsearchClient; + totalFieldsLimit: number; +} + +export async function updateIndices({ + logger, + esClient, + name, + totalFieldsLimit, +}: CreateOrUpdateSpacesIndexParams): Promise { + logger.info(`Updating indices - ${name}`); + + // check if data stream exists + let indices: IndexName[] = []; + try { + const response = await retryTransientEsErrors( + () => esClient.indices.get({ index: name, expand_wildcards: 'all' }), + { logger } + ); + indices = Object.keys(response); + } catch (error) { + if (error?.statusCode !== 404) { + logger.error(`Error fetching indices for ${name} - ${error.message}`); + throw error; + } + } + if (indices.length > 0) { + await updateIndexMappings({ + logger, + esClient, + totalFieldsLimit, + indexNames: indices, + }); + } +} diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_index_template.test.ts b/packages/kbn-index-adapter/src/create_or_update_index_template.test.ts similarity index 100% rename from packages/kbn-data-stream-adapter/src/create_or_update_index_template.test.ts rename to packages/kbn-index-adapter/src/create_or_update_index_template.test.ts diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_index_template.ts b/packages/kbn-index-adapter/src/create_or_update_index_template.ts similarity index 100% rename from packages/kbn-data-stream-adapter/src/create_or_update_index_template.ts rename to packages/kbn-index-adapter/src/create_or_update_index_template.ts diff --git a/packages/kbn-data-stream-adapter/src/field_maps/ecs_field_map.ts b/packages/kbn-index-adapter/src/field_maps/ecs_field_map.ts similarity index 100% rename from packages/kbn-data-stream-adapter/src/field_maps/ecs_field_map.ts rename to packages/kbn-index-adapter/src/field_maps/ecs_field_map.ts diff --git a/packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.test.ts b/packages/kbn-index-adapter/src/field_maps/mapping_from_field_map.test.ts similarity index 100% rename from packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.test.ts rename to packages/kbn-index-adapter/src/field_maps/mapping_from_field_map.test.ts diff --git a/packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.ts b/packages/kbn-index-adapter/src/field_maps/mapping_from_field_map.ts similarity index 100% rename from packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.ts rename to packages/kbn-index-adapter/src/field_maps/mapping_from_field_map.ts diff --git a/packages/kbn-data-stream-adapter/src/field_maps/types.ts b/packages/kbn-index-adapter/src/field_maps/types.ts similarity index 100% rename from packages/kbn-data-stream-adapter/src/field_maps/types.ts rename to packages/kbn-index-adapter/src/field_maps/types.ts diff --git a/packages/kbn-index-adapter/src/index_adapter.ts b/packages/kbn-index-adapter/src/index_adapter.ts new file mode 100644 index 0000000000000..eef2ce529d78a --- /dev/null +++ b/packages/kbn-index-adapter/src/index_adapter.ts @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import type { + ClusterPutComponentTemplateRequest, + IndicesPutIndexTemplateRequest, +} from '@elastic/elasticsearch/lib/api/types'; +import type { Logger, ElasticsearchClient } from '@kbn/core/server'; +import type { Subject } from 'rxjs'; +import { createOrUpdateComponentTemplate } from './create_or_update_component_template'; +import { createOrUpdateIndex } from './create_or_update_index'; +import { createOrUpdateIndexTemplate } from './create_or_update_index_template'; +import { InstallShutdownError, installWithTimeout } from './install_with_timeout'; +import { + getComponentTemplate, + getIndexTemplate, + type GetComponentTemplateOpts, + type GetIndexTemplateOpts, +} from './resource_installer_utils'; + +export interface IndexAdapterParams { + kibanaVersion: string; + totalFieldsLimit?: number; +} +export type SetComponentTemplateParams = GetComponentTemplateOpts; +export type SetIndexTemplateParams = Omit< + GetIndexTemplateOpts, + 'indexPatterns' | 'kibanaVersion' | 'totalFieldsLimit' +>; +export interface GetInstallFnParams { + logger: Logger; + pluginStop$: Subject; + tasksTimeoutMs?: number; +} +export interface InstallParams { + logger: Logger; + esClient: ElasticsearchClient | Promise; + pluginStop$: Subject; + tasksTimeoutMs?: number; +} + +const DEFAULT_FIELDS_LIMIT = 2500; + +export class IndexAdapter { + protected readonly kibanaVersion: string; + protected readonly totalFieldsLimit: number; + protected componentTemplates: ClusterPutComponentTemplateRequest[] = []; + protected indexTemplates: IndicesPutIndexTemplateRequest[] = []; + protected installed: boolean; + + constructor(protected readonly name: string, options: IndexAdapterParams) { + this.installed = false; + this.kibanaVersion = options.kibanaVersion; + this.totalFieldsLimit = options.totalFieldsLimit ?? DEFAULT_FIELDS_LIMIT; + } + + public setComponentTemplate(params: SetComponentTemplateParams) { + if (this.installed) { + throw new Error('Cannot set component template after install'); + } + this.componentTemplates.push(getComponentTemplate(params)); + } + + public setIndexTemplate(params: SetIndexTemplateParams) { + if (this.installed) { + throw new Error('Cannot set index template after install'); + } + this.indexTemplates.push( + getIndexTemplate({ + ...params, + indexPatterns: [this.name], + kibanaVersion: this.kibanaVersion, + totalFieldsLimit: this.totalFieldsLimit, + }) + ); + } + + protected getInstallFn({ logger, pluginStop$, tasksTimeoutMs }: GetInstallFnParams) { + return async (promise: Promise, description?: string): Promise => { + try { + await installWithTimeout({ + installFn: () => promise, + description, + timeoutMs: tasksTimeoutMs, + pluginStop$, + }); + } catch (err) { + if (err instanceof InstallShutdownError) { + logger.info(err.message); + } else { + throw err; + } + } + }; + } + + protected async installTemplates(params: InstallParams) { + const { logger, pluginStop$, tasksTimeoutMs } = params; + const esClient = await params.esClient; + const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs }); + + // Install component templates in parallel + await Promise.all( + this.componentTemplates.map((componentTemplate) => + installFn( + createOrUpdateComponentTemplate({ + template: componentTemplate, + esClient, + logger, + totalFieldsLimit: this.totalFieldsLimit, + }), + `create or update ${componentTemplate.name} component template` + ) + ) + ); + + // Install index templates in parallel + await Promise.all( + this.indexTemplates.map((indexTemplate) => + installFn( + createOrUpdateIndexTemplate({ + template: indexTemplate, + esClient, + logger, + }), + `create or update ${indexTemplate.name} index template` + ) + ) + ); + } + + public async install(params: InstallParams) { + this.installed = true; + const { logger, pluginStop$, tasksTimeoutMs } = params; + const esClient = await params.esClient; + + await this.installTemplates(params); + + const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs }); + + // create index when everything is ready + await installFn( + createOrUpdateIndex({ + name: this.name, + esClient, + logger, + totalFieldsLimit: this.totalFieldsLimit, + }), + `${this.name} index` + ); + } +} diff --git a/packages/kbn-index-adapter/src/index_pattern_adapter.ts b/packages/kbn-index-adapter/src/index_pattern_adapter.ts new file mode 100644 index 0000000000000..38a96a3c65b83 --- /dev/null +++ b/packages/kbn-index-adapter/src/index_pattern_adapter.ts @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { createIndex, updateIndices } from './create_or_update_index'; +import { IndexAdapter, type IndexAdapterParams, type InstallParams } from './index_adapter'; + +export type InstallIndex = (indexSuffix: string) => Promise; + +export class IndexPatternAdapter extends IndexAdapter { + protected installationPromises: Map>; + protected installIndexPromise?: Promise; + + constructor(protected readonly prefix: string, options: IndexAdapterParams) { + super(`${prefix}-*`, options); // make indexTemplate `indexPatterns` match all index names + this.installationPromises = new Map(); + } + + /** Method to create/update the templates, update existing indices and setup internal state for the adapter. */ + public async install(params: InstallParams): Promise { + this.installIndexPromise = this._install(params); + await this.installIndexPromise; + } + + protected async _install(params: InstallParams): Promise { + const { logger, pluginStop$, tasksTimeoutMs } = params; + + await this.installTemplates(params); + + const esClient = await params.esClient; + const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs }); + + // Update existing specific indices + await installFn( + updateIndices({ + name: this.name, // `${prefix}-*` + esClient, + logger, + totalFieldsLimit: this.totalFieldsLimit, + }), + `update specific indices` + ); + + // Define the function to create concrete indices on demand + return async (name: string) => + installFn(createIndex({ name, esClient, logger }), `create ${name} index`); + } + + /** + * Method to create the index for a given index suffix. + * Stores the installations promises to avoid concurrent installations for the same index. + * Index creation will only be attempted once per index suffix and existence will be checked before creating. + */ + public async createIndex(indexSuffix: string): Promise { + if (!this.installIndexPromise) { + throw new Error('Cannot installIndex before install'); + } + + const existingInstallation = this.installationPromises.get(indexSuffix); + if (existingInstallation) { + return existingInstallation; + } + const indexName = this.getIndexName(indexSuffix); + + // Awaits for installIndexPromise to resolve to ensure templates are installed before the specific index is created. + // This is a safety measure since the initial `install` call may not be awaited from the plugin lifecycle caller. + // However, the promise will most likely be already fulfilled by the time `createIndex` is called, so this is a no-op. + const installation = this.installIndexPromise + .then((installIndex) => installIndex(indexName)) + .catch((err) => { + this.installationPromises.delete(indexSuffix); + throw err; + }); + + this.installationPromises.set(indexSuffix, installation); + return installation; + } + + /** Method to get the full index name for a given index suffix. */ + public getIndexName(indexSuffix: string): string { + return `${this.prefix}-${indexSuffix}`; + } + + /** Method to get the full index name for a given index suffix. It returns undefined if the index does not exist. */ + public async getInstalledIndexName(indexSuffix: string): Promise { + const existingInstallation = this.installationPromises.get(indexSuffix); + if (!existingInstallation) { + return undefined; + } + return existingInstallation.then(() => this.getIndexName(indexSuffix)).catch(() => undefined); + } +} diff --git a/packages/kbn-data-stream-adapter/src/install_with_timeout.test.ts b/packages/kbn-index-adapter/src/install_with_timeout.test.ts similarity index 100% rename from packages/kbn-data-stream-adapter/src/install_with_timeout.test.ts rename to packages/kbn-index-adapter/src/install_with_timeout.test.ts diff --git a/packages/kbn-data-stream-adapter/src/install_with_timeout.ts b/packages/kbn-index-adapter/src/install_with_timeout.ts similarity index 100% rename from packages/kbn-data-stream-adapter/src/install_with_timeout.ts rename to packages/kbn-index-adapter/src/install_with_timeout.ts diff --git a/packages/kbn-data-stream-adapter/src/resource_installer_utils.test.ts b/packages/kbn-index-adapter/src/resource_installer_utils.test.ts similarity index 92% rename from packages/kbn-data-stream-adapter/src/resource_installer_utils.test.ts rename to packages/kbn-index-adapter/src/resource_installer_utils.test.ts index 93d421bb5605c..31d4a3abcbb0d 100644 --- a/packages/kbn-data-stream-adapter/src/resource_installer_utils.test.ts +++ b/packages/kbn-index-adapter/src/resource_installer_utils.test.ts @@ -24,7 +24,6 @@ describe('getIndexTemplate', () => { expect(indexTemplate).toEqual({ name: defaultParams.name, body: { - data_stream: { hidden: true }, index_patterns: defaultParams.indexPatterns, composed_of: defaultParams.componentTemplateRefs, template: { @@ -57,8 +56,17 @@ describe('getIndexTemplate', () => { }); }); + it('should create data stream index template with given parameters and defaults', () => { + const indexTemplate = getIndexTemplate({ ...defaultParams, isDataStream: true }); + expect(indexTemplate.body).toEqual( + expect.objectContaining({ + data_stream: { hidden: true }, + }) + ); + }); + it('should create not hidden index template', () => { - const { body } = getIndexTemplate({ ...defaultParams, hidden: false }); + const { body } = getIndexTemplate({ ...defaultParams, isDataStream: true, hidden: false }); expect(body?.data_stream?.hidden).toEqual(false); expect(body?.template?.settings?.hidden).toEqual(false); }); diff --git a/packages/kbn-data-stream-adapter/src/resource_installer_utils.ts b/packages/kbn-index-adapter/src/resource_installer_utils.ts similarity index 93% rename from packages/kbn-data-stream-adapter/src/resource_installer_utils.ts rename to packages/kbn-index-adapter/src/resource_installer_utils.ts index 96b220cf0983c..eb6e2490000b2 100644 --- a/packages/kbn-data-stream-adapter/src/resource_installer_utils.ts +++ b/packages/kbn-index-adapter/src/resource_installer_utils.ts @@ -19,7 +19,7 @@ import type { import type { FieldMap } from './field_maps/types'; import { mappingFromFieldMap } from './field_maps/mapping_from_field_map'; -interface GetComponentTemplateOpts { +export interface GetComponentTemplateOpts { name: string; fieldMap: FieldMap; settings?: IndicesIndexSettings; @@ -47,7 +47,7 @@ export const getComponentTemplate = ({ }, }); -interface GetIndexTemplateOpts { +export interface GetIndexTemplateOpts { name: string; indexPatterns: string[]; kibanaVersion: string; @@ -56,6 +56,7 @@ interface GetIndexTemplateOpts { namespace?: string; template?: IndicesPutIndexTemplateIndexTemplateMapping; hidden?: boolean; + isDataStream?: boolean; } export const getIndexTemplate = ({ @@ -67,6 +68,7 @@ export const getIndexTemplate = ({ namespace = 'default', template = {}, hidden = true, + isDataStream = false, }: GetIndexTemplateOpts): IndicesPutIndexTemplateRequest => { const indexMetadata: Metadata = { kibana: { @@ -79,7 +81,7 @@ export const getIndexTemplate = ({ return { name, body: { - data_stream: { hidden }, + ...(isDataStream && { data_stream: { hidden } }), index_patterns: indexPatterns, composed_of: componentTemplateRefs, template: { diff --git a/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.test.ts b/packages/kbn-index-adapter/src/retry_transient_es_errors.test.ts similarity index 100% rename from packages/kbn-data-stream-adapter/src/retry_transient_es_errors.test.ts rename to packages/kbn-index-adapter/src/retry_transient_es_errors.test.ts diff --git a/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.ts b/packages/kbn-index-adapter/src/retry_transient_es_errors.ts similarity index 100% rename from packages/kbn-data-stream-adapter/src/retry_transient_es_errors.ts rename to packages/kbn-index-adapter/src/retry_transient_es_errors.ts diff --git a/packages/kbn-index-adapter/tsconfig.json b/packages/kbn-index-adapter/tsconfig.json new file mode 100644 index 0000000000000..cca50adbf7eb8 --- /dev/null +++ b/packages/kbn-index-adapter/tsconfig.json @@ -0,0 +1,20 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node", + ] + }, + "include": ["**/*.ts"], + "kbn_references": [ + "@kbn/core", + "@kbn/std", + "@kbn/safer-lodash-set", + "@kbn/logging-mocks", + ], + "exclude": [ + "target/**/*" + ], +} diff --git a/src/plugins/controls/public/controls/timeslider_control/init_time_range_subscription.ts b/src/plugins/controls/public/controls/timeslider_control/init_time_range_subscription.ts index 7934e9deaa9b4..a31b9810c7481 100644 --- a/src/plugins/controls/public/controls/timeslider_control/init_time_range_subscription.ts +++ b/src/plugins/controls/public/controls/timeslider_control/init_time_range_subscription.ts @@ -11,7 +11,8 @@ import { TimeRange } from '@kbn/es-query'; import { i18n } from '@kbn/i18n'; import { apiHasParentApi, apiPublishesTimeRange } from '@kbn/presentation-publishing'; import moment from 'moment'; -import { BehaviorSubject, skip } from 'rxjs'; +import { BehaviorSubject, Subscription, skip } from 'rxjs'; +import { apiPublishesReload } from '@kbn/presentation-publishing/interfaces/fetch/publishes_reload'; import { getTimeRangeMeta, getTimezone, TimeRangeMeta } from './get_time_range_meta'; import { getMomentTimezone } from './time_utils'; @@ -26,6 +27,13 @@ export function initTimeRangeSubscription(controlGroupApi: unknown) { timeRangeMeta$.next(getTimeRangeMeta(timeRange)); }); + let reloadSubscription: undefined | Subscription; + if (apiHasParentApi(controlGroupApi) && apiPublishesReload(controlGroupApi.parentApi)) { + reloadSubscription = controlGroupApi.parentApi.reload$.subscribe(() => { + timeRangeMeta$.next(getTimeRangeMeta(timeRange$.value)); + }); + } + return { timeRangeMeta$, formatDate: (epoch: number) => { @@ -35,6 +43,7 @@ export function initTimeRangeSubscription(controlGroupApi: unknown) { .format(timeRangeMeta$.value.format); }, cleanupTimeRangeSubscription: () => { + reloadSubscription?.unsubscribe(); timeRangeSubscription.unsubscribe(); }, }; diff --git a/src/plugins/home/public/application/components/tutorial/replace_template_strings.js b/src/plugins/home/public/application/components/tutorial/replace_template_strings.js index 75da52e9af2b5..09abb7300866a 100644 --- a/src/plugins/home/public/application/components/tutorial/replace_template_strings.js +++ b/src/plugins/home/public/application/components/tutorial/replace_template_strings.js @@ -38,7 +38,6 @@ export function replaceTemplateStrings(text, params = {}) { filebeat: docLinks.links.filebeat.base, metricbeat: docLinks.links.metricbeat.base, heartbeat: docLinks.links.heartbeat.base, - functionbeat: docLinks.links.functionbeat.base, winlogbeat: docLinks.links.winlogbeat.base, auditbeat: docLinks.links.auditbeat.base, }, diff --git a/src/plugins/vis_types/timeseries/kibana.jsonc b/src/plugins/vis_types/timeseries/kibana.jsonc index bd0e4ac352daa..03cb4697162ed 100644 --- a/src/plugins/vis_types/timeseries/kibana.jsonc +++ b/src/plugins/vis_types/timeseries/kibana.jsonc @@ -5,7 +5,7 @@ "@elastic/kibana-visualizations" ], "group": "platform", - "visibility": "private", + "visibility": "shared", "description": "Registers the TSVB visualization. TSVB has its one editor, works with index patterns and index strings and contains 6 types of charts: timeseries, topN, table. markdown, metric and gauge.", "plugin": { "id": "visTypeTimeseries", diff --git a/test/functional/apps/dashboard_elements/input_control_vis/input_control_options.ts b/test/functional/apps/dashboard_elements/input_control_vis/input_control_options.ts index cccf19de4070a..675f386eea34a 100644 --- a/test/functional/apps/dashboard_elements/input_control_vis/input_control_options.ts +++ b/test/functional/apps/dashboard_elements/input_control_vis/input_control_options.ts @@ -28,6 +28,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('input control options', () => { before(async () => { await visualize.initTests(); + await timePicker.resetDefaultAbsoluteRangeViaUiSettings(); await common.navigateToApp('visualize'); await visualize.loadSavedVisualization('input control options', { navigateToVisualize: false, diff --git a/test/functional/page_objects/visualize_page.ts b/test/functional/page_objects/visualize_page.ts index d565c5168641b..315ecba11c6d5 100644 --- a/test/functional/page_objects/visualize_page.ts +++ b/test/functional/page_objects/visualize_page.ts @@ -41,6 +41,7 @@ export class VisualizePageObject extends FtrService { private readonly elasticChart = this.ctx.getService('elasticChart'); private readonly common = this.ctx.getPageObject('common'); private readonly header = this.ctx.getPageObject('header'); + private readonly timePicker = this.ctx.getPageObject('timePicker'); private readonly visChart = this.ctx.getPageObject('visChart'); private readonly toasts = this.ctx.getService('toasts'); @@ -63,6 +64,7 @@ export class VisualizePageObject extends FtrService { [FORMATS_UI_SETTINGS.FORMAT_BYTES_DEFAULT_PATTERN]: '0,0.[000]b', 'visualization:visualize:legacyHeatmapChartsLibrary': isLegacyChart, 'histogram:maxBars': 100, + 'timepicker:timeDefaults': `{ "from": "${this.timePicker.defaultStartTimeUTC}", "to": "${this.timePicker.defaultEndTimeUTC}"}`, }); } diff --git a/tsconfig.base.json b/tsconfig.base.json index 68faf44ed74d4..a525823e98e9d 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1034,6 +1034,8 @@ "@kbn/import-locator/*": ["packages/kbn-import-locator/*"], "@kbn/import-resolver": ["packages/kbn-import-resolver"], "@kbn/import-resolver/*": ["packages/kbn-import-resolver/*"], + "@kbn/index-adapter": ["packages/kbn-index-adapter"], + "@kbn/index-adapter/*": ["packages/kbn-index-adapter/*"], "@kbn/index-lifecycle-management-common-shared": ["x-pack/packages/index-lifecycle-management/index_lifecycle_management_common_shared"], "@kbn/index-lifecycle-management-common-shared/*": ["x-pack/packages/index-lifecycle-management/index_lifecycle_management_common_shared/*"], "@kbn/index-lifecycle-management-plugin": ["x-pack/plugins/index_lifecycle_management"], diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/capabilities/index.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/capabilities/index.ts index 54c24f6ce7b8f..d883dfe98d564 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/capabilities/index.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/capabilities/index.ts @@ -19,6 +19,5 @@ export type AssistantFeatureKey = keyof AssistantFeatures; * Default features available to the elastic assistant */ export const defaultAssistantFeatures = Object.freeze({ - assistantKnowledgeBaseByDefault: true, assistantModelEvaluation: false, }); diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/capabilities/get_capabilities_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/capabilities/get_capabilities_route.gen.ts index b3ab7cca5bc02..0f8b6235d7dc9 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/capabilities/get_capabilities_route.gen.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/capabilities/get_capabilities_route.gen.ts @@ -18,6 +18,5 @@ import { z } from '@kbn/zod'; export type GetCapabilitiesResponse = z.infer; export const GetCapabilitiesResponse = z.object({ - assistantKnowledgeBaseByDefault: z.boolean(), assistantModelEvaluation: z.boolean(), }); diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/capabilities/get_capabilities_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/capabilities/get_capabilities_route.schema.yaml index 01b5eb0e15823..a042abd391796 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/capabilities/get_capabilities_route.schema.yaml +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/capabilities/get_capabilities_route.schema.yaml @@ -20,12 +20,9 @@ paths: schema: type: object properties: - assistantKnowledgeBaseByDefault: - type: boolean assistantModelEvaluation: type: boolean required: - - assistantKnowledgeBaseByDefault - assistantModelEvaluation '400': description: Generic Error diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.gen.ts index 4f03dbe0b1343..a4f38cafd460b 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.gen.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.gen.ts @@ -55,20 +55,6 @@ export type CreateKnowledgeBaseRequestParamsInput = z.input< export type CreateKnowledgeBaseResponse = z.infer; export const CreateKnowledgeBaseResponse = KnowledgeBaseResponse; -export type DeleteKnowledgeBaseRequestParams = z.infer; -export const DeleteKnowledgeBaseRequestParams = z.object({ - /** - * The KnowledgeBase `resource` value. - */ - resource: z.string().optional(), -}); -export type DeleteKnowledgeBaseRequestParamsInput = z.input< - typeof DeleteKnowledgeBaseRequestParams ->; - -export type DeleteKnowledgeBaseResponse = z.infer; -export const DeleteKnowledgeBaseResponse = KnowledgeBaseResponse; - export type ReadKnowledgeBaseRequestParams = z.infer; export const ReadKnowledgeBaseRequestParams = z.object({ /** diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.schema.yaml index b4c16189e2387..67193212abb49 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.schema.yaml +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.schema.yaml @@ -100,40 +100,6 @@ paths: type: string message: type: string - delete: - x-codegen-enabled: true - x-labels: [ess, serverless] - operationId: DeleteKnowledgeBase - description: Deletes KnowledgeBase with the `resource` field. - summary: Deletes a KnowledgeBase - tags: - - KnowledgeBase API - parameters: - - name: resource - in: path - description: The KnowledgeBase `resource` value. - schema: - type: string - responses: - 200: - description: Indicates a successful call. - content: - application/json: - schema: - $ref: '#/components/schemas/KnowledgeBaseResponse' - 400: - description: Generic Error - content: - application/json: - schema: - type: object - properties: - statusCode: - type: number - error: - type: string - message: - type: string components: schemas: diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/entries/bulk_crud_knowledge_base_entries_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/entries/bulk_crud_knowledge_base_entries_route.schema.yaml index 7670114c7164a..db68416b14561 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/entries/bulk_crud_knowledge_base_entries_route.schema.yaml +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/entries/bulk_crud_knowledge_base_entries_route.schema.yaml @@ -6,7 +6,7 @@ paths: /internal/elastic_assistant/knowledge_base/entries/_bulk_action: post: x-codegen-enabled: true - # This API is still behind the `assistantKnowledgeBaseByDefault` feature flag + # Targeted to update to public by 8.18 x-internal: true x-labels: [ess, serverless] operationId: PerformKnowledgeBaseEntryBulkAction diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/entries/crud_knowledge_base_entries_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/entries/crud_knowledge_base_entries_route.schema.yaml index 7479b5cca8225..10105ef7dce90 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/entries/crud_knowledge_base_entries_route.schema.yaml +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/entries/crud_knowledge_base_entries_route.schema.yaml @@ -6,7 +6,7 @@ paths: /internal/elastic_assistant/knowledge_base/entries: post: x-codegen-enabled: true - # This API is still behind the `assistantKnowledgeBaseByDefault` feature flag + # Targeted to update to public by 8.18 x-internal: true x-labels: [ess, serverless] operationId: CreateKnowledgeBaseEntry @@ -37,7 +37,7 @@ paths: /internal/elastic_assistant/knowledge_base/entries/{id}: get: x-codegen-enabled: true - # This API is still behind the `assistantKnowledgeBaseByDefault` feature flag + # Targeted to update to public by 8.18 x-internal: true x-labels: [ess, serverless] operationId: ReadKnowledgeBaseEntry @@ -67,7 +67,7 @@ paths: $ref: './common_attributes.schema.yaml#/components/schemas/KnowledgeBaseEntryErrorSchema' put: x-codegen-enabled: true - # This API is still behind the `assistantKnowledgeBaseByDefault` feature flag + # Targeted to update to public by 8.18 x-internal: true x-labels: [ess, serverless] operationId: UpdateKnowledgeBaseEntry @@ -103,7 +103,7 @@ paths: $ref: './common_attributes.schema.yaml#/components/schemas/KnowledgeBaseEntryErrorSchema' delete: x-codegen-enabled: true - # This API is still behind the `assistantKnowledgeBaseByDefault` feature flag + # Targeted to update to public by 8.18 x-internal: true x-labels: [ess, serverless] operationId: DeleteKnowledgeBaseEntry diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/entries/find_knowledge_base_entries_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/entries/find_knowledge_base_entries_route.schema.yaml index 8794a94b0efc9..9b9696e8760fc 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/entries/find_knowledge_base_entries_route.schema.yaml +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/entries/find_knowledge_base_entries_route.schema.yaml @@ -6,7 +6,7 @@ paths: /internal/elastic_assistant/knowledge_base/entries/_find: get: x-codegen-enabled: true - # This API is still behind the `assistantKnowledgeBaseByDefault` feature flag + # Targeted to update to public by 8.18 x-internal: true x-labels: [ess, serverless] operationId: FindKnowledgeBaseEntries diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/api.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/api.test.tsx index 5509f43037444..2a1ffc5072570 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/api.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/api.test.tsx @@ -7,12 +7,7 @@ import { HttpSetup } from '@kbn/core-http-browser'; -import { - deleteKnowledgeBase, - getKnowledgeBaseIndices, - getKnowledgeBaseStatus, - postKnowledgeBase, -} from './api'; +import { getKnowledgeBaseIndices, getKnowledgeBaseStatus, postKnowledgeBase } from './api'; jest.mock('@kbn/core-http-browser'); @@ -78,29 +73,6 @@ describe('API tests', () => { }); }); - describe('deleteKnowledgeBase', () => { - it('calls the knowledge base API when correct resource path', async () => { - await deleteKnowledgeBase(knowledgeBaseArgs); - - expect(mockHttp.fetch).toHaveBeenCalledWith( - '/internal/elastic_assistant/knowledge_base/a-resource', - { - method: 'DELETE', - signal: undefined, - version: '1', - } - ); - }); - it('returns error when error is an error', async () => { - const error = 'simulated error'; - (mockHttp.fetch as jest.Mock).mockImplementation(() => { - throw new Error(error); - }); - - await expect(deleteKnowledgeBase(knowledgeBaseArgs)).resolves.toThrowError('simulated error'); - }); - }); - describe('getKnowledgeBaseIndices', () => { it('calls the knowledge base API when correct resource path', async () => { await getKnowledgeBaseIndices({ http: mockHttp }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/api.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/api.tsx index 4db8c0787a1e1..00fe022ad9517 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/api.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/api.tsx @@ -9,8 +9,6 @@ import { API_VERSIONS, CreateKnowledgeBaseRequestParams, CreateKnowledgeBaseResponse, - DeleteKnowledgeBaseRequestParams, - DeleteKnowledgeBaseResponse, ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_INDICES_URL, ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_URL, GetKnowledgeBaseIndicesResponse, @@ -79,38 +77,6 @@ export const postKnowledgeBase = async ({ return response as CreateKnowledgeBaseResponse; }; -/** - * API call for deleting the Knowledge Base. Provide a resource to delete that specific resource. - * - * @param {Object} options - The options object. - * @param {HttpSetup} options.http - HttpSetup - * @param {string} [options.resource] - Resource to be deleted from the KB, otherwise delete the entire KB - * @param {AbortSignal} [options.signal] - AbortSignal - * - * @returns {Promise} - */ -export const deleteKnowledgeBase = async ({ - http, - resource, - signal, -}: DeleteKnowledgeBaseRequestParams & { - http: HttpSetup; - signal?: AbortSignal | undefined; -}): Promise => { - try { - const path = ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_URL.replace('{resource?}', resource || ''); - const response = await http.fetch(path, { - method: 'DELETE', - signal, - version: API_VERSIONS.internal.v1, - }); - - return response as DeleteKnowledgeBaseResponse; - } catch (error) { - return error as IHttpFetchError; - } -}; - /** * API call for getting indices that have fields of `semantic_text` type. * diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/use_delete_knowledge_base.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/use_delete_knowledge_base.test.tsx deleted file mode 100644 index b50c345edb3b3..0000000000000 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/use_delete_knowledge_base.test.tsx +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { act, renderHook } from '@testing-library/react-hooks'; -import { useDeleteKnowledgeBase, UseDeleteKnowledgeBaseParams } from './use_delete_knowledge_base'; -import { deleteKnowledgeBase as _deleteKnowledgeBase } from './api'; -import { useMutation as _useMutation } from '@tanstack/react-query'; - -const useMutationMock = _useMutation as jest.Mock; -const deleteKnowledgeBaseMock = _deleteKnowledgeBase as jest.Mock; - -jest.mock('./api', () => { - const actual = jest.requireActual('./api'); - return { - ...actual, - deleteKnowledgeBase: jest.fn((...args) => actual.deleteKnowledgeBase(...args)), - }; -}); -jest.mock('./use_knowledge_base_status'); - -jest.mock('@tanstack/react-query', () => ({ - useMutation: jest.fn().mockImplementation(async (queryKey, fn, opts) => { - try { - const res = await fn(); - return Promise.resolve(res); - } catch (e) { - opts.onError(e); - } - }), -})); - -const statusResponse = { - success: true, -}; - -const http = { - fetch: jest.fn().mockResolvedValue(statusResponse), -}; -const toasts = { - addError: jest.fn(), -}; -const defaultProps = { http, toasts } as unknown as UseDeleteKnowledgeBaseParams; - -describe('useDeleteKnowledgeBase', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - it('should call api to delete knowledge base', async () => { - await act(async () => { - const { waitForNextUpdate } = renderHook(() => useDeleteKnowledgeBase(defaultProps)); - await waitForNextUpdate(); - - expect(defaultProps.http.fetch).toHaveBeenCalledWith( - '/internal/elastic_assistant/knowledge_base/', - { - method: 'DELETE', - signal: undefined, - version: '1', - } - ); - expect(toasts.addError).not.toHaveBeenCalled(); - }); - }); - it('should call api to delete knowledge base with resource arg', async () => { - useMutationMock.mockImplementation(async (queryKey, fn, opts) => { - try { - const res = await fn('something'); - return Promise.resolve(res); - } catch (e) { - opts.onError(e); - } - }); - await act(async () => { - const { waitForNextUpdate } = renderHook(() => useDeleteKnowledgeBase(defaultProps)); - await waitForNextUpdate(); - - expect(defaultProps.http.fetch).toHaveBeenCalledWith( - '/internal/elastic_assistant/knowledge_base/something', - { - method: 'DELETE', - signal: undefined, - version: '1', - } - ); - }); - }); - - it('should return delete response', async () => { - await act(async () => { - const { result, waitForNextUpdate } = renderHook(() => useDeleteKnowledgeBase(defaultProps)); - await waitForNextUpdate(); - - await expect(result.current).resolves.toStrictEqual(statusResponse); - }); - }); - - it('should display error toast when api throws error', async () => { - deleteKnowledgeBaseMock.mockRejectedValue(new Error('this is an error')); - await act(async () => { - const { waitForNextUpdate } = renderHook(() => useDeleteKnowledgeBase(defaultProps)); - await waitForNextUpdate(); - - expect(toasts.addError).toHaveBeenCalled(); - }); - }); -}); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/use_delete_knowledge_base.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/use_delete_knowledge_base.tsx deleted file mode 100644 index 5e4ce82bde3bd..0000000000000 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/use_delete_knowledge_base.tsx +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { useMutation } from '@tanstack/react-query'; -import type { IToasts } from '@kbn/core-notifications-browser'; -import type { HttpSetup, IHttpFetchError, ResponseErrorBody } from '@kbn/core-http-browser'; -import { i18n } from '@kbn/i18n'; -import { deleteKnowledgeBase } from './api'; -import { useInvalidateKnowledgeBaseStatus } from './use_knowledge_base_status'; - -const DELETE_KNOWLEDGE_BASE_MUTATION_KEY = ['elastic-assistant', 'delete-knowledge-base']; - -export interface UseDeleteKnowledgeBaseParams { - http: HttpSetup; - toasts?: IToasts; -} - -/** - * Hook for deleting the Knowledge Base. Provide a resource name to delete a - * specific resource within KB. - * - * @param {Object} options - The options object. - * @param {HttpSetup} options.http - HttpSetup - * @param {IToasts} [options.toasts] - IToasts - * - * @returns {useMutation} hook for deleting the Knowledge Base - */ -export const useDeleteKnowledgeBase = ({ http, toasts }: UseDeleteKnowledgeBaseParams) => { - const invalidateKnowledgeBaseStatus = useInvalidateKnowledgeBaseStatus(); - return useMutation( - DELETE_KNOWLEDGE_BASE_MUTATION_KEY, - (resource?: string | void) => { - // Optional params workaround: see: https://github.com/TanStack/query/issues/1077#issuecomment-1431247266 - return deleteKnowledgeBase({ http, resource: resource ?? undefined }); - }, - { - onError: (error: IHttpFetchError) => { - if (error.name !== 'AbortError') { - toasts?.addError( - error.body && error.body.message ? new Error(error.body.message) : error, - { - title: i18n.translate('xpack.elasticAssistant.knowledgeBase.deleteError', { - defaultMessage: 'Error deleting Knowledge Base', - }), - } - ); - } - }, - onSettled: () => { - invalidateKnowledgeBaseStatus(); - }, - } - ); -}; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx index ef37506f2af17..406ef8be16c73 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_header/index.tsx @@ -23,7 +23,7 @@ import { Conversation } from '../../..'; import { AssistantTitle } from '../assistant_title'; import { ConnectorSelectorInline } from '../../connectorland/connector_selector_inline/connector_selector_inline'; import { FlyoutNavigation } from '../assistant_overlay/flyout_navigation'; -import { AssistantSettingsButton } from '../settings/assistant_settings_button'; +import { AssistantSettingsModal } from '../settings/assistant_settings_modal'; import * as i18n from './translations'; import { AIConnector } from '../../connectorland/connector_selector'; import { SettingsContextMenu } from '../settings/settings_context_menu/settings_context_menu'; @@ -113,7 +113,7 @@ export const AssistantHeader: React.FC = ({ > - { QUICK_PROMPTS_TAB, SYSTEM_PROMPTS_TAB, ])('%s', (tab) => { - it('Opens the tab on button click', () => { - (useAssistantContext as jest.Mock).mockImplementation(() => ({ - ...mockContext, - selectedSettingsTab: tab === CONVERSATIONS_TAB ? ANONYMIZATION_TAB : CONVERSATIONS_TAB, - })); - const { getByTestId } = render(, { - wrapper, - }); - fireEvent.click(getByTestId(`${tab}-button`)); - expect(setSelectedSettingsTab).toHaveBeenCalledWith(tab); - }); it('renders with the correct tab open', () => { (useAssistantContext as jest.Mock).mockImplementation(() => ({ ...mockContext, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx index 350780ea5b168..f325e411bae2b 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings.tsx @@ -9,14 +9,10 @@ import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { EuiButton, EuiButtonEmpty, - EuiIcon, EuiModal, EuiModalFooter, - EuiKeyPadMenu, - EuiKeyPadMenuItem, EuiPage, EuiPageBody, - EuiPageSidebar, EuiSplitPanel, } from '@elastic/eui'; @@ -80,16 +76,7 @@ export const AssistantSettings: React.FC = React.memo( conversations, conversationsLoaded, }) => { - const { - assistantFeatures: { - assistantModelEvaluation: modelEvaluatorEnabled, - assistantKnowledgeBaseByDefault, - }, - http, - toasts, - selectedSettingsTab, - setSelectedSettingsTab, - } = useAssistantContext(); + const { http, toasts, selectedSettingsTab, setSelectedSettingsTab } = useAssistantContext(); useEffect(() => { if (selectedSettingsTab == null) { @@ -214,115 +201,6 @@ export const AssistantSettings: React.FC = React.memo( return ( - {!assistantKnowledgeBaseByDefault && ( - - - setSelectedSettingsTab(CONVERSATIONS_TAB)} - data-test-subj={`${CONVERSATIONS_TAB}-button`} - > - <> - - - - - setSelectedSettingsTab(QUICK_PROMPTS_TAB)} - data-test-subj={`${QUICK_PROMPTS_TAB}-button`} - > - <> - - - - - setSelectedSettingsTab(SYSTEM_PROMPTS_TAB)} - data-test-subj={`${SYSTEM_PROMPTS_TAB}-button`} - > - - - - setSelectedSettingsTab(ANONYMIZATION_TAB)} - data-test-subj={`${ANONYMIZATION_TAB}-button`} - > - - - setSelectedSettingsTab(KNOWLEDGE_BASE_TAB)} - data-test-subj={`${KNOWLEDGE_BASE_TAB}-button`} - > - - - {modelEvaluatorEnabled && ( - setSelectedSettingsTab(EVALUATION_TAB)} - data-test-subj={`${EVALUATION_TAB}-button`} - > - - - )} - - - )} - ({ ), })); -describe('AssistantSettingsButton', () => { +describe('AssistantSettingsModal', () => { beforeEach(() => { jest.clearAllMocks(); }); - it('Clicking the settings gear opens the conversations tab', () => { - const { getByTestId } = render(); - fireEvent.click(getByTestId('settings')); - expect(setSelectedSettingsTab).toHaveBeenCalledWith(CONVERSATIONS_TAB); - expect(setIsSettingsModalVisible).toHaveBeenCalledWith(true); - }); - it('Settings modal is visible and calls correct actions per click', () => { const { getByTestId } = render( - + ); fireEvent.click(getByTestId('on-close')); expect(setIsSettingsModalVisible).toHaveBeenCalledWith(false); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_modal.tsx similarity index 60% rename from x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx rename to x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_modal.tsx index 3d6544643ba3e..5f2d677adc9ee 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_button.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/assistant_settings_modal.tsx @@ -6,7 +6,6 @@ */ import React, { useCallback } from 'react'; -import { EuiButtonIcon, EuiToolTip } from '@elastic/eui'; import { QueryObserverResult, RefetchOptions, RefetchQueryFilters } from '@tanstack/react-query'; import { DataStreamApis } from '../use_data_stream_apis'; import { AIConnector } from '../../connectorland/connector_selector'; @@ -14,7 +13,6 @@ import { Conversation } from '../../..'; import { AssistantSettings } from './assistant_settings'; import * as i18n from './translations'; import { useAssistantContext } from '../../assistant_context'; -import { CONVERSATIONS_TAB } from './const'; interface Props { defaultConnector?: AIConnector; @@ -32,12 +30,11 @@ interface Props { } /** - * Gear button that opens the assistant settings modal + * Assistant settings modal */ -export const AssistantSettingsButton: React.FC = React.memo( +export const AssistantSettingsModal: React.FC = React.memo( ({ defaultConnector, - isDisabled = false, isSettingsModalVisible, setIsSettingsModalVisible, selectedConversationId, @@ -47,11 +44,7 @@ export const AssistantSettingsButton: React.FC = React.memo( refetchCurrentUserConversations, refetchPrompts, }) => { - const { - assistantFeatures: { assistantKnowledgeBaseByDefault }, - toasts, - setSelectedSettingsTab, - } = useAssistantContext(); + const { toasts } = useAssistantContext(); // Modal control functions const cleanupAndCloseModal = useCallback(() => { @@ -79,41 +72,20 @@ export const AssistantSettingsButton: React.FC = React.memo( [cleanupAndCloseModal, refetchCurrentUserConversations, refetchPrompts, toasts] ); - const handleShowConversationSettings = useCallback(() => { - setSelectedSettingsTab(CONVERSATIONS_TAB); - setIsSettingsModalVisible(true); - }, [setIsSettingsModalVisible, setSelectedSettingsTab]); - return ( - <> - {!assistantKnowledgeBaseByDefault && ( - - - - )} - - {isSettingsModalVisible && ( - - )} - + isSettingsModalVisible && ( + + ) ); } ); -AssistantSettingsButton.displayName = 'AssistantSettingsButton'; +AssistantSettingsModal.displayName = 'AssistantSettingsModal'; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/settings_context_menu/settings_context_menu.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/settings_context_menu/settings_context_menu.tsx index baed2ff4cdb86..7b55e994b47ad 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/settings_context_menu/settings_context_menu.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/settings_context_menu/settings_context_menu.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { ReactElement, useCallback, useMemo, useState } from 'react'; +import React, { useCallback, useMemo, useState } from 'react'; import { EuiFlexGroup, EuiFlexItem, @@ -32,11 +32,7 @@ interface Params { export const SettingsContextMenu: React.FC = React.memo( ({ isDisabled = false, onChatCleared }: Params) => { - const { - navigateToApp, - knowledgeBase, - assistantFeatures: { assistantKnowledgeBaseByDefault: enableKnowledgeBaseByDefault }, - } = useAssistantContext(); + const { navigateToApp, knowledgeBase } = useAssistantContext(); const [isPopoverOpen, setPopover] = useState(false); @@ -91,12 +87,11 @@ export const SettingsContextMenu: React.FC = React.memo( closePopover(); }, [closePopover, showAlertSettingsModal]); - // We are migrating away from the settings modal in favor of the new Stack Management UI - // Currently behind `assistantKnowledgeBaseByDefault` FF - const newItems: ReactElement[] = useMemo( + const items = useMemo( () => [ = React.memo( , = React.memo( , = React.memo( , = React.memo( , - ], - [ - handleNavigateToAnonymization, - handleNavigateToKnowledgeBase, - handleNavigateToSettings, - handleShowAlertsModal, - knowledgeBase.latestAlerts, - ] - ); - - const items = useMemo( - () => [ - ...(enableKnowledgeBaseByDefault ? newItems : []), = React.memo( , ], - [enableKnowledgeBaseByDefault, newItems, showDestroyModal] + [ + handleNavigateToAnonymization, + handleNavigateToKnowledgeBase, + handleNavigateToSettings, + handleShowAlertsModal, + knowledgeBase.latestAlerts, + showDestroyModal, + ] ); const handleReset = useCallback(() => { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings.test.tsx index 763a2578ee273..b44dc682218d0 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings.test.tsx @@ -47,15 +47,6 @@ const defaultProps = { }, setUpdatedKnowledgeBaseSettings, }; -const mockDelete = jest.fn(); -jest.mock('../assistant/api/knowledge_base/use_delete_knowledge_base', () => ({ - useDeleteKnowledgeBase: jest.fn(() => { - return { - mutate: mockDelete, - isLoading: false, - }; - }), -})); const mockSetup = jest.fn(); jest.mock('../assistant/api/knowledge_base/use_setup_knowledge_base', () => ({ diff --git a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.test.tsx index 180b88fc3cdc8..4900a6b0966e3 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.test.tsx @@ -32,7 +32,6 @@ const mockContext = { http: { get: jest.fn(), }, - assistantFeatures: { assistantKnowledgeBaseByDefault: true }, selectedSettingsTab: null, assistantAvailability: { isAssistantEnabled: true, @@ -175,17 +174,6 @@ describe('KnowledgeBaseSettingsManagement', () => { isLoading: false, }); }); - it('renders old kb settings when enableKnowledgeBaseByDefault is not enabled', () => { - (useAssistantContext as jest.Mock).mockImplementation(() => ({ - ...mockContext, - assistantFeatures: { - assistantKnowledgeBaseByDefault: false, - }, - })); - render(, { wrapper }); - - expect(screen.getByTestId('knowledge-base-settings')).toBeInTheDocument(); - }); it('renders loading spinner when data is not fetched', () => { (useKnowledgeBaseStatus as jest.Mock).mockReturnValue({ data: {}, isFetched: false }); render(, { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.tsx index 86b3594daa3cd..183e74a18247a 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.tsx @@ -48,7 +48,6 @@ import { Flyout } from '../../assistant/common/components/assistant_settings_man import { useFlyoutModalVisibility } from '../../assistant/common/components/assistant_settings_management/flyout/use_flyout_modal_visibility'; import { IndexEntryEditor } from './index_entry_editor'; import { DocumentEntryEditor } from './document_entry_editor'; -import { KnowledgeBaseSettings } from '../knowledge_base_settings'; import { SetupKnowledgeBaseButton } from '../setup_knowledge_base_button'; import { useDeleteKnowledgeBaseEntries } from '../../assistant/api/knowledge_base/entries/use_delete_knowledge_base_entries'; import { @@ -73,7 +72,6 @@ interface Params { export const KnowledgeBaseSettingsManagement: React.FC = React.memo(({ dataViews }) => { const { - assistantFeatures: { assistantKnowledgeBaseByDefault: enableKnowledgeBaseByDefault }, assistantAvailability: { hasManageGlobalKnowledgeBase, isAssistantEnabled }, http, toasts, @@ -162,7 +160,7 @@ export const KnowledgeBaseSettingsManagement: React.FC = React.memo(({ d } = useKnowledgeBaseEntries({ http, toasts, - enabled: enableKnowledgeBaseByDefault && isAssistantEnabled, + enabled: isAssistantEnabled, isRefetching: kbStatus?.is_setup_in_progress, }); @@ -332,21 +330,6 @@ export const KnowledgeBaseSettingsManagement: React.FC = React.memo(({ d } }, [createEntry, duplicateKBItem, resetStateAndCloseFlyout]); - if (!enableKnowledgeBaseByDefault) { - return ( - <> - - - - ); - } return ( <> diff --git a/x-pack/packages/kbn-elastic-assistant/impl/tour/knowledge_base/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/tour/knowledge_base/index.test.tsx index 4dfd4657212f8..898a97ec2e233 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/tour/knowledge_base/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/tour/knowledge_base/index.test.tsx @@ -35,9 +35,6 @@ describe('Attack discovery tour', () => { jest.clearAllMocks(); (useAssistantContext as jest.Mock).mockReturnValue({ navigateToApp, - assistantFeatures: { - assistantKnowledgeBaseByDefault: true, - }, }); jest.mocked(useLocalStorage).mockReturnValue([ { @@ -68,25 +65,6 @@ describe('Attack discovery tour', () => { expect(screen.queryByTestId('knowledgeBase-tour-step-2')).toBeNull(); }); - it('should not render any tour steps when knowledge base feature flag is not activated', () => { - (useAssistantContext as jest.Mock).mockReturnValue({ - navigateToApp, - assistantFeatures: { - assistantKnowledgeBaseByDefault: false, - }, - }); - render( - -

{'Hello world'}

- , - { - wrapper: TestProviders, - } - ); - expect(screen.queryByTestId('knowledgeBase-tour-step-1')).toBeNull(); - expect(screen.queryByTestId('knowledgeBase-tour-step-2')).toBeNull(); - }); - it('should not render any tour steps when tour is on step 2 and page is not knowledge base', () => { jest.mocked(useLocalStorage).mockReturnValue([ { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/tour/knowledge_base/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/tour/knowledge_base/index.tsx index f7ef0252147c0..8d71b4491a2fd 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/tour/knowledge_base/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/tour/knowledge_base/index.tsx @@ -28,10 +28,7 @@ const KnowledgeBaseTourComp: React.FC<{ children?: EuiTourStepProps['children']; isKbSettingsPage?: boolean; }> = ({ children, isKbSettingsPage = false }) => { - const { - navigateToApp, - assistantFeatures: { assistantKnowledgeBaseByDefault: enableKnowledgeBaseByDefault }, - } = useAssistantContext(); + const { navigateToApp } = useAssistantContext(); const [tourState, setTourState] = useLocalStorage( NEW_FEATURES_TOUR_STORAGE_KEYS.KNOWLEDGE_BASE, @@ -106,7 +103,7 @@ const KnowledgeBaseTourComp: React.FC<{ return () => clearTimeout(timer); }, []); - if (!enableKnowledgeBaseByDefault || isTestAutomation || !tourState?.isTourActive) { + if (isTestAutomation || !tourState?.isTourActive) { return children ?? null; } diff --git a/x-pack/packages/kbn-entities-schema/src/schema/entity.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity.ts index 9ab02e0931d9c..7bfe505face19 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/entity.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/entity.ts @@ -11,9 +11,9 @@ import { arrayOfStringsSchema } from './common'; export const entityBaseSchema = z.object({ id: z.string(), type: z.string(), - identity_fields: arrayOfStringsSchema, + identity_fields: z.union([arrayOfStringsSchema, z.string()]), display_name: z.string(), - metrics: z.record(z.string(), z.number()), + metrics: z.optional(z.record(z.string(), z.number())), definition_version: z.string(), schema_version: z.string(), definition_id: z.string(), @@ -24,10 +24,13 @@ export interface MetadataRecord { } const literalSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]); + type Literal = z.infer; -type Metadata = Literal | { [key: string]: Metadata } | Metadata[]; +interface Metadata { + [key: string]: Metadata | Literal | Literal[]; +} export const entityMetadataSchema: z.ZodType = z.lazy(() => - z.union([literalSchema, z.array(entityMetadataSchema), z.record(entityMetadataSchema)]) + z.record(z.string(), z.union([literalSchema, z.array(literalSchema), entityMetadataSchema])) ); export const entityLatestSchema = z @@ -39,3 +42,6 @@ export const entityLatestSchema = z ), }) .and(entityMetadataSchema); + +export type EntityInstance = z.infer; +export type EntityMetadata = z.infer; diff --git a/x-pack/packages/observability/observability_utils/array/join_by_key.test.ts b/x-pack/packages/observability/observability_utils/array/join_by_key.test.ts index 8e0fc6ad09479..bb1d5a2e2410e 100644 --- a/x-pack/packages/observability/observability_utils/array/join_by_key.test.ts +++ b/x-pack/packages/observability/observability_utils/array/join_by_key.test.ts @@ -221,4 +221,50 @@ describe('joinByKey', () => { }, }); }); + + it('deeply merges by unflatten keys', () => { + const joined = joinByKey( + [ + { + service: { + name: 'opbeans-node', + metrics: { + cpu: 0.1, + }, + }, + properties: { + foo: 'bar', + }, + }, + { + service: { + environment: 'prod', + metrics: { + memory: 0.5, + }, + }, + properties: { + foo: 'bar', + }, + }, + ], + 'properties.foo' + ); + + expect(joined).toEqual([ + { + service: { + name: 'opbeans-node', + environment: 'prod', + metrics: { + cpu: 0.1, + memory: 0.5, + }, + }, + properties: { + foo: 'bar', + }, + }, + ]); + }); }); diff --git a/x-pack/packages/observability/observability_utils/array/join_by_key.ts b/x-pack/packages/observability/observability_utils/array/join_by_key.ts index 54e8ecdaf409b..93ec4261d04dc 100644 --- a/x-pack/packages/observability/observability_utils/array/join_by_key.ts +++ b/x-pack/packages/observability/observability_utils/array/join_by_key.ts @@ -18,18 +18,29 @@ export type JoinedReturnType< } >; -type ArrayOrSingle = T | T[]; +function getValueByPath(obj: any, path: string): any { + return path.split('.').reduce((acc, keyPart) => { + // Check if acc is a valid object and has the key + return acc && Object.prototype.hasOwnProperty.call(acc, keyPart) ? acc[keyPart] : undefined; + }, obj); +} +type NestedKeys = T extends object + ? { [K in keyof T]: K extends string ? `${K}` | `${K}.${NestedKeys}` : never }[keyof T] + : never; + +type ArrayOrSingle = T | T[]; +type CombinedNestedKeys = (NestedKeys & NestedKeys) | (keyof T & keyof U); export function joinByKey< T extends Record, U extends UnionToIntersection, - V extends ArrayOrSingle + V extends ArrayOrSingle> >(items: T[], key: V): JoinedReturnType; export function joinByKey< T extends Record, U extends UnionToIntersection, - V extends ArrayOrSingle, + V extends ArrayOrSingle>, W extends JoinedReturnType, X extends (a: T, b: T) => ValuesType >(items: T[], key: V, mergeFn: X): W; @@ -45,7 +56,7 @@ export function joinByKey( items.forEach((current) => { // The key of the map is a stable JSON string of the values from given keys. // We need stable JSON string to support plain object values. - const stableKey = stableStringify(keys.map((k) => current[k])); + const stableKey = stableStringify(keys.map((k) => current[k] ?? getValueByPath(current, k))); if (map.has(stableKey)) { const item = map.get(stableKey); diff --git a/x-pack/packages/observability/observability_utils/es/client/create_observability_es_client.ts b/x-pack/packages/observability/observability_utils/es/client/create_observability_es_client.ts index 0011e0f17c1c0..09013dcd5a506 100644 --- a/x-pack/packages/observability/observability_utils/es/client/create_observability_es_client.ts +++ b/x-pack/packages/observability/observability_utils/es/client/create_observability_es_client.ts @@ -9,6 +9,7 @@ import type { ElasticsearchClient, Logger } from '@kbn/core/server'; import type { ESQLSearchResponse, ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; import { withSpan } from '@kbn/apm-utils'; import type { EsqlQueryRequest } from '@elastic/elasticsearch/lib/api/types'; +import { esqlResultToPlainObjects } from '../utils/esql_result_to_plain_objects'; type SearchRequest = ESSearchRequest & { index: string | string[]; @@ -16,6 +17,20 @@ type SearchRequest = ESSearchRequest & { size: number | boolean; }; +type EsqlQueryParameters = EsqlQueryRequest & { parseOutput?: boolean }; +type EsqlOutputParameters = Omit & { + parseOutput?: true; + format?: 'json'; + columnar?: false; +}; + +type EsqlParameters = EsqlOutputParameters | EsqlQueryParameters; + +export type InferEsqlResponseOf< + TOutput = unknown, + TParameters extends EsqlParameters = EsqlParameters +> = TParameters extends EsqlOutputParameters ? TOutput[] : ESQLSearchResponse; + /** * An Elasticsearch Client with a fully typed `search` method and built-in * APM instrumentation. @@ -25,7 +40,14 @@ export interface ObservabilityElasticsearchClient { operationName: string, parameters: TSearchRequest ): Promise>; - esql(operationName: string, parameters: EsqlQueryRequest): Promise; + esql( + operationName: string, + parameters: TQueryParams + ): Promise>; + esql( + operationName: string, + parameters: TQueryParams + ): Promise>; client: ElasticsearchClient; } @@ -40,11 +62,14 @@ export function createObservabilityEsClient({ }): ObservabilityElasticsearchClient { return { client, - esql(operationName: string, parameters: EsqlQueryRequest) { + esql( + operationName: string, + { parseOutput = true, format = 'json', columnar = false, ...parameters }: TSearchRequest + ) { logger.trace(() => `Request (${operationName}):\n${JSON.stringify(parameters, null, 2)}`); return withSpan({ name: operationName, labels: { plugin } }, () => { return client.esql.query( - { ...parameters }, + { ...parameters, format, columnar }, { querystring: { drop_null_columns: true, @@ -54,7 +79,11 @@ export function createObservabilityEsClient({ }) .then((response) => { logger.trace(() => `Response (${operationName}):\n${JSON.stringify(response, null, 2)}`); - return response as unknown as ESQLSearchResponse; + + const esqlResponse = response as unknown as ESQLSearchResponse; + + const shouldParseOutput = parseOutput && !columnar && format === 'json'; + return shouldParseOutput ? esqlResultToPlainObjects(esqlResponse) : esqlResponse; }) .catch((error) => { throw error; diff --git a/x-pack/packages/observability/observability_utils/es/utils/esql_result_to_plain_objects.ts b/x-pack/packages/observability/observability_utils/es/utils/esql_result_to_plain_objects.ts index 96049f75ef156..717983a2958c5 100644 --- a/x-pack/packages/observability/observability_utils/es/utils/esql_result_to_plain_objects.ts +++ b/x-pack/packages/observability/observability_utils/es/utils/esql_result_to_plain_objects.ts @@ -6,25 +6,28 @@ */ import type { ESQLSearchResponse } from '@kbn/es-types'; +import { unflattenObject } from '../../object/unflatten_object'; -export function esqlResultToPlainObjects>( +export function esqlResultToPlainObjects( result: ESQLSearchResponse -): T[] { +): TDocument[] { return result.values.map((row) => { - return row.reduce>((acc, value, index) => { - const column = result.columns[index]; + return unflattenObject( + row.reduce>((acc, value, index) => { + const column = result.columns[index]; - if (!column) { - return acc; - } + if (!column) { + return acc; + } - // Removes the type suffix from the column name - const name = column.name.replace(/\.(text|keyword)$/, ''); - if (!acc[name]) { - acc[name] = value; - } + // Removes the type suffix from the column name + const name = column.name.replace(/\.(text|keyword)$/, ''); + if (!acc[name]) { + acc[name] = value; + } - return acc; - }, {}); - }) as T[]; + return acc; + }, {}) + ) as TDocument; + }); } diff --git a/x-pack/packages/security-solution/features/src/assistant/kibana_sub_features.ts b/x-pack/packages/security-solution/features/src/assistant/kibana_sub_features.ts index dbf9505193ecf..f542933f13b0c 100644 --- a/x-pack/packages/security-solution/features/src/assistant/kibana_sub_features.ts +++ b/x-pack/packages/security-solution/features/src/assistant/kibana_sub_features.ts @@ -107,6 +107,7 @@ export const getAssistantSubFeaturesMap = ( ): Map => { const assistantSubFeaturesList: Array<[AssistantSubFeatureId, SubFeatureConfig]> = [ [AssistantSubFeatureId.updateAnonymization, updateAnonymizationSubFeature], + [AssistantSubFeatureId.manageGlobalKnowledgeBase, manageGlobalKnowledgeBaseSubFeature], ]; // Use the following code to add feature based on feature flag @@ -114,13 +115,6 @@ export const getAssistantSubFeaturesMap = ( // assistantSubFeaturesList.push([AssistantSubFeatureId.featureId, featureSubFeature]); // } - if (experimentalFeatures.assistantKnowledgeBaseByDefault) { - assistantSubFeaturesList.push([ - AssistantSubFeatureId.manageGlobalKnowledgeBase, - manageGlobalKnowledgeBaseSubFeature, - ]); - } - const assistantSubFeaturesMap = new Map( assistantSubFeaturesList ); diff --git a/x-pack/plugins/actions/docs/openapi/components/schemas/connector_types.yaml b/x-pack/plugins/actions/docs/openapi/components/schemas/connector_types.yaml index 1db9e155f2eec..db6262f04c010 100644 --- a/x-pack/plugins/actions/docs/openapi/components/schemas/connector_types.yaml +++ b/x-pack/plugins/actions/docs/openapi/components/schemas/connector_types.yaml @@ -4,7 +4,6 @@ description: The type of connector. For example, `.email`, `.index`, `.jira`, `. enum: - .bedrock - .gemini - - .inference - .cases-webhook - .d3security - .email diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/use_view_in_log_pattern_analysis_action.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/use_view_in_log_pattern_analysis_action.tsx index ec1f6774b6b46..d92ce014d68fb 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/use_view_in_log_pattern_analysis_action.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/use_view_in_log_pattern_analysis_action.tsx @@ -104,7 +104,7 @@ export const useViewInLogPatternAnalysisAction = (dataViewId?: string): TableIte return ( > to refer to indices, aliases\nand data streams. This can be useful for time series data, for example to access\ntoday's index:\n\n[source,esql]\n----\nFROM \n----\n\nUse comma-separated lists or wildcards to query multiple data streams, indices,\nor aliases:\n\n[source,esql]\n----\nFROM employees-00001,employees-*\n----\n", - }, - }, - ], - }, - status: 200, - }, - { - took: 3, - timed_out: false, - _shards: { - total: 1, - successful: 1, - skipped: 0, - failed: 0, - }, - hits: { - total: { - value: 14, - relation: 'eq', - }, - max_score: 0.034783483, - hits: [ - { - _index: '.kibana-elastic-ai-assistant-kb', - _id: '280d4882-0f64-4471-a268-669a3f8c958f', - _score: 0.034783483, - _ignored: ['text.keyword'], - _source: { - metadata: { - source: - '/Users/andrew.goldstein/Projects/forks/andrew-goldstein/kibana/x-pack/plugins/elastic_assistant/server/knowledge_base/esql/example_queries/esql_example_query_0001.asciidoc', - required: true, - kbResource: 'esql', - }, - vector: { - tokens: { - user: 1.1084619, - // truncated for mock - }, - model_id: '.elser_model_2', - }, - text: '[[esql-example-queries]]\n\nThe following is an example an ES|QL query:\n\n```\nFROM logs-*\n| WHERE NOT CIDR_MATCH(destination.ip, "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16")\n| STATS destcount = COUNT(destination.ip) by user.name, host.name\n| ENRICH ldap_lookup_new ON user.name\n| WHERE group.name IS NOT NULL\n| EVAL follow_up = CASE(\n destcount >= 100, "true",\n "false")\n| SORT destcount desc\n| KEEP destcount, host.name, user.name, group.name, follow_up\n```\n', - }, - }, - ], - }, - status: 200, - }, - ], -}; diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/raw_attack_discoveries.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/raw_attack_discoveries.ts deleted file mode 100644 index 1c43f112da2bb..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/raw_attack_discoveries.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -/** - * A mock response from invoking the `attack-discovery` tool. - * This is a JSON string that represents the response from the tool - */ -export const getRawAttackDiscoveriesMock = () => - '{\n "alertsContextCount": 20,\n "attackDiscoveries": [\n {\n "alertIds": [\n "9bb601522d0c0b83783488a27a3ede5bd6a788f4f1ceef07cc8f12ac55f27563",\n "b9d6df8ab34e36c6868c097ff28dd01075df85a5ac1f084ef569ee8c6a4cf660",\n "014b433c3436ef5325cadacc35b6cb2ba8932a9c2ea0ba26d899f95c6fb61395",\n "28017987e64abb6ac486f1410f977d97ebd3a7172189cfdf943a48a59b968066"\n ],\n "detailsMarkdown": "- {{ host.name cb186c4a-3d70-4878-8ffe-18d84b5df86f }} (macOS {{ host.os.version 13.4 }}) executed a suspicious process {{ process.name unix1 }} with command line {{ process.command_line /Users/james/unix1 /Users/james/library/Keychains/login.keychain-db TempTemp1234!! }}\\\\n- The process was spawned by another suspicious process {{ process.parent.name My Go Application.app }} with command line {{ process.parent.command_line /private/var/folders/_b/rmcpc65j6nv11ygrs50ctcjr0000gn/T/AppTranslocation/6D63F08A-011C-4511-8556-EAEF9AFD6340/d/Setup.app/Contents/MacOS/My Go Application.app }}\\\\n- The parent process was launched by the system process {{ process.parent.parent.name launchd }}\\\\n- Both the child and parent processes had untrusted code signatures\\\\n- The child process attempted to access the user\'s login keychain, potentially indicating credential theft",\n "entitySummaryMarkdown": "Suspicious activity on {{ host.name cb186c4a-3d70-4878-8ffe-18d84b5df86f }} by {{ user.name 3c8c81bd-0e52-4ce7-a836-48e718dfb6e4 }}",\n "mitreAttackTactics": [\n "Credential Access",\n "Defense Evasion",\n "Execution"\n ],\n "summaryMarkdown": "Suspicious activity detected on a macOS host involving a potentially malicious process attempting to access user credentials. The process was spawned by another untrusted process launched by the system, indicating a multi-stage attack potentially involving credential theft and defense evasion techniques.",\n "title": "Potential Credential Theft on macOS Host"\n },\n {\n "alertIds": [\n "64bcd8a322e6e6aebaee252982d0249cc96bdd75023ea05f58c228a7417c0dfc"\n ],\n "detailsMarkdown": "- {{ host.name cb186c4a-3d70-4878-8ffe-18d84b5df86f }} (macOS {{ host.os.version 13.4 }}) executed the system utility {{ process.name osascript }} with command line {{ process.command_line osascript -e display dialog \\"MacOS wants to access System Preferences\\\\n\\\\t\\\\t\\\\nPlease enter your password.\\" with title \\"System Preferences\\" with icon file \\"System:Library:CoreServices:CoreTypes.bundle:Contents:Resources:ToolbarAdvanced.icns\\" default answer \\"\\" giving up after 30 with hidden answer ¬ }}\\\\n- This appears to be an attempt to phish for user credentials by displaying a fake system dialog\\\\n- The osascript process was spawned by the suspicious process {{ process.parent.name My Go Application.app }} with untrusted code signature",\n "entitySummaryMarkdown": "Potential credential phishing attempt on {{ host.name cb186c4a-3d70-4878-8ffe-18d84b5df86f }} targeting {{ user.name 3c8c81bd-0e52-4ce7-a836-48e718dfb6e4 }}",\n "mitreAttackTactics": [\n "Credential Access",\n "Initial Access",\n "Execution"\n ],\n "summaryMarkdown": "A credential phishing attempt was detected on a macOS host, likely initiated by a malicious process. The attack used osascript to display a fake system dialog prompting the user to enter their password.",\n "title": "Credential Phishing Attempt on macOS"\n },\n {\n "alertIds": [\n "245b60b908ddd84cad06671e273aa7be50699abd27e59423be4415f38c4aeb99",\n "616ac711e967e07a9b725e66aa93321eabf29e4b51f9598a4a11f21ab7ed0f12",\n "035c0295b1c64fd2ebba1b751a3565fd6759942247e9df6e1496c5e332d51840"\n ],\n "detailsMarkdown": "- {{ host.name cb186c4a-3d70-4878-8ffe-18d84b5df86f }} (macOS {{ host.os.version 13.4 }}) executed a suspicious process {{ process.name My Go Application.app }} with command line {{ process.command_line xpcproxy application.Appify by Machine Box.My Go Application.20.23 }}\\\\n- This process had an untrusted code signature and was launched by the system process {{ process.parent.name launchd }}\\\\n- It appears to have spawned the process {{ process.name unix1 }} in an attempt to obfuscate its activities\\\\n- The unix1 process attempted to make itself executable by running {{ process.name chmod }} with arguments {{ process.command_line chmod 777 /Users/james/unix1 }}",\n "entitySummaryMarkdown": "Suspicious activity involving process obfuscation on {{ host.name cb186c4a-3d70-4878-8ffe-18d84b5df86f }} by {{ user.name fec12d87-2476-4b82-a50d-0829f3815a42 }}",\n "mitreAttackTactics": [\n "Defense Evasion",\n "Execution"\n ],\n "summaryMarkdown": "A suspicious process was detected on a macOS host that appeared to be attempting to obfuscate its activities by spawning other processes and making them executable. The initial process had an untrusted code signature, indicating potentially malicious intent.",\n "title": "Process Obfuscation on macOS Host"\n },\n {\n "alertIds": [\n "54901fb5b0ed88f0c8d737613868a3d62ebc541d31b757349bbe7999d868ce48"\n ],\n "detailsMarkdown": "- {{ host.name 23166d28-d6da-4801-b701-d21ce1a489e5 }} (Windows {{ host.os.version 21H2 (10.0.20348.1607) }}) created a suspicious script file {{ file.path C:\\\\ProgramData\\\\WindowsAppPool\\\\AppPool.vbs }}\\\\n- The file was created by a Microsoft Word process ({{ process.name WINWORD.EXE }}) with trusted code signature\\\\n- This may indicate an attempt to establish persistence or command-and-control through scripting",\n "entitySummaryMarkdown": "Suspicious script file created on {{ host.name 23166d28-d6da-4801-b701-d21ce1a489e5 }} by {{ user.name 45bec1b8-eb98-4ddc-aafb-e3f7e02236dc }}",\n "mitreAttackTactics": [\n "Command and Control",\n "Execution"\n ],\n "summaryMarkdown": "A suspicious VBScript file was created on a Windows host, potentially by an compromised Microsoft Word process. This may be an attempt to establish persistence or command-and-control capabilities through scripting.",\n "title": "Suspicious Script File Creation on Windows"\n },\n {\n "alertIds": [\n "7fe0025f2d2b0d32f04b0e533466666967a21a98adae7499cb05add3355b48fc",\n "3875cbad10604636b892d15f7ff753a02a37d3e4bbe91a39a0fcf72f89101e31",\n "bb2767ebef06a5dc2511e2b865f5ed012dfdf20081bc33cab5c9f20b99e01d8f",\n "76d99c72442819a019dfbf3936cda9a6c5713d84a9ae685b2c4e0bb55e5b9862",\n "0f985965cb3d3b14007873290b9fc8f26f1b6ca0945499dfb693787ea6569265"\n ],\n "detailsMarkdown": "- {{ host.name 9a0ea998-7ce5-4dbb-a690-9856eca617ac }} (Windows {{ host.os.version 21H2 (10.0.20348.1607) }}) executed a suspicious PowerShell script {{ process.command_line \\"C:\\\\Windows\\\\System32\\\\WindowsPowerShell\\\\v1.0\\\\powershell.exe\\" -exec bypass -file C:\\\\ProgramData\\\\WindowsAppPool\\\\AppPool.ps1 }}\\\\n- The script was launched by the wscript process, which was spawned by a Microsoft Word process ({{ process.parent.name WINWORD.EXE }})\\\\n- The Word process also created a scheduled task to periodically execute the script\\\\n- The PowerShell script appears to be obfuscated, potentially to hide malicious activities\\\\n- This chain of events indicates a multi-stage attack potentially initiated by a malicious Office document",\n "entitySummaryMarkdown": "Suspicious PowerShell activity on {{ host.name 9a0ea998-7ce5-4dbb-a690-9856eca617ac }} by {{ user.name 45bec1b8-eb98-4ddc-aafb-e3f7e02236dc }}",\n "mitreAttackTactics": [\n "Initial Access",\n "Execution",\n "Defense Evasion"\n ],\n "summaryMarkdown": "A multi-stage attack was detected on a Windows host, potentially initiated by a malicious Microsoft Office document. The attack involved creating a scheduled task to execute an obfuscated PowerShell script, likely to hide malicious activities. This indicates techniques for initial access, execution, and defense evasion.",\n "title": "Multi-Stage Attack on Windows Host"\n },\n {\n "alertIds": [\n "a0c49fb228eca1685bd41df0ab66ca1977140de7916663e7a0918087220dd402",\n "a252ca3096831e3eeab07ab70e9269f98b5a66617b44d709425898813326ca63",\n "0ff7d411ca25a5b851e43562c9c660062624498f908ff4b63590d4b5304682af",\n "4d612c721e432598a5b7ea7bbeb2aaa2944c0a35e263d9984297b5416530c88f"\n ],\n "detailsMarkdown": "- {{ host.name 634eb7d8-0ce0-4591-b5f5-fb65803b89d8 }} (Windows {{ host.os.version 21H2 (10.0.20348.1607) }}) executed a suspicious PowerShell script {{ process.command_line \\"C:\\\\Windows\\\\System32\\\\WindowsPowerShell\\\\v1.0\\\\powershell.exe\\" -ep bypass -file \\"C:\\\\Users\\\\ADMINI~1\\\\AppData\\\\Local\\\\Temp\\\\2\\\\Package Installation Dir\\\\chch.ps1\\" }}\\\\n- The script was launched by the msiexec.exe process, which may indicate an attempt to use a trusted Windows utility for defense evasion\\\\n- Elastic Endpoint detected the Bb malware family in the PowerShell process memory\\\\n- The PowerShell process also made network connections, potentially for command-and-control or data exfiltration",\n "entitySummaryMarkdown": "Malware detected on {{ host.name 634eb7d8-0ce0-4591-b5f5-fb65803b89d8 }} targeting {{ user.name 45bec1b8-eb98-4ddc-aafb-e3f7e02236dc }}",\n "mitreAttackTactics": [\n "Defense Evasion",\n "Execution"\n ],\n "summaryMarkdown": "The B malware was detected on a Windows host, executed through a PowerShell script launched by the msiexec.exe process. This appears to be an attempt to use a trusted Windows utility for defense evasion. The malware process also made network connections, potentially for command-and-control or data exfiltration.",\n "title": "Bb Malware Execution on Windows"\n },\n {\n "alertIds": [\n "764c0944288db1704f7a0fff2db7fe19e8285fa4272dec828ae4186ba0dfd3b3",\n "85672064aeb762a1121139a6d98fd3c5f6be8f18b49e4504c3f5e5a36679afe7"\n ],\n "detailsMarkdown": "- {{ host.name d813c7ba-6141-4292-8f40-c800c27645a4 }} (Linux {{ host.os.version 22.04.1 }}) executed a suspicious process {{ process.command_line sh -c /bin/rm -f /dev/shm/kdmtmpflush;/bin/cp ./74ef6cc38f5a1a80148752b63c117e6846984debd2af806c65887195a8eccc56 /dev/shm/kdmtmpflush && /bin/chmod 755 /dev/shm/kdmtmpflush && /dev/shm/kdmtmpflush --init && /bin/rm -f /dev/shm/kdmtmpflush }}\\\\n- This copied a file with SHA256 hash {{ file.hash.sha256 74ef6cc38f5a1a80148752b63c117e6846984debd2af806c65887195a8eccc56 }} to /dev/shm/kdmtmpflush, made it executable, and executed it\\\\n- Elastic Endpoint detected the Door malware family associated with this file",\n "entitySummaryMarkdown": "Malware executed on {{ host.name d813c7ba-6141-4292-8f40-c800c27645a4 }} by {{ user.name fec12d87-2476-4b82-a50d-0829f3815a42 }}",\n "mitreAttackTactics": [\n "Execution"\n ],\n "summaryMarkdown": "The Door malware was executed on a Linux host by copying an untrusted file to a temporary path, making it executable, and running it. This indicates malicious code execution on the compromised system.",\n "title": "Door Malware Execution on Linux"\n }\n ]\n}'; - -export const getRawAttackDiscoveriesReplacementsMock = () => ({ - '3c8c81bd-0e52-4ce7-a836-48e718dfb6e4': 'james', - 'cb186c4a-3d70-4878-8ffe-18d84b5df86f': 'SRVMAC08', - 'fec12d87-2476-4b82-a50d-0829f3815a42': 'root', - '45bec1b8-eb98-4ddc-aafb-e3f7e02236dc': 'Administrator', - '23166d28-d6da-4801-b701-d21ce1a489e5': 'SRVWIN07-PRIV', - '9a0ea998-7ce5-4dbb-a690-9856eca617ac': 'SRVWIN07', - '634eb7d8-0ce0-4591-b5f5-fb65803b89d8': 'SRVWIN06', - 'd813c7ba-6141-4292-8f40-c800c27645a4': 'SRVNIX05', -}); diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts index f6f3007c8f948..698645e8d3c55 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts @@ -67,13 +67,6 @@ export const getPostKnowledgeBaseRequest = (resource?: string) => query: { resource }, }); -export const getDeleteKnowledgeBaseRequest = (resource?: string) => - requestMock.create({ - method: 'delete', - path: ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_URL, - query: { resource }, - }); - export const getGetCapabilitiesRequest = () => requestMock.create({ method: 'get', diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts index d53ceaa586975..a065c7de42586 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts @@ -127,11 +127,11 @@ const createElasticAssistantRequestContextMock = ( () => clients.elasticAssistant.getAIAssistantKnowledgeBaseDataClient ) as unknown as jest.MockInstance< Promise, - [params: GetAIAssistantKnowledgeBaseDataClientParams], + [params?: GetAIAssistantKnowledgeBaseDataClientParams], unknown > & (( - params: GetAIAssistantKnowledgeBaseDataClientParams + params?: GetAIAssistantKnowledgeBaseDataClientParams ) => Promise), getCurrentUser: jest.fn(), getServerBasePath: jest.fn(), diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/terms.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/terms.ts deleted file mode 100644 index 0606c905d6df3..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/terms.ts +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { Field, FieldValue, QueryDslTermQuery } from '@elastic/elasticsearch/lib/api/types'; - -/** - * These (mock) terms may be used in multiple queries. - * - * For example, it may be be used in a vector search to exclude the required `esql` KB docs. - * - * It may also be used in a terms search to find all of the required `esql` KB docs. - */ -export const mockTerms: Array>> = [ - { - term: { - 'metadata.kbResource': 'esql', - }, - }, - { - term: { - 'metadata.required': true, - }, - }, -]; diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/terms_search_query.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/terms_search_query.ts deleted file mode 100644 index c8af748516a1f..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/terms_search_query.ts +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; - -/** - * This Elasticsearch query DSL is a terms search for required `esql` KB docs - */ -export const mockTermsSearchQuery: QueryDslQueryContainer = { - bool: { - must: [ - { - term: { - 'metadata.kbResource': 'esql', - }, - }, - { - term: { - 'metadata.required': true, - }, - }, - ], - }, -}; diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/vector_search_query.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/vector_search_query.ts deleted file mode 100644 index 04263c5d242bb..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/vector_search_query.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; - -/** - * A mock vector search query DSL - */ -export const mockVectorSearchQuery: QueryDslQueryContainer = { - bool: { - must_not: [ - { - term: { - 'metadata.kbResource': 'esql', - }, - }, - { - term: { - 'metadata.required': true, - }, - }, - ], - must: [ - { - semantic: { - field: 'semantic_text', - query: - 'Generate an ES|QL query that will count the number of connections made to external IP addresses, broken down by user. If the count is greater than 100 for a specific user, add a new field called "follow_up" that contains a value of "true", otherwise, it should contain "false". The user names should also be enriched with their respective group names.', - }, - }, - ], - }, -} as QueryDslQueryContainer; diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/create_knowledge_base_entry.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/create_knowledge_base_entry.ts index 77a1e37df965f..8e1d749c7f78b 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/create_knowledge_base_entry.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/create_knowledge_base_entry.ts @@ -14,11 +14,9 @@ import { } from '@kbn/core/server'; import { - DocumentEntryCreateFields, KnowledgeBaseEntryCreateProps, KnowledgeBaseEntryResponse, KnowledgeBaseEntryUpdateProps, - Metadata, } from '@kbn/elastic-assistant-common'; import { CREATE_KNOWLEDGE_BASE_ENTRY_ERROR_EVENT, @@ -33,9 +31,8 @@ export interface CreateKnowledgeBaseEntryParams { logger: Logger; spaceId: string; user: AuthenticatedUser; - knowledgeBaseEntry: KnowledgeBaseEntryCreateProps | LegacyKnowledgeBaseEntryCreateProps; + knowledgeBaseEntry: KnowledgeBaseEntryCreateProps; global?: boolean; - isV2?: boolean; telemetry: AnalyticsServiceSetup; } @@ -47,25 +44,16 @@ export const createKnowledgeBaseEntry = async ({ knowledgeBaseEntry, logger, global = false, - isV2 = false, telemetry, }: CreateKnowledgeBaseEntryParams): Promise => { const createdAt = new Date().toISOString(); - const body = isV2 - ? transformToCreateSchema({ - createdAt, - spaceId, - user, - entry: knowledgeBaseEntry as unknown as KnowledgeBaseEntryCreateProps, - global, - }) - : transformToLegacyCreateSchema({ - createdAt, - spaceId, - user, - entry: knowledgeBaseEntry as unknown as TransformToLegacyCreateSchemaProps['entry'], - global, - }); + const body = transformToCreateSchema({ + createdAt, + spaceId, + user, + entry: knowledgeBaseEntry as unknown as KnowledgeBaseEntryCreateProps, + global, + }); const telemetryPayload = { entryType: body.type, required: body.required ?? false, @@ -156,13 +144,7 @@ export const transformToUpdateSchema = ({ }; }; -export const getUpdateScript = ({ - entry, - isPatch, -}: { - entry: UpdateKnowledgeBaseEntrySchema; - isPatch?: boolean; -}) => { +export const getUpdateScript = ({ entry }: { entry: UpdateKnowledgeBaseEntrySchema }) => { // Cannot use script for updating documents with semantic_text fields return { doc: { @@ -230,45 +212,3 @@ export const transformToCreateSchema = ({ semantic_text: entry.text, }; }; - -export type LegacyKnowledgeBaseEntryCreateProps = Omit< - DocumentEntryCreateFields, - 'kbResource' | 'source' -> & { - metadata: Metadata; -}; - -interface TransformToLegacyCreateSchemaProps { - createdAt: string; - spaceId: string; - user: AuthenticatedUser; - entry: LegacyKnowledgeBaseEntryCreateProps; - global?: boolean; -} - -export const transformToLegacyCreateSchema = ({ - createdAt, - spaceId, - user, - entry, - global = false, -}: TransformToLegacyCreateSchemaProps): CreateKnowledgeBaseEntrySchema => { - return { - '@timestamp': createdAt, - created_at: createdAt, - created_by: user.profile_uid ?? 'unknown', - updated_at: createdAt, - updated_by: user.profile_uid ?? 'unknown', - namespace: spaceId, - users: global - ? [] - : [ - { - id: user.profile_uid, - name: user.username, - }, - ], - ...entry, - vector: undefined, - }; -}; diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/field_maps_configuration.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/field_maps_configuration.ts index 348efb5a18f7d..1a075202cf3cd 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/field_maps_configuration.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/field_maps_configuration.ts @@ -9,89 +9,6 @@ import { FieldMap } from '@kbn/data-stream-adapter'; export const ASSISTANT_ELSER_INFERENCE_ID = 'elastic-security-ai-assistant-elser2'; export const knowledgeBaseFieldMap: FieldMap = { - '@timestamp': { - type: 'date', - array: false, - required: false, - }, - id: { - type: 'keyword', - array: false, - required: true, - }, - created_at: { - type: 'date', - array: false, - required: false, - }, - created_by: { - type: 'keyword', - array: false, - required: false, - }, - updated_at: { - type: 'date', - array: false, - required: false, - }, - updated_by: { - type: 'keyword', - array: false, - required: false, - }, - users: { - type: 'nested', - array: true, - required: false, - }, - 'users.id': { - type: 'keyword', - array: false, - required: true, - }, - 'users.name': { - type: 'keyword', - array: false, - required: false, - }, - metadata: { - type: 'object', - array: false, - required: false, - }, - 'metadata.kbResource': { - type: 'keyword', - array: false, - required: false, - }, - 'metadata.required': { - type: 'boolean', - array: false, - required: false, - }, - 'metadata.source': { - type: 'keyword', - array: false, - required: false, - }, - text: { - type: 'text', - array: false, - required: true, - }, - vector: { - type: 'object', - array: false, - required: false, - }, - 'vector.tokens': { - type: 'rank_features', - array: false, - required: false, - }, -} as const; - -export const knowledgeBaseFieldMapV2: FieldMap = { // Base fields '@timestamp': { type: 'date', diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/helpers.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/helpers.ts index a19b3f0945086..88ecae26cf19f 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/helpers.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/helpers.ts @@ -27,37 +27,29 @@ export const isModelAlreadyExistsError = (error: Error) => { * * @param filter - Optional filter to apply to the search * @param kbResource - Specific resource tag to filter for, e.g. 'esql' or 'user' - * @param modelId - ID of the model to search with, e.g. `.elser_model_2` * @param query - The search query provided by the user * @param required - Whether to only include required entries * @param user - The authenticated user - * @param v2KnowledgeBaseEnabled whether the new v2 KB is enabled * @returns */ export const getKBVectorSearchQuery = ({ filter, kbResource, - modelId, query, required, user, - v2KnowledgeBaseEnabled = false, }: { filter?: QueryDslQueryContainer | undefined; kbResource?: string | undefined; - modelId: string; query?: string; required?: boolean | undefined; user: AuthenticatedUser; - v2KnowledgeBaseEnabled: boolean; }): QueryDslQueryContainer => { - const kbResourceKey = v2KnowledgeBaseEnabled ? 'kb_resource' : 'metadata.kbResource'; - const requiredKey = v2KnowledgeBaseEnabled ? 'required' : 'metadata.required'; const resourceFilter = kbResource ? [ { term: { - [kbResourceKey]: kbResource, + kb_resource: kbResource, }, }, ] @@ -66,7 +58,7 @@ export const getKBVectorSearchQuery = ({ ? [ { term: { - [requiredKey]: required, + required, }, }, ] @@ -120,7 +112,7 @@ export const getKBVectorSearchQuery = ({ text_expansion: { 'vector.tokens': { model_id: string; model_text: string } }; }> = []; - if (v2KnowledgeBaseEnabled && query) { + if (query) { semanticTextFilter = [ { semantic: { @@ -129,17 +121,6 @@ export const getKBVectorSearchQuery = ({ }, }, ]; - } else if (!v2KnowledgeBaseEnabled) { - semanticTextFilter = [ - { - text_expansion: { - 'vector.tokens': { - model_id: modelId, - model_text: query as string, - }, - }, - }, - ]; } return { diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/index.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/index.ts index 50e124321fe6c..fae987b6d5083 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/index.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/index.ts @@ -29,20 +29,11 @@ import { AnalyticsServiceSetup, ElasticsearchClient } from '@kbn/core/server'; import { IndexPatternsFetcher } from '@kbn/data-views-plugin/server'; import { map } from 'lodash'; import { AIAssistantDataClient, AIAssistantDataClientParams } from '..'; -import { AssistantToolParams, GetElser } from '../../types'; -import { - createKnowledgeBaseEntry, - LegacyKnowledgeBaseEntryCreateProps, - transformToCreateSchema, - transformToLegacyCreateSchema, -} from './create_knowledge_base_entry'; +import { GetElser } from '../../types'; +import { createKnowledgeBaseEntry, transformToCreateSchema } from './create_knowledge_base_entry'; import { EsDocumentEntry, EsIndexEntry, EsKnowledgeBaseEntrySchema } from './types'; import { transformESSearchToKnowledgeBaseEntry } from './transforms'; -import { - ESQL_DOCS_LOADED_QUERY, - SECURITY_LABS_RESOURCE, - USER_RESOURCE, -} from '../../routes/knowledge_base/constants'; +import { SECURITY_LABS_RESOURCE, USER_RESOURCE } from '../../routes/knowledge_base/constants'; import { getKBVectorSearchQuery, getStructuredToolForIndexEntry, @@ -61,7 +52,6 @@ import { ASSISTANT_ELSER_INFERENCE_ID } from './field_maps_configuration'; */ export interface GetAIAssistantKnowledgeBaseDataClientParams { modelIdOverride?: string; - v2KnowledgeBaseEnabled?: boolean; manageGlobalKnowledgeBaseAIAssistant?: boolean; } @@ -71,7 +61,6 @@ interface KnowledgeBaseDataClientParams extends AIAssistantDataClientParams { getIsKBSetupInProgress: () => boolean; ingestPipelineResourceName: string; setIsKBSetupInProgress: (isInProgress: boolean) => void; - v2KnowledgeBaseEnabled: boolean; manageGlobalKnowledgeBaseAIAssistant: boolean; } export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient { @@ -82,11 +71,6 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient { public get isSetupInProgress() { return this.options.getIsKBSetupInProgress(); } - - public get isV2KnowledgeBaseEnabled() { - return this.options.v2KnowledgeBaseEnabled; - } - /** * Returns whether setup of the Knowledge Base can be performed (essentially an ML features check) * @@ -150,70 +134,39 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient { }; /** - * Deploy the ELSER model with default configuration - */ - private deployModel = async () => { - const elserId = await this.options.getElserId(); - this.options.logger.debug(`Deploying ELSER model '${elserId}'...`); - try { - const esClient = await this.options.elasticsearchClientPromise; - await esClient.ml.startTrainedModelDeployment({ - model_id: elserId, - wait_for: 'fully_allocated', - }); - } catch (error) { - this.options.logger.error(`Error deploying ELSER model '${elserId}':\n${error}`); - throw new Error(`Error deploying ELSER model '${elserId}':\n${error}`); - } - }; - - /** - * Checks if the provided model is deployed and allocated in Elasticsearch + * Checks if the inference endpoint is deployed and allocated in Elasticsearch * * @returns Promise indicating whether the model is deployed */ - public isModelDeployed = async (): Promise => { - const elserId = await this.options.getElserId(); - this.options.logger.debug(`Checking if ELSER model '${elserId}' is deployed...`); - - try { - if (this.isV2KnowledgeBaseEnabled) { - return await this.isInferenceEndpointExists(); - } else { - const esClient = await this.options.elasticsearchClientPromise; - const getResponse = await esClient.ml.getTrainedModelsStats({ - model_id: elserId, - }); - - // For standardized way of checking deployment status see: https://github.com/elastic/elasticsearch/issues/106986 - const isReadyESS = (stats: MlTrainedModelStats) => - stats.deployment_stats?.state === 'started' && - stats.deployment_stats?.allocation_status.state === 'fully_allocated'; - - const isReadyServerless = (stats: MlTrainedModelStats) => - (stats.deployment_stats?.nodes as unknown as MlTrainedModelDeploymentNodesStats[])?.some( - (node) => node.routing_state.routing_state === 'started' - ); - - return getResponse.trained_model_stats?.some( - (stats) => isReadyESS(stats) || isReadyServerless(stats) - ); - } - } catch (e) { - this.options.logger.debug(`Error checking if ELSER model '${elserId}' is deployed: ${e}`); - // Returns 404 if it doesn't exist - return false; - } - }; - public isInferenceEndpointExists = async (): Promise => { try { const esClient = await this.options.elasticsearchClientPromise; - return !!(await esClient.inference.get({ + const inferenceExists = !!(await esClient.inference.get({ inference_id: ASSISTANT_ELSER_INFERENCE_ID, task_type: 'sparse_embedding', })); + if (!inferenceExists) { + return false; + } + const elserId = await this.options.getElserId(); + const getResponse = await esClient.ml.getTrainedModelsStats({ + model_id: elserId, + }); + + // For standardized way of checking deployment status see: https://github.com/elastic/elasticsearch/issues/106986 + const isReadyESS = (stats: MlTrainedModelStats) => + stats.deployment_stats?.state === 'started' && + stats.deployment_stats?.allocation_status.state === 'fully_allocated'; + + const isReadyServerless = (stats: MlTrainedModelStats) => + (stats.deployment_stats?.nodes as unknown as MlTrainedModelDeploymentNodesStats[])?.some( + (node) => node.routing_state.routing_state === 'started' + ); + + return getResponse.trained_model_stats?.some( + (stats) => isReadyESS(stats) || isReadyServerless(stats) + ); } catch (error) { this.options.logger.debug( `Error checking if Inference endpoint ${ASSISTANT_ELSER_INFERENCE_ID} exists: ${error}` @@ -227,25 +180,24 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient { this.options.logger.debug(`Deploying ELSER model '${elserId}'...`); try { const esClient = await this.options.elasticsearchClientPromise; - if (this.isV2KnowledgeBaseEnabled) { - await esClient.inference.put({ - task_type: 'sparse_embedding', - inference_id: ASSISTANT_ELSER_INFERENCE_ID, - inference_config: { - service: 'elasticsearch', - service_settings: { - adaptive_allocations: { - enabled: true, - min_number_of_allocations: 0, - max_number_of_allocations: 8, - }, - num_threads: 1, - model_id: elserId, + + await esClient.inference.put({ + task_type: 'sparse_embedding', + inference_id: ASSISTANT_ELSER_INFERENCE_ID, + inference_config: { + service: 'elasticsearch', + service_settings: { + adaptive_allocations: { + enabled: true, + min_number_of_allocations: 0, + max_number_of_allocations: 8, }, - task_settings: {}, + num_threads: 1, + model_id: elserId, }, - }); - } + task_settings: {}, + }, + }); } catch (error) { this.options.logger.error( `Error creating inference endpoint for ELSER model '${elserId}':\n${error}` @@ -268,11 +220,9 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient { */ public setupKnowledgeBase = async ({ soClient, - v2KnowledgeBaseEnabled = true, ignoreSecurityLabs = false, }: { soClient: SavedObjectsClientContract; - v2KnowledgeBaseEnabled?: boolean; ignoreSecurityLabs?: boolean; }): Promise => { if (this.options.getIsKBSetupInProgress()) { @@ -284,40 +234,38 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient { this.options.setIsKBSetupInProgress(true); const elserId = await this.options.getElserId(); - if (v2KnowledgeBaseEnabled) { - // Delete legacy ESQL knowledge base docs if they exist, and silence the error if they do not - try { - const esClient = await this.options.elasticsearchClientPromise; - const legacyESQL = await esClient.deleteByQuery({ - index: this.indexTemplateAndPattern.alias, - query: { - bool: { - must: [{ terms: { 'metadata.kbResource': ['esql', 'unknown'] } }], - }, + // Delete legacy ESQL knowledge base docs if they exist, and silence the error if they do not + try { + const esClient = await this.options.elasticsearchClientPromise; + const legacyESQL = await esClient.deleteByQuery({ + index: this.indexTemplateAndPattern.alias, + query: { + bool: { + must: [{ terms: { 'metadata.kbResource': ['esql', 'unknown'] } }], }, - }); - if (legacyESQL?.total != null && legacyESQL?.total > 0) { - this.options.logger.info( - `Removed ${legacyESQL?.total} ESQL knowledge base docs from knowledge base data stream: ${this.indexTemplateAndPattern.alias}.` - ); - } - // Delete any existing Security Labs content - const securityLabsDocs = await esClient.deleteByQuery({ - index: this.indexTemplateAndPattern.alias, - query: { - bool: { - must: [{ terms: { kb_resource: [SECURITY_LABS_RESOURCE] } }], - }, + }, + }); + if (legacyESQL?.total != null && legacyESQL?.total > 0) { + this.options.logger.info( + `Removed ${legacyESQL?.total} ESQL knowledge base docs from knowledge base data stream: ${this.indexTemplateAndPattern.alias}.` + ); + } + // Delete any existing Security Labs content + const securityLabsDocs = await esClient.deleteByQuery({ + index: this.indexTemplateAndPattern.alias, + query: { + bool: { + must: [{ terms: { kb_resource: [SECURITY_LABS_RESOURCE] } }], }, - }); - if (securityLabsDocs?.total) { - this.options.logger.info( - `Removed ${securityLabsDocs?.total} Security Labs knowledge base docs from knowledge base data stream: ${this.indexTemplateAndPattern.alias}.` - ); - } - } catch (e) { - this.options.logger.info('No legacy ESQL or Security Labs knowledge base docs to delete'); + }, + }); + if (securityLabsDocs?.total) { + this.options.logger.info( + `Removed ${securityLabsDocs?.total} Security Labs knowledge base docs from knowledge base data stream: ${this.indexTemplateAndPattern.alias}.` + ); } + } catch (e) { + this.options.logger.info('No legacy ESQL or Security Labs knowledge base docs to delete'); } try { @@ -336,39 +284,22 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient { this.options.logger.debug(`ELSER model '${elserId}' is already installed`); } - if (!this.isV2KnowledgeBaseEnabled) { - const isDeployed = await this.isModelDeployed(); - if (!isDeployed) { - await this.deployModel(); - await pRetry( - async () => - (await this.isModelDeployed()) - ? Promise.resolve() - : Promise.reject(new Error('Model not deployed')), - { minTimeout: 2000, retries: 10 } - ); - this.options.logger.debug(`ELSER model '${elserId}' successfully deployed!`); - } else { - this.options.logger.debug(`ELSER model '${elserId}' is already deployed`); - } - } else { - const inferenceExists = await this.isInferenceEndpointExists(); - if (!inferenceExists) { - await this.createInferenceEndpoint(); + const inferenceExists = await this.isInferenceEndpointExists(); + if (!inferenceExists) { + await this.createInferenceEndpoint(); - this.options.logger.debug( - `Inference endpoint for ELSER model '${elserId}' successfully deployed!` - ); - } else { - this.options.logger.debug( - `Inference endpoint for ELSER model '${elserId}' is already deployed` - ); - } + this.options.logger.debug( + `Inference endpoint for ELSER model '${elserId}' successfully deployed!` + ); + } else { + this.options.logger.debug( + `Inference endpoint for ELSER model '${elserId}' is already deployed` + ); } this.options.logger.debug(`Checking if Knowledge Base docs have been loaded...`); - if (v2KnowledgeBaseEnabled && !ignoreSecurityLabs) { + if (!ignoreSecurityLabs) { const labsDocsLoaded = await this.isSecurityLabsDocsLoaded(); if (!labsDocsLoaded) { this.options.logger.debug(`Loading Security Labs KB docs...`); @@ -415,39 +346,20 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient { const { errors, docs_created: docsCreated } = await writer.bulk({ documentsToCreate: documents.map((doc) => { // v1 schema has metadata nested in a `metadata` object - if (this.options.v2KnowledgeBaseEnabled) { - return transformToCreateSchema({ - createdAt: changedAt, - spaceId: this.spaceId, - user: authenticatedUser, - entry: { - type: DocumentEntryType.value, - name: 'unknown', - text: doc.pageContent, - kbResource: doc.metadata.kbResource ?? 'unknown', - required: doc.metadata.required ?? false, - source: doc.metadata.source ?? 'unknown', - }, - global, - }); - } else { - return transformToLegacyCreateSchema({ - createdAt: changedAt, - spaceId: this.spaceId, - user: authenticatedUser, - entry: { - type: DocumentEntryType.value, - name: 'unknown', - text: doc.pageContent, - metadata: { - kbResource: doc.metadata.kbResource ?? 'unknown', - required: doc.metadata.required ?? false, - source: doc.metadata.source ?? 'unknown', - }, - }, - global, - }); - } + return transformToCreateSchema({ + createdAt: changedAt, + spaceId: this.spaceId, + user: authenticatedUser, + entry: { + type: DocumentEntryType.value, + name: 'unknown', + text: doc.pageContent, + kbResource: doc.metadata.kbResource ?? 'unknown', + required: doc.metadata.required ?? false, + source: doc.metadata.source ?? 'unknown', + }, + global, + }); }), authenticatedUser, }); @@ -467,18 +379,6 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient { return created?.data ? transformESSearchToKnowledgeBaseEntry(created?.data) : []; }; - /** - * Returns if ES|QL KB docs have been loaded - */ - public isESQLDocsLoaded = async (): Promise => { - const esqlDocs = await this.getKnowledgeBaseDocumentEntries({ - query: ESQL_DOCS_LOADED_QUERY, - // kbResource, // Note: `8.15` installs have kbResource as `unknown`, so don't filter yet - required: true, - }); - return esqlDocs.length > 0; - }; - /** * Returns if user's KB docs exists */ @@ -492,15 +392,12 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient { } const esClient = await this.options.elasticsearchClientPromise; - const modelId = await this.options.getElserId(); try { const vectorSearchQuery = getKBVectorSearchQuery({ kbResource: USER_RESOURCE, required: false, user, - modelId, - v2KnowledgeBaseEnabled: this.options.v2KnowledgeBaseEnabled, }); const result = await esClient.search({ @@ -531,15 +428,12 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient { const expectedDocsCount = await getSecurityLabsDocsCount({ logger: this.options.logger }); const esClient = await this.options.elasticsearchClientPromise; - const modelId = await this.options.getElserId(); try { const vectorSearchQuery = getKBVectorSearchQuery({ kbResource: SECURITY_LABS_RESOURCE, required: false, user, - modelId, - v2KnowledgeBaseEnabled: this.options.v2KnowledgeBaseEnabled, }); const result = await esClient.search({ @@ -585,7 +479,6 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient { } const esClient = await this.options.elasticsearchClientPromise; - const modelId = await this.options.getElserId(); const vectorSearchQuery = getKBVectorSearchQuery({ filter, @@ -593,8 +486,6 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient { query, required, user, - modelId, - v2KnowledgeBaseEnabled: this.options.v2KnowledgeBaseEnabled, }); try { @@ -605,14 +496,11 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient { }); const results = result.hits.hits.map((hit) => { - const metadata = this.options.v2KnowledgeBaseEnabled - ? { - source: hit?._source?.source, - required: hit?._source?.required, - kbResource: hit?._source?.kb_resource, - } - : // @ts-ignore v1 schema has metadata nested in a `metadata` object and kbResource vs kb_resource - hit?._source?.metadata ?? {}; + const metadata = { + source: hit?._source?.source, + required: hit?._source?.required, + kbResource: hit?._source?.kb_resource, + }; return new Document({ pageContent: hit?._source?.text ?? '', metadata, @@ -691,7 +579,7 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient { telemetry, global = false, }: { - knowledgeBaseEntry: KnowledgeBaseEntryCreateProps | LegacyKnowledgeBaseEntryCreateProps; + knowledgeBaseEntry: KnowledgeBaseEntryCreateProps; global?: boolean; telemetry: AnalyticsServiceSetup; }): Promise => { @@ -721,7 +609,6 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient { knowledgeBaseEntry, global, telemetry, - isV2: this.options.v2KnowledgeBaseEnabled, }); }; @@ -732,10 +619,8 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient { * is scoped to system user. */ public getAssistantTools = async ({ - assistantToolParams, esClient, }: { - assistantToolParams: AssistantToolParams; esClient: ElasticsearchClient; }): Promise => { const user = this.options.currentUser; @@ -746,9 +631,7 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient { } try { - const elserId = this.isV2KnowledgeBaseEnabled - ? ASSISTANT_ELSER_INFERENCE_ID - : await this.options.getElserId(); + const elserId = ASSISTANT_ELSER_INFERENCE_ID; const userFilter = getKBUserFilter(user); const results = await this.findDocuments({ // Note: This is a magic number to set some upward bound as to not blow the context with too diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/ingest_pipeline.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/ingest_pipeline.ts index 8f459848af420..74da4d43d1400 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/ingest_pipeline.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/ingest_pipeline.ts @@ -5,31 +5,8 @@ * 2.0. */ -export const knowledgeBaseIngestPipeline = ({ - id, - modelId, - v2KnowledgeBaseEnabled, -}: { - id: string; - modelId: string; - v2KnowledgeBaseEnabled: boolean; -}) => ({ +export const knowledgeBaseIngestPipeline = ({ id }: { id: string }) => ({ id, description: 'Embedding pipeline for Elastic AI Assistant ELSER Knowledge Base', - processors: !v2KnowledgeBaseEnabled - ? [ - { - inference: { - if: 'ctx?.text != null', - model_id: modelId, - input_output: [ - { - input_field: 'text', - output_field: 'vector.tokens', - }, - ], - }, - }, - ] - : [], + processors: [], }); diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/helpers.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/helpers.ts index 93338174364fc..57b7745a89c78 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/helpers.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/helpers.ts @@ -53,8 +53,6 @@ export const pipelineExists = async ({ esClient, id }: PipelineExistsParams): Pr interface CreatePipelineParams { esClient: ElasticsearchClient; id: string; - modelId: string; - v2KnowledgeBaseEnabled: boolean; } /** @@ -63,22 +61,14 @@ interface CreatePipelineParams { * @param params params * @param params.esClient Elasticsearch client with privileges to check for ingest pipelines * @param params.id ID of the ingest pipeline - * @param params.modelId ID of the ELSER model * * @returns Promise indicating whether the pipeline was created */ -export const createPipeline = async ({ - esClient, - id, - modelId, - v2KnowledgeBaseEnabled, -}: CreatePipelineParams): Promise => { +export const createPipeline = async ({ esClient, id }: CreatePipelineParams): Promise => { try { const response = await esClient.ingest.putPipeline( knowledgeBaseIngestPipeline({ id, - modelId, - v2KnowledgeBaseEnabled, }) ); diff --git a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts index 15274f2323259..d7eff095b4be5 100644 --- a/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts +++ b/x-pack/plugins/elastic_assistant/server/ai_assistant_service/index.ts @@ -27,10 +27,7 @@ import { conversationsFieldMap } from '../ai_assistant_data_clients/conversation import { assistantPromptsFieldMap } from '../ai_assistant_data_clients/prompts/field_maps_configuration'; import { assistantAnonymizationFieldsFieldMap } from '../ai_assistant_data_clients/anonymization_fields/field_maps_configuration'; import { AIAssistantDataClient } from '../ai_assistant_data_clients'; -import { - knowledgeBaseFieldMap, - knowledgeBaseFieldMapV2, -} from '../ai_assistant_data_clients/knowledge_base/field_maps_configuration'; +import { knowledgeBaseFieldMap } from '../ai_assistant_data_clients/knowledge_base/field_maps_configuration'; import { AIAssistantKnowledgeBaseDataClient, GetAIAssistantKnowledgeBaseDataClientParams, @@ -85,8 +82,6 @@ export class AIAssistantService { private resourceInitializationHelper: ResourceInstallationHelper; private initPromise: Promise; private isKBSetupInProgress: boolean = false; - // Temporary 'feature flag' to determine if we should initialize the new kb mappings, toggled when accessing kbDataClient - private v2KnowledgeBaseEnabled: boolean = false; private hasInitializedV2KnowledgeBase: boolean = false; constructor(private readonly options: AIAssistantServiceOpts) { @@ -156,7 +151,7 @@ export class AIAssistantService { // Apply `default_pipeline` if pipeline exists for resource ...(resource in this.resourceNames.pipelines && // Remove this param and initialization when the `assistantKnowledgeBaseByDefault` feature flag is removed - !(resource === 'knowledgeBase' && this.v2KnowledgeBaseEnabled) + !(resource === 'knowledgeBase') ? { template: { settings: { @@ -185,16 +180,6 @@ export class AIAssistantService { pluginStop$: this.options.pluginStop$, }); - // If v2 is enabled, re-install data stream resources for new mappings - if (this.v2KnowledgeBaseEnabled) { - this.options.logger.debug(`Using V2 Knowledge Base Mappings`); - this.knowledgeBaseDataStream = this.createDataStream({ - resource: 'knowledgeBase', - kibanaVersion: this.options.kibanaVersion, - fieldMap: knowledgeBaseFieldMapV2, - }); - } - await this.knowledgeBaseDataStream.install({ esClient, logger: this.options.logger, @@ -206,28 +191,18 @@ export class AIAssistantService { esClient, id: this.resourceNames.pipelines.knowledgeBase, }); - // TODO: When FF is removed, ensure pipeline is re-created for those upgrading - if ( - // Install for v1 - (!this.v2KnowledgeBaseEnabled && !pipelineCreated) || - // Upgrade from v1 to v2 - (pipelineCreated && this.v2KnowledgeBaseEnabled) - ) { + // ensure pipeline is re-created for those upgrading + // pipeline is noop now, so if one does not exist we do not need one + if (pipelineCreated) { this.options.logger.debug( `Installing ingest pipeline - ${this.resourceNames.pipelines.knowledgeBase}` ); const response = await createPipeline({ esClient, id: this.resourceNames.pipelines.knowledgeBase, - modelId: await this.getElserId(), - v2KnowledgeBaseEnabled: this.v2KnowledgeBaseEnabled, }); this.options.logger.debug(`Installed ingest pipeline: ${response}`); - } else { - this.options.logger.debug( - `Ingest pipeline already exists - ${this.resourceNames.pipelines.knowledgeBase}` - ); } await this.promptsDataStream.install({ @@ -363,25 +338,16 @@ export class AIAssistantService { opts: CreateAIAssistantClientParams & GetAIAssistantKnowledgeBaseDataClientParams ): Promise { // If modelIdOverride is set, swap getElserId(), and ensure the pipeline is re-created with the correct model - if (opts.modelIdOverride != null) { + if (opts?.modelIdOverride != null) { const modelIdOverride = opts.modelIdOverride; this.getElserId = async () => modelIdOverride; } - // Note: Due to plugin lifecycle and feature flag registration timing, we need to pass in the feature flag here - // Remove this param and initialization when the `assistantKnowledgeBaseByDefault` feature flag is removed - if (opts.v2KnowledgeBaseEnabled) { - this.v2KnowledgeBaseEnabled = true; - } - - // If either v2 KB or a modelIdOverride is provided, we need to reinitialize all persistence resources to make sure + // If a V2 KnowledgeBase has never been initialized or a modelIdOverride is provided, we need to reinitialize all persistence resources to make sure // they're using the correct model/mappings. Technically all existing KB data is stale since it was created // with a different model/mappings, but modelIdOverride is only intended for testing purposes at this time // Added hasInitializedV2KnowledgeBase to prevent the console noise from re-init on each KB request - if ( - !this.hasInitializedV2KnowledgeBase && - (opts.v2KnowledgeBaseEnabled || opts.modelIdOverride != null) - ) { + if (!this.hasInitializedV2KnowledgeBase || opts?.modelIdOverride != null) { await this.initializeResources(); this.hasInitializedV2KnowledgeBase = true; } @@ -404,7 +370,6 @@ export class AIAssistantService { ml: this.options.ml, setIsKBSetupInProgress: this.setIsKBSetupInProgress.bind(this), spaceId: opts.spaceId, - v2KnowledgeBaseEnabled: opts.v2KnowledgeBaseEnabled ?? false, manageGlobalKnowledgeBaseAIAssistant: opts.manageGlobalKnowledgeBaseAIAssistant ?? false, }); } diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/elasticsearch_store.test.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/elasticsearch_store.test.ts deleted file mode 100644 index 4d32d7bc02da9..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/elasticsearch_store.test.ts +++ /dev/null @@ -1,408 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; -import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; -import { - IndicesCreateResponse, - MlGetTrainedModelsStatsResponse, -} from '@elastic/elasticsearch/lib/api/types'; -import { Document } from 'langchain/document'; - -import { - ElasticsearchStore, - FALLBACK_SIMILARITY_SEARCH_SIZE, - TERMS_QUERY_SIZE, -} from './elasticsearch_store'; -import { mockMsearchResponse } from '../../../__mocks__/msearch_response'; -import { mockQueryText } from '../../../__mocks__/query_text'; -import { coreMock } from '@kbn/core/server/mocks'; -import { - KNOWLEDGE_BASE_EXECUTION_ERROR_EVENT, - KNOWLEDGE_BASE_EXECUTION_SUCCESS_EVENT, -} from '../../telemetry/event_based_telemetry'; -import { Metadata } from '@kbn/elastic-assistant-common'; - -jest.mock('uuid', () => ({ - v4: jest.fn(), -})); - -jest.mock('@kbn/core/server', () => ({ - ElasticsearchClient: jest.fn(), - Logger: jest.fn().mockImplementation(() => ({ - debug: jest.fn(), - error: jest.fn(), - info: jest.fn(), - })), -})); - -const mockEsClient = elasticsearchServiceMock.createElasticsearchClient(); -const mockLogger = loggingSystemMock.createLogger(); -const reportEvent = jest.fn(); -const mockTelemetry = { ...coreMock.createSetup().analytics, reportEvent }; -const KB_INDEX = '.elastic-assistant-kb'; - -const getElasticsearchStore = () => { - return new ElasticsearchStore(mockEsClient, KB_INDEX, mockLogger, mockTelemetry); -}; - -describe('ElasticsearchStore', () => { - let esStore: ElasticsearchStore; - - beforeEach(() => { - esStore = getElasticsearchStore(); - jest.clearAllMocks(); - }); - - describe('Index Management', () => { - it('Checks if index exists', async () => { - mockEsClient.indices.exists.mockResolvedValue(true); - - const exists = await esStore.indexExists(); - - expect(exists).toBe(true); - expect(mockEsClient.indices.exists).toHaveBeenCalledWith({ index: KB_INDEX }); - }); - - it('Creates an index', async () => { - mockEsClient.indices.create.mockResolvedValue({ - acknowledged: true, - } as IndicesCreateResponse); - - const created = await esStore.createIndex(); - - expect(created).toBe(true); - expect(mockEsClient.indices.create).toHaveBeenCalledWith({ - index: KB_INDEX, - mappings: { - properties: { - metadata: { - properties: { - kbResource: { type: 'keyword' }, - required: { type: 'boolean' }, - source: { type: 'keyword' }, - }, - }, - vector: { properties: { tokens: { type: 'rank_features' } } }, - }, - }, - settings: { default_pipeline: '.kibana-elastic-ai-assistant-kb-ingest-pipeline' }, - }); - }); - - it('Deletes an index', async () => { - mockEsClient.indices.delete.mockResolvedValue({ acknowledged: true }); - - const deleted = await esStore.deleteIndex(); - - expect(deleted).toBe(true); - expect(mockEsClient.indices.delete).toHaveBeenCalledWith({ index: KB_INDEX }); - }); - }); - - describe('Pipeline Management', () => { - it('Checks if pipeline exists', async () => { - mockEsClient.ingest.getPipeline.mockResolvedValue({}); - - const exists = await esStore.pipelineExists(); - - expect(exists).toBe(false); - expect(mockEsClient.ingest.getPipeline).toHaveBeenCalledWith({ - id: '.kibana-elastic-ai-assistant-kb-ingest-pipeline', - }); - }); - - it('Creates an ingest pipeline', async () => { - mockEsClient.ingest.putPipeline.mockResolvedValue({ acknowledged: true }); - - const created = await esStore.createPipeline(); - - expect(created).toBe(true); - expect(mockEsClient.ingest.putPipeline).toHaveBeenCalledWith({ - description: 'Embedding pipeline for Elastic AI Assistant ELSER Knowledge Base', - id: '.kibana-elastic-ai-assistant-kb-ingest-pipeline', - processors: [ - { - inference: { - field_map: { text: 'text_field' }, - inference_config: { text_expansion: { results_field: 'tokens' } }, - model_id: '.elser_model_2', - target_field: 'vector', - }, - }, - ], - }); - }); - - it('Deletes an ingest pipeline', async () => { - mockEsClient.ingest.deletePipeline.mockResolvedValue({ acknowledged: true }); - - const deleted = await esStore.deletePipeline(); - - expect(deleted).toBe(true); - expect(mockEsClient.ingest.deletePipeline).toHaveBeenCalledWith({ - id: '.kibana-elastic-ai-assistant-kb-ingest-pipeline', - }); - }); - }); - - describe('isModelInstalled', () => { - it('returns true if model is started and fully allocated', async () => { - mockEsClient.ml.getTrainedModelsStats.mockResolvedValue({ - trained_model_stats: [ - { - deployment_stats: { - state: 'started', - allocation_status: { - state: 'fully_allocated', - }, - }, - }, - ], - } as MlGetTrainedModelsStatsResponse); - - const isInstalled = await esStore.isModelInstalled('.elser_model_2'); - - expect(isInstalled).toBe(true); - expect(mockEsClient.ml.getTrainedModelsStats).toHaveBeenCalledWith({ - model_id: '.elser_model_2', - }); - }); - - it('returns false if model is not started', async () => { - mockEsClient.ml.getTrainedModelsStats.mockResolvedValue({ - trained_model_stats: [ - { - deployment_stats: { - state: 'starting', - allocation_status: { - state: 'fully_allocated', - }, - }, - }, - ], - } as MlGetTrainedModelsStatsResponse); - - const isInstalled = await esStore.isModelInstalled('.elser_model_2'); - - expect(isInstalled).toBe(false); - expect(mockEsClient.ml.getTrainedModelsStats).toHaveBeenCalledWith({ - model_id: '.elser_model_2', - }); - }); - - it('returns false if model is not fully allocated', async () => { - mockEsClient.ml.getTrainedModelsStats.mockResolvedValue({ - trained_model_stats: [ - { - deployment_stats: { - state: 'started', - allocation_status: { - state: 'starting', - }, - }, - }, - ], - } as MlGetTrainedModelsStatsResponse); - - const isInstalled = await esStore.isModelInstalled('.elser_model_2'); - - expect(isInstalled).toBe(false); - expect(mockEsClient.ml.getTrainedModelsStats).toHaveBeenCalledWith({ - model_id: '.elser_model_2', - }); - }); - }); - - describe('addDocuments', () => { - it('Checks if documents are added', async () => { - mockEsClient.bulk.mockResolvedValue({ - errors: false, - took: 515, - ingest_took: 4026, - items: [ - { - index: { - _index: '.kibana-elastic-ai-assistant-kb', - _id: 'be2584a9-ad2e-4f13-a11c-c0b79423079c', - _version: 2, - result: 'updated', - forced_refresh: true, - _shards: { - total: 2, - successful: 1, - failed: 0, - }, - _seq_no: 1, - _primary_term: 1, - status: 200, - }, - }, - ], - }); - - const document = new Document({ - pageContent: 'interesting stuff', - metadata: { kbResource: 'esql', required: false, source: '1' }, - }); - - const docsInstalled = await esStore.addDocuments([document]); - - expect(docsInstalled).toStrictEqual(['be2584a9-ad2e-4f13-a11c-c0b79423079c']); - expect(mockEsClient.bulk).toHaveBeenCalledWith({ - operations: [ - { - index: { - _id: undefined, - _index: '.elastic-assistant-kb', - }, - }, - { - metadata: { - kbResource: 'esql', - required: false, - source: '1', - }, - text: 'interesting stuff', - }, - ], - refresh: true, - }); - }); - }); - - describe('similaritySearch', () => { - it('Checks if documents are found', async () => { - mockEsClient.msearch.mockResolvedValue(mockMsearchResponse); - - const searchResults = await esStore.similaritySearch(mockQueryText); - - expect(searchResults).toStrictEqual([ - { - pageContent: - "[[esql-from]]\n=== `FROM`\n\nThe `FROM` source command returns a table with up to 10,000 documents from a\ndata stream, index, or alias. Each row in the resulting table represents a\ndocument. Each column corresponds to a field, and can be accessed by the name\nof that field.\n\n[source,esql]\n----\nFROM employees\n----\n\nYou can use <> to refer to indices, aliases\nand data streams. This can be useful for time series data, for example to access\ntoday's index:\n\n[source,esql]\n----\nFROM \n----\n\nUse comma-separated lists or wildcards to query multiple data streams, indices,\nor aliases:\n\n[source,esql]\n----\nFROM employees-00001,employees-*\n----\n", - metadata: { - source: - '/Users/andrew.goldstein/Projects/forks/andrew-goldstein/kibana/x-pack/plugins/elastic_assistant/server/knowledge_base/esql/documentation/source_commands/from.asciidoc', - }, - }, - { - pageContent: - '[[esql-example-queries]]\n\nThe following is an example an ES|QL query:\n\n```\nFROM logs-*\n| WHERE NOT CIDR_MATCH(destination.ip, "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16")\n| STATS destcount = COUNT(destination.ip) by user.name, host.name\n| ENRICH ldap_lookup_new ON user.name\n| WHERE group.name IS NOT NULL\n| EVAL follow_up = CASE(\n destcount >= 100, "true",\n "false")\n| SORT destcount desc\n| KEEP destcount, host.name, user.name, group.name, follow_up\n```\n', - metadata: { - source: - '/Users/andrew.goldstein/Projects/forks/andrew-goldstein/kibana/x-pack/plugins/elastic_assistant/server/knowledge_base/esql/example_queries/esql_example_query_0001.asciidoc', - }, - }, - ]); - - expect(mockEsClient.msearch).toHaveBeenCalledWith({ - body: [ - { - index: '.elastic-assistant-kb', - }, - { - query: { - bool: { - must_not: [{ term: { 'metadata.required': true } }], - must: [ - { - text_expansion: { - 'vector.tokens': { - model_id: '.elser_model_2', - model_text: mockQueryText, - }, - }, - }, - ], - }, - }, - size: FALLBACK_SIMILARITY_SEARCH_SIZE, // <-- `FALLBACK_SIMILARITY_SEARCH_SIZE` is used when `k` is not provided - }, - { - index: '.elastic-assistant-kb', - }, - { - query: { - bool: { - must: [{ term: { 'metadata.required': true } }], - }, - }, - size: TERMS_QUERY_SIZE, - }, - ], - }); - }); - - it('uses the value of `k` instead of the `FALLBACK_SIMILARITY_SEARCH_SIZE` when `k` is provided', async () => { - mockEsClient.msearch.mockResolvedValue(mockMsearchResponse); - - const k = 4; - await esStore.similaritySearch(mockQueryText, k); - - expect(mockEsClient.msearch).toHaveBeenCalledWith({ - body: [ - { - index: '.elastic-assistant-kb', - }, - { - query: { - bool: { - must_not: [{ term: { 'metadata.required': true } }], - must: [ - { - text_expansion: { - 'vector.tokens': { - model_id: '.elser_model_2', - model_text: mockQueryText, - }, - }, - }, - ], - }, - }, - size: k, // <-- `k` is used instead of `FALLBACK_SIMILARITY_SEARCH_SIZE` - }, - { - index: '.elastic-assistant-kb', - }, - { - query: { - bool: { - must: [{ term: { 'metadata.required': true } }], - }, - }, - size: TERMS_QUERY_SIZE, - }, - ], - }); - }); - - it('Reports successful telemetry event', async () => { - mockEsClient.msearch.mockResolvedValue(mockMsearchResponse); - - await esStore.similaritySearch(mockQueryText); - - expect(reportEvent).toHaveBeenCalledWith(KNOWLEDGE_BASE_EXECUTION_SUCCESS_EVENT.eventType, { - model: '.elser_model_2', - responseTime: 142, - resultCount: 2, - }); - }); - - it('Reports error telemetry event', async () => { - mockEsClient.msearch.mockRejectedValue(new Error('Oh no!')); - - await esStore.similaritySearch(mockQueryText); - - expect(reportEvent).toHaveBeenCalledWith(KNOWLEDGE_BASE_EXECUTION_ERROR_EVENT.eventType, { - model: '.elser_model_2', - errorMessage: 'Oh no!', - }); - }); - }); -}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/elasticsearch_store.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/elasticsearch_store.ts deleted file mode 100644 index 78c1b104685ad..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/elasticsearch_store.ts +++ /dev/null @@ -1,478 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { type AnalyticsServiceSetup, ElasticsearchClient, Logger } from '@kbn/core/server'; -import { - MappingTypeMapping, - MlTrainedModelDeploymentNodesStats, - MlTrainedModelStats, -} from '@elastic/elasticsearch/lib/api/types'; -import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { Callbacks } from '@langchain/core/callbacks/manager'; -import { Document } from 'langchain/document'; -import { VectorStore } from '@langchain/core/vectorstores'; -import * as uuid from 'uuid'; - -import { Metadata } from '@kbn/elastic-assistant-common'; -import { transformError } from '@kbn/securitysolution-es-utils'; -import { ElasticsearchEmbeddings } from '../embeddings/elasticsearch_embeddings'; -import { FlattenedHit, getFlattenedHits } from './helpers/get_flattened_hits'; -import { getMsearchQueryBody } from './helpers/get_msearch_query_body'; -import { getTermsSearchQuery } from './helpers/get_terms_search_query'; -import { getVectorSearchQuery } from './helpers/get_vector_search_query'; -import type { MsearchResponse } from './helpers/types'; -import { - KNOWLEDGE_BASE_INDEX_PATTERN, - KNOWLEDGE_BASE_INGEST_PIPELINE, -} from '../../../routes/knowledge_base/constants'; -import { getRequiredKbDocsTermsQueryDsl } from './helpers/get_required_kb_docs_terms_query_dsl'; -import { - KNOWLEDGE_BASE_EXECUTION_ERROR_EVENT, - KNOWLEDGE_BASE_EXECUTION_SUCCESS_EVENT, -} from '../../telemetry/event_based_telemetry'; -import { AIAssistantKnowledgeBaseDataClient } from '../../../ai_assistant_data_clients/knowledge_base'; - -interface CreatePipelineParams { - id?: string; - description?: string; -} - -interface CreateIndexParams { - index?: string; - pipeline?: string; -} - -/** - * A fallback for the query `size` that determines how many documents to - * return from Elasticsearch when performing a similarity search. - * - * The size is typically determined by the implementation of LangChain's - * `VectorStoreRetriever._getRelevantDocuments` function, so this fallback is - * only required when using the `ElasticsearchStore` directly. - */ -export const FALLBACK_SIMILARITY_SEARCH_SIZE = 10; - -/** The maximum number of hits to return from a `terms` query, via the `size` parameter */ -export const TERMS_QUERY_SIZE = 10000; - -/** - * Basic ElasticsearchStore implementation only leveraging ELSER for storage and retrieval. - */ -export class ElasticsearchStore extends VectorStore { - declare FilterType: QueryDslQueryContainer; - - private readonly esClient: ElasticsearchClient; - private readonly kbDataClient: AIAssistantKnowledgeBaseDataClient | undefined; - private readonly index: string; - private readonly logger: Logger; - private readonly telemetry: AnalyticsServiceSetup; - private readonly model: string; - private kbResource?: string; - - _vectorstoreType(): string { - return 'elasticsearch'; - } - - constructor( - esClient: ElasticsearchClient, - index: string, - logger: Logger, - telemetry: AnalyticsServiceSetup, - model?: string, - kbResource?: string | undefined, - kbDataClient?: AIAssistantKnowledgeBaseDataClient - ) { - super(new ElasticsearchEmbeddings(logger), { esClient, index }); - this.esClient = esClient; - this.index = index ?? KNOWLEDGE_BASE_INDEX_PATTERN; - this.logger = logger; - this.telemetry = telemetry; - this.model = model ?? '.elser_model_2'; - this.kbResource = kbResource; - this.kbDataClient = kbDataClient; - } - - setKbResource(kbResource: string) { - this.kbResource = kbResource; - } - - /** - * Add documents to the store. Embeddings are created on ingest into index configured with - * ELSER ingest pipeline. Returns a list of document IDs. - * - * @param documents Documents to add to the store - * @param options Any additional options as defined in the interface - * @returns Promise of document IDs added to the store - */ - addDocuments = async ( - documents: Array>, - options?: Record - ): Promise => { - // Code path for when `assistantKnowledgeBaseByDefault` FF is enabled - // Once removed replace addDocuments() w/ addDocumentsViaDataClient() - if (this.kbDataClient != null) { - return this.addDocumentsViaDataClient(documents, options); - } - - const pipelineExists = await this.pipelineExists(); - if (!pipelineExists) { - await this.createPipeline(); - } - - const operations = documents.flatMap(({ pageContent, metadata }) => [ - { index: { _index: this.index, _id: uuid.v4() } }, - { text: pageContent, metadata }, - ]); - - try { - const response = await this.esClient.bulk({ refresh: true, operations }); - this.logger.debug(() => `Add Documents Response:\n ${JSON.stringify(response)}`); - - const errorIds = response.items.filter((i) => i.index?.error != null); - operations.forEach((op, i) => { - if (errorIds.some((e) => e.index?._id === op.index?._id)) { - this.logger.error(`Error adding document to KB: ${JSON.stringify(operations?.[i + 1])}`); - } - }); - - return response.items.flatMap((i) => - i.index?._id != null && i.index.error == null ? [i.index._id] : [] - ); - } catch (e) { - this.logger.error(`Error loading data into KB\n ${e}`); - return []; - } - }; - - addDocumentsViaDataClient = async ( - documents: Array>, - options?: Record - ): Promise => { - if (!this.kbDataClient) { - this.logger.error('No kbDataClient provided'); - return []; - } - - try { - const response = await this.kbDataClient.addKnowledgeBaseDocuments({ - documents, - global: true, - }); - return response.map((doc) => doc.id); - } catch (e) { - this.logger.error(`Error loading data into KB\n ${e}`); - return []; - } - }; - - /** - * Add vectors to the store. Returns a list of document IDs. - * - * @param vectors Vector representation of documents to add to the store - * @param documents Documents corresponding to the provided vectors - * @param options Any additional options as defined in the interface - * @returns Promise of document IDs added to the store - */ - addVectors = ( - vectors: number[][], - documents: Document[], - options?: {} - ): Promise => { - // Note: implement if/when needed - this.logger.info('ElasticsearchStore.addVectors not implemented'); - return Promise.resolve(undefined); - }; - - /** - * Performs similarity search on the store using the provided query vector and filter, returning k similar - * documents along with their score. - * - * @param query Query vector to search with - * @param k Number of similar documents to return - * @param filter Optional filter to apply to the search - * - * @returns Promise> of similar documents and their scores - */ - similaritySearchVectorWithScore = ( - query: number[], - k: number, - filter?: this['FilterType'] - ): Promise> => { - // Note: Implement if needed - this.logger.info('ElasticsearchStore.similaritySearchVectorWithScore not implemented'); - return Promise.resolve([]); - }; - - // Non-abstract function overrides - - /** - * Performs similarity search on the store using the provided query string and filter, returning k similar - * @param query Query vector to search with - * @param k Number of similar documents to return - * @param filter Optional filter to apply to the search - * @param _callbacks Optional callbacks - * @param filterRequiredDocs Optional whether or not to exclude the required docs filter - * - * Fun facts: - * - This function is called by LangChain's `VectorStoreRetriever._getRelevantDocuments` - * - The `k` parameter is typically determined by LangChain's `VectorStoreRetriever._getRelevantDocuments`, and has been observed to default to `4` in the wild (see langchain/dist/vectorstores/base.ts) - * @returns Promise of similar documents - */ - similaritySearch = async ( - query: string, - k?: number, - filter?: this['FilterType'] | undefined, - _callbacks?: Callbacks | undefined, - filterRequiredDocs = true - ): Promise => { - // requiredDocs is an array of filters that can be used in a `bool` Elasticsearch DSL query to filter in/out required KB documents: - const requiredDocs = filterRequiredDocs ? getRequiredKbDocsTermsQueryDsl(this.kbResource) : []; - - // The `k` parameter is typically provided by LangChain's `VectorStoreRetriever._getRelevantDocuments`, which calls this function: - const vectorSearchQuerySize = k ?? FALLBACK_SIMILARITY_SEARCH_SIZE; - - // build a vector search query: - const vectorSearchQuery = getVectorSearchQuery({ - filter, - modelId: this.model, - mustNotTerms: requiredDocs, - query, - }); - - // build a (separate) terms search query: - const termsSearchQuery = getTermsSearchQuery(requiredDocs); - - // combine the vector search query and the terms search queries into a single multi-search query: - const mSearchQueryBody = getMsearchQueryBody({ - index: this.index, - termsSearchQuery, - termsSearchQuerySize: TERMS_QUERY_SIZE, - vectorSearchQuery, - vectorSearchQuerySize, - }); - - try { - // execute both queries via a single multi-search request: - const result = await this.esClient.msearch(mSearchQueryBody); - - // flatten the results of the combined queries into a single array of hits: - const results: FlattenedHit[] = result.responses.flatMap((response) => { - const maybeEsqlMsearchResponse: MsearchResponse = response as MsearchResponse; - - return getFlattenedHits(maybeEsqlMsearchResponse); - }); - - this.telemetry.reportEvent(KNOWLEDGE_BASE_EXECUTION_SUCCESS_EVENT.eventType, { - model: this.model, - ...(this.kbResource != null ? { resourceAccessed: this.kbResource } : {}), - resultCount: results.length, - responseTime: result.took ?? 0, - }); - - this.logger.debug( - () => - `Similarity search metadata source:\n${JSON.stringify( - results.map((r) => r?.metadata?.source ?? '(missing metadata.source)'), - null, - 2 - )}` - ); - - return results; - } catch (e) { - const error = transformError(e); - this.telemetry.reportEvent(KNOWLEDGE_BASE_EXECUTION_ERROR_EVENT.eventType, { - model: this.model, - ...(this.kbResource != null ? { resourceAccessed: this.kbResource } : {}), - errorMessage: error.message, - }); - this.logger.error(e); - return []; - } - }; - - // ElasticsearchStore explicit utility functions - - /** - * Checks if the provided index exists in Elasticsearch - * - * @returns Promise indicating whether the index exists - * @param index Index to check - * @returns Promise indicating whether the index exists - */ - indexExists = async (index?: string): Promise => { - return this.esClient.indices.exists({ index: index ?? this.index }); - }; - - /** - * Create index for ELSER embeddings in Elasticsearch - * - * @returns Promise indicating whether the index was created - */ - createIndex = async ({ index, pipeline }: CreateIndexParams = {}): Promise => { - const mappings: MappingTypeMapping = { - properties: { - metadata: { - properties: { - /** the category of knowledge, e.g. `esql` */ - kbResource: { type: 'keyword' }, - /** when `true`, return this document in all searches for the `kbResource` */ - required: { type: 'boolean' }, - /** often a file path when the document was created via a LangChain `DirectoryLoader`, this metadata describes the origin of the document */ - source: { type: 'keyword' }, - }, - }, - vector: { - properties: { tokens: { type: 'rank_features' } }, - }, - }, - }; - - const settings = { default_pipeline: pipeline ?? KNOWLEDGE_BASE_INGEST_PIPELINE }; - - const response = await this.esClient.indices.create({ - index: index ?? this.index, - mappings, - settings, - }); - - return response.acknowledged; - }; - - /** - * Delete index for ELSER embeddings in Elasticsearch - * @param index Index to delete, otherwise uses the default index - * - * @returns Promise indicating whether the index was created - */ - deleteIndex = async (index?: string): Promise => { - // Code path for when `assistantKnowledgeBaseByDefault` FF is enabled - // We won't be supporting delete operations for the KB data stream going forward, so this can be removed along with the FF - if (this.kbDataClient != null) { - const response = await this.esClient.indices.deleteDataStream({ name: index ?? this.index }); - return response.acknowledged; - } - - const response = await this.esClient.indices.delete({ - index: index ?? this.index, - }); - - return response.acknowledged; - }; - - /** - * Checks if the provided ingest pipeline exists in Elasticsearch - * - * @param pipelineId ID of the ingest pipeline to check - * @returns Promise indicating whether the pipeline exists - */ - pipelineExists = async (pipelineId?: string): Promise => { - try { - const id = - pipelineId ?? - this.kbDataClient?.options.ingestPipelineResourceName ?? - KNOWLEDGE_BASE_INGEST_PIPELINE; - const response = await this.esClient.ingest.getPipeline({ - id, - }); - return Object.keys(response).length > 0; - } catch (e) { - // The GET /_ingest/pipeline/{pipelineId} API returns an empty object w/ 404 Not Found. - return false; - } - }; - - /** - * Create ingest pipeline for ELSER in Elasticsearch - * - * @returns Promise indicating whether the pipeline was created - */ - createPipeline = async ({ id, description }: CreatePipelineParams = {}): Promise => { - const response = await this.esClient.ingest.putPipeline({ - id: - id ?? - this.kbDataClient?.options.ingestPipelineResourceName ?? - KNOWLEDGE_BASE_INGEST_PIPELINE, - description: - description ?? 'Embedding pipeline for Elastic AI Assistant ELSER Knowledge Base', - processors: [ - { - inference: { - model_id: this.model, - target_field: 'vector', - field_map: { - text: 'text_field', - }, - inference_config: { - // @ts-expect-error - text_expansion: { - results_field: 'tokens', - }, - }, - }, - }, - ], - }); - - return response.acknowledged; - }; - - /** - * Delete ingest pipeline for ELSER in Elasticsearch - * - * @returns Promise indicating whether the pipeline was created - */ - deletePipeline = async (pipelineId?: string): Promise => { - const response = await this.esClient.ingest.deletePipeline({ - id: - pipelineId ?? - this.kbDataClient?.options.ingestPipelineResourceName ?? - KNOWLEDGE_BASE_INGEST_PIPELINE, - }); - - return response.acknowledged; - }; - - /** - * Checks if the provided model is installed in Elasticsearch - * - * @param modelId ID of the model to check - * @returns Promise indicating whether the model is installed - */ - async isModelInstalled(modelId?: string): Promise { - try { - // Code path for when `assistantKnowledgeBaseByDefault` FF is enabled - if (this.kbDataClient != null) { - // esStore.isModelInstalled() is actually checking if the model is deployed, not installed, so do that instead - return this.kbDataClient.isModelDeployed(); - } - - const getResponse = await this.esClient.ml.getTrainedModelsStats({ - model_id: modelId ?? this.model, - }); - - this.logger.debug(`modelId: ${modelId}`); - - // For standardized way of checking deployment status see: https://github.com/elastic/elasticsearch/issues/106986 - const isReadyESS = (stats: MlTrainedModelStats) => - stats.deployment_stats?.state === 'started' && - stats.deployment_stats?.allocation_status.state === 'fully_allocated'; - - const isReadyServerless = (stats: MlTrainedModelStats) => - (stats.deployment_stats?.nodes as unknown as MlTrainedModelDeploymentNodesStats[]).some( - (node) => node.routing_state.routing_state === 'started' - ); - - return getResponse.trained_model_stats.some( - (stats) => isReadyESS(stats) || isReadyServerless(stats) - ); - } catch (e) { - // Returns 404 if it doesn't exist - return false; - } - } -} diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_flattened_hits.test.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_flattened_hits.test.ts deleted file mode 100644 index cc08a95cdb532..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_flattened_hits.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { getFlattenedHits } from './get_flattened_hits'; -import { mockMsearchResponse } from '../../../../__mocks__/msearch_response'; -import type { MsearchResponse } from './types'; - -describe('getFlattenedHits', () => { - it('returns an empty array when the response is undefined', () => { - const result = getFlattenedHits(undefined); - - expect(result).toEqual([]); - }); - - it('returns an empty array when hits > hits is empty', () => { - const result = getFlattenedHits({ hits: { hits: [] } }); - - expect(result).toEqual([]); - }); - - it('returns the expected flattened hits given a non-empty `MsearchResponse`', () => { - const expected = [ - { - pageContent: - "[[esql-from]]\n=== `FROM`\n\nThe `FROM` source command returns a table with up to 10,000 documents from a\ndata stream, index, or alias. Each row in the resulting table represents a\ndocument. Each column corresponds to a field, and can be accessed by the name\nof that field.\n\n[source,esql]\n----\nFROM employees\n----\n\nYou can use <> to refer to indices, aliases\nand data streams. This can be useful for time series data, for example to access\ntoday's index:\n\n[source,esql]\n----\nFROM \n----\n\nUse comma-separated lists or wildcards to query multiple data streams, indices,\nor aliases:\n\n[source,esql]\n----\nFROM employees-00001,employees-*\n----\n", - metadata: { - source: - '/Users/andrew.goldstein/Projects/forks/andrew-goldstein/kibana/x-pack/plugins/elastic_assistant/server/knowledge_base/esql/documentation/source_commands/from.asciidoc', - }, - }, - ]; - - const result = getFlattenedHits(mockMsearchResponse.responses[0] as MsearchResponse); - - expect(result).toEqual(expected); - }); - - it('returns an array of FlattenedHits with empty strings when given an MsearchResponse with missing fields', () => { - const msearchResponse = { - hits: { - hits: [ - { - _source: { - metadata: { - source: '/source/1', - }, - }, - }, - { - _source: { - text: 'Source 2 text', - }, - }, - ], - }, - }; - - const expected = [ - { - pageContent: '', // <-- missing text field - metadata: { - source: '/source/1', - }, - }, - { - pageContent: 'Source 2 text', - metadata: { - source: '', // <-- missing source field - }, - }, - ]; - - const result = getFlattenedHits(msearchResponse); - - expect(result).toEqual(expected); - }); -}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_flattened_hits.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_flattened_hits.ts deleted file mode 100644 index f6c3a3ef0e9fa..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_flattened_hits.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { MsearchKbHit, MsearchResponse } from './types'; - -/** - * Represents a flattened hit from an Elasticsearch Msearch response - * - * It contains the page content and metadata source of a KB document - */ -export interface FlattenedHit { - pageContent: string; - metadata: { - source: string; - }; -} - -/** - * Returns an array of flattened hits from the specified Msearch response - * that contain the page content and metadata source of KB documents - * - * @param maybeMsearchResponse An Elasticsearch Msearch response, which returns the results of multiple searches in a single request - * @returns Returns an array of flattened hits from the specified Msearch response that contain the page content and metadata source of KB documents - */ -export const getFlattenedHits = ( - maybeMsearchResponse: MsearchResponse | undefined -): FlattenedHit[] => - maybeMsearchResponse?.hits?.hits?.flatMap((hit: MsearchKbHit) => ({ - pageContent: hit?._source?.text ?? '', - metadata: { - source: hit?._source?.metadata?.source ?? '', - }, - })) ?? []; diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_msearch_query_body.test.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_msearch_query_body.test.ts deleted file mode 100644 index 2697aaf76a085..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_msearch_query_body.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { TERMS_QUERY_SIZE } from '../elasticsearch_store'; -import { getMsearchQueryBody } from './get_msearch_query_body'; -import { mockTermsSearchQuery } from '../../../../__mocks__/terms_search_query'; -import { mockVectorSearchQuery } from '../../../../__mocks__/vector_search_query'; - -describe('getMsearchQueryBody', () => { - it('returns the expected multi-search request body', () => { - const index = '.kibana-elastic-ai-assistant-kb'; - - const vectorSearchQuery = mockVectorSearchQuery; - const vectorSearchQuerySize = 4; - - const termsSearchQuery = mockTermsSearchQuery; - const termsSearchQuerySize = TERMS_QUERY_SIZE; - - const result = getMsearchQueryBody({ - index, - termsSearchQuery, - termsSearchQuerySize, - vectorSearchQuery, - vectorSearchQuerySize, - }); - - expect(result).toEqual({ - body: [ - { index }, - { - query: mockVectorSearchQuery, - size: vectorSearchQuerySize, - }, - { index }, - { - query: mockTermsSearchQuery, - size: TERMS_QUERY_SIZE, - }, - ], - }); - }); -}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_msearch_query_body.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_msearch_query_body.ts deleted file mode 100644 index c93c3f2e30954..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_msearch_query_body.ts +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; - -/** - * Represents an entry in a multi-search request body that specifies the name of an index to search - */ -export interface MsearchQueryBodyIndexEntry { - index: string; -} - -/** - * Represents an entry in a multi-search request body that specifies a query to execute - */ -export interface MsearchQueryBodyQueryEntry { - query: QueryDslQueryContainer; - size: number; -} - -/** - * Represents a multi-search request body, which returns the results of multiple searches in a single request - */ -export interface MsearchQueryBody { - body: Array; -} - -/** - * Returns a multi-search request body, which returns the results of multiple searches in a single request - * - * @param index The KB index to search, e.g. `.kibana-elastic-ai-assistant-kb` - * @param termsSearchQuery An Elasticsearch DSL query that performs a terms search, typically used to search for required KB documents - * @param termsSearchQuerySize The maximum number of required KB documents to return - * @param vectorSearchQuery An Elasticsearch DSL query that performs a vector search, typically used to search for similar KB documents - * @param vectorSearchQuerySize The maximum number of similar KB documents to return - * @returns A multi-search request body, which returns the results of multiple searches in a single request - */ -export const getMsearchQueryBody = ({ - index, - termsSearchQuery, - termsSearchQuerySize, - vectorSearchQuery, - vectorSearchQuerySize, -}: { - index: string; - termsSearchQuery: QueryDslQueryContainer; - termsSearchQuerySize: number; - vectorSearchQuery: QueryDslQueryContainer; - vectorSearchQuerySize: number; -}): MsearchQueryBody => ({ - body: [ - { index }, - { - query: vectorSearchQuery, - size: vectorSearchQuerySize, - }, - { index }, - { - query: termsSearchQuery, - size: termsSearchQuerySize, - }, - ], -}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_required_kb_docs_terms_query_dsl.test.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_required_kb_docs_terms_query_dsl.test.ts deleted file mode 100644 index 5c4f944e83178..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_required_kb_docs_terms_query_dsl.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { getRequiredKbDocsTermsQueryDsl } from './get_required_kb_docs_terms_query_dsl'; - -const kbResource = 'esql'; - -describe('getRequiredKbDocsTermsQueryDsl', () => { - it('returns the expected terms query DSL', () => { - const result = getRequiredKbDocsTermsQueryDsl(kbResource); - - expect(result).toEqual([ - { term: { 'metadata.kbResource': 'esql' } }, - { term: { 'metadata.required': true } }, - ]); - }); -}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_required_kb_docs_terms_query_dsl.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_required_kb_docs_terms_query_dsl.ts deleted file mode 100644 index df3e8f42ad63b..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_required_kb_docs_terms_query_dsl.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { Field, FieldValue, QueryDslTermQuery } from '@elastic/elasticsearch/lib/api/types'; - -/** - * For the specified topic, returns an array of filters that can be used in a - * `bool` Elasticsearch DSL query to filter in/out required KB documents. - * - * The returned filters can be used in different types of queries to, for example: - * - To filter out required KB documents from a vector search - * - To filter in required KB documents in a terms query - * - * @param kbResource Search for required KB documents for this topic - * - * @returns An array of `term`s that may be used in a `bool` Elasticsearch DSL query to filter in/out required KB documents - */ -export const getRequiredKbDocsTermsQueryDsl = ( - kbResource?: string -): Array>> => [ - ...(kbResource != null - ? [ - { - term: { - 'metadata.kbResource': kbResource, - }, - }, - ] - : []), - { - term: { - 'metadata.required': true, - }, - }, -]; diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_terms_search_query.test.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_terms_search_query.test.ts deleted file mode 100644 index 98d3b2c5d36c2..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_terms_search_query.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { getTermsSearchQuery } from './get_terms_search_query'; -import { mockTerms } from '../../../../__mocks__/terms'; - -describe('getTermsSearchQuery', () => { - it('returns the expected Elasticsearch query DSL', () => { - const query = getTermsSearchQuery(mockTerms); - - expect(query).toEqual({ - bool: { - must: mockTerms, - }, - }); - }); -}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_terms_search_query.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_terms_search_query.ts deleted file mode 100644 index 8fcc7b3b20851..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_terms_search_query.ts +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { - Field, - FieldValue, - QueryDslTermQuery, - QueryDslQueryContainer, -} from '@elastic/elasticsearch/lib/api/types'; - -/** - * Returns an Elasticsearch DSL query that performs a terms search, - * such that all of the specified terms must be present in the search results. - * - * @param mustTerms All of the specified terms must be present in the search results - * - * @returns An Elasticsearch DSL query that performs a terms search, such that all of the specified terms must be present in the search results - */ -export const getTermsSearchQuery = ( - mustTerms: Array>> -): QueryDslQueryContainer => ({ - bool: { - must: [...mustTerms], // all of the specified terms must be present in the search results - }, -}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_vector_search_query.test.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_vector_search_query.test.ts deleted file mode 100644 index da6a7227953f2..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_vector_search_query.test.ts +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; - -import { getVectorSearchQuery } from './get_vector_search_query'; -import { mockTerms } from '../../../../__mocks__/terms'; -import { mockQueryText } from '../../../../__mocks__/query_text'; - -describe('getVectorSearchQuery', () => { - it('returns the expected query when mustNotTerms is empty', () => { - const result = getVectorSearchQuery({ - filter: undefined, - modelId: '.elser_model_2', - mustNotTerms: [], // <--- empty - query: mockQueryText, - }); - - expect(result).toEqual({ - bool: { - filter: undefined, - must: [ - { - text_expansion: { - 'vector.tokens': { - model_id: '.elser_model_2', - model_text: - 'Generate an ES|QL query that will count the number of connections made to external IP addresses, broken down by user. If the count is greater than 100 for a specific user, add a new field called follow_up that contains a value of true, otherwise, it should contain false. The user names should also be enriched with their respective group names.', - }, - }, - }, - ], - must_not: [], - }, - }); - }); - - it('returns the expected query when mustNotTerms are provided', () => { - const result = getVectorSearchQuery({ - filter: undefined, - modelId: '.elser_model_2', - mustNotTerms: mockTerms, // <--- mock terms - query: mockQueryText, - }); - - expect(result).toEqual({ - bool: { - filter: undefined, - must: [ - { - text_expansion: { - 'vector.tokens': { - model_id: '.elser_model_2', - model_text: - 'Generate an ES|QL query that will count the number of connections made to external IP addresses, broken down by user. If the count is greater than 100 for a specific user, add a new field called follow_up that contains a value of true, otherwise, it should contain false. The user names should also be enriched with their respective group names.', - }, - }, - }, - ], - must_not: [ - { - term: { - 'metadata.kbResource': 'esql', - }, - }, - { - term: { - 'metadata.required': true, - }, - }, - ], - }, - }); - }); - - it('returns the expected results when a filter is provided', () => { - const filter: QueryDslQueryContainer = { - bool: { - must: [ - { - term: { - 'some.field': 'value', - }, - }, - ], - }, - }; - - const result = getVectorSearchQuery({ - filter, - modelId: '.elser_model_2', - mustNotTerms: mockTerms, // <--- mock terms - query: mockQueryText, - }); - - expect(result).toEqual({ - bool: { - filter, - must: [ - { - text_expansion: { - 'vector.tokens': { - model_id: '.elser_model_2', - model_text: - 'Generate an ES|QL query that will count the number of connections made to external IP addresses, broken down by user. If the count is greater than 100 for a specific user, add a new field called follow_up that contains a value of true, otherwise, it should contain false. The user names should also be enriched with their respective group names.', - }, - }, - }, - ], - must_not: [ - { - term: { - 'metadata.kbResource': 'esql', - }, - }, - { - term: { - 'metadata.required': true, - }, - }, - ], - }, - }); - }); -}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_vector_search_query.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_vector_search_query.ts deleted file mode 100644 index 613ee5c501560..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/get_vector_search_query.ts +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { - Field, - FieldValue, - QueryDslQueryContainer, - QueryDslTermQuery, -} from '@elastic/elasticsearch/lib/api/types'; - -/** - * Returns an Elasticsearch query DSL that performs a vector search - * that excludes a set of documents from the search results. - * - * @param filter Optional filter to apply to the search - * @param modelId ID of the model to search with, e.g. `.elser_model_2` - * @param mustNotTerms Array of objects that may be used in a `bool` Elasticsearch DSL query to, for example, exclude the required KB docs from the vector search, so there's no overlap - * @param query The search query provided by the user - * @returns - */ -export const getVectorSearchQuery = ({ - filter, - modelId, - mustNotTerms, - query, -}: { - filter: QueryDslQueryContainer | undefined; - modelId: string; - mustNotTerms: Array>>; - query: string; -}): QueryDslQueryContainer => ({ - bool: { - must_not: [...mustNotTerms], - must: [ - { - text_expansion: { - 'vector.tokens': { - model_id: modelId, - model_text: query, - }, - }, - }, - ], - filter, - }, -}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/types.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/types.ts deleted file mode 100644 index a0f549a00ab26..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/elasticsearch_store/helpers/types.ts +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -/** - * A hit from the response to an Elasticsearch multi-search request, - * which returns the results of multiple searches in a single request. - * - * Search hits may contain the following properties that may be present in - * knowledge base documents: - * - * 1) the `metadata` property, an object that may have the following properties: - * - `kbResource`: The name of the Knowledge Base resource that the document belongs to, e.g. `esql` - * - `required`: A boolean indicating whether the document is required for searches on the `kbResource` topic - * - `source`: Describes the origin of the document, sometimes a file path via a LangChain DirectoryLoader - * 2) the `text` property, a string containing the text of the document - * 3) the `vector` property, containing the document's embeddings - */ -export interface MsearchKbHit { - _id?: string; - _ignored?: string[]; - _index?: string; - _score?: number; - _source?: { - metadata?: { - kbResource?: string; - required?: boolean; - source?: string; - }; - text?: string; - vector?: { - tokens?: Record; - }; - }; -} - -/** - * A Response from an Elasticsearch multi-search request, which returns the - * results of multiple searches in a single request. - */ -export interface MsearchResponse { - hits?: { - hits?: MsearchKbHit[]; - }; -} diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/embeddings/elasticsearch_embeddings.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/embeddings/elasticsearch_embeddings.ts deleted file mode 100644 index 570f692ecd5ac..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/embeddings/elasticsearch_embeddings.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { Embeddings, EmbeddingsParams } from '@langchain/core/embeddings'; -import { Logger } from '@kbn/core/server'; - -/** - * Shell class for Elasticsearch embeddings as not needed in ElasticsearchStore since ELSER embeds on index - */ -export class ElasticsearchEmbeddings extends Embeddings { - private readonly logger: Logger; - constructor(logger: Logger, params?: EmbeddingsParams) { - super(params ?? {}); - this.logger = logger; - } - - /** - * TODO: Use inference API if not re-indexing to create embedding vectors, e.g. - * - * POST _ml/trained_models/.elser_model_2/_infer - * { - * "docs":[{"text_field": "The fool doth think he is wise, but the wise man knows himself to be a fool."}] - * } - */ - - embedDocuments(documents: string[]): Promise { - // Note: implement if/when needed - this.logger.info('ElasticsearchEmbeddings.embedDocuments not implemented'); - return Promise.resolve([]); - } - - embedQuery(_: string): Promise { - // Note: implement if/when needed - this.logger.info('ElasticsearchEmbeddings.embedQuery not implemented'); - return Promise.resolve([]); - } -} diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/graphs/default_assistant_graph/index.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/graphs/default_assistant_graph/index.ts index f55006e452cd0..e9d2c1dd2618b 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/graphs/default_assistant_graph/index.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/graphs/default_assistant_graph/index.ts @@ -93,8 +93,9 @@ export const callAssistantGraph: AgentExecutor = async ({ const latestMessage = langChainMessages.slice(-1); // the last message - // Check if KB is available - const isEnabledKnowledgeBase = (await dataClients?.kbDataClient?.isModelDeployed()) ?? false; + // Check if KB is available (not feature flag related) + const isEnabledKnowledgeBase = + (await dataClients?.kbDataClient?.isInferenceEndpointExists()) ?? false; // Fetch any applicable tools that the source plugin may have registered const assistantToolParams: AssistantToolParams = { @@ -118,9 +119,8 @@ export const callAssistantGraph: AgentExecutor = async ({ ); // If KB enabled, fetch for any KB IndexEntries and generate a tool for each - if (isEnabledKnowledgeBase && dataClients?.kbDataClient?.isV2KnowledgeBaseEnabled) { + if (isEnabledKnowledgeBase) { const kbTools = await dataClients?.kbDataClient?.getAssistantTools({ - assistantToolParams, esClient, }); if (kbTools) { diff --git a/x-pack/plugins/elastic_assistant/server/routes/chat/chat_complete_route.test.ts b/x-pack/plugins/elastic_assistant/server/routes/chat/chat_complete_route.test.ts index f03a3394cdaac..5d277abb00667 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/chat/chat_complete_route.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/chat/chat_complete_route.test.ts @@ -86,7 +86,7 @@ const mockContext = { indexTemplateAndPattern: { alias: 'knowledge-base-alias', }, - isModelDeployed: jest.fn().mockResolvedValue(true), + isInferenceEndpointExists: jest.fn().mockResolvedValue(true), }), getAIAssistantAnonymizationFieldsDataClient: jest.fn().mockResolvedValue({ findDocuments: jest.fn().mockResolvedValue(getFindAnonymizationFieldsResultWithSingleHit()), diff --git a/x-pack/plugins/elastic_assistant/server/routes/chat/chat_complete_route.ts b/x-pack/plugins/elastic_assistant/server/routes/chat/chat_complete_route.ts index c6eb81dd86ebd..35b4999a30249 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/chat/chat_complete_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/chat/chat_complete_route.ts @@ -25,9 +25,7 @@ import { buildResponse } from '../../lib/build_response'; import { appendAssistantMessageToConversation, createConversationWithUserInput, - DEFAULT_PLUGIN_NAME, getIsKnowledgeBaseInstalled, - getPluginNameFromRequest, langChainExecute, performChecks, } from '../helpers'; @@ -222,25 +220,14 @@ export const chatCompleteRoute = ( }); } catch (err) { const error = transformError(err as Error); - const pluginName = getPluginNameFromRequest({ - request, - defaultPluginName: DEFAULT_PLUGIN_NAME, - logger, - }); - const v2KnowledgeBaseEnabled = - ctx.elasticAssistant.getRegisteredFeatures(pluginName).assistantKnowledgeBaseByDefault; const kbDataClient = - (await ctx.elasticAssistant.getAIAssistantKnowledgeBaseDataClient({ - v2KnowledgeBaseEnabled, - })) ?? undefined; + (await ctx.elasticAssistant.getAIAssistantKnowledgeBaseDataClient()) ?? undefined; const isKnowledgeBaseInstalled = await getIsKnowledgeBaseInstalled(kbDataClient); telemetry?.reportEvent(INVOKE_ASSISTANT_ERROR_EVENT.eventType, { actionTypeId: actionTypeId ?? '', model: request.body.model, errorMessage: error.message, - // TODO rm actionTypeId check when llmClass for bedrock streaming is implemented - // tracked here: https://github.com/elastic/security-team/issues/7363 assistantStreamingEnabled: request.body.isStream ?? false, isEnabledKnowledgeBase: isKnowledgeBaseInstalled, }); diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts index e4f520b190b5a..4e4b7e5fcd251 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts @@ -33,7 +33,7 @@ import { omit } from 'lodash/fp'; import { buildResponse } from '../../lib/build_response'; import { AssistantDataClients } from '../../lib/langchain/executors/types'; import { AssistantToolParams, ElasticAssistantRequestHandlerContext, GetElser } from '../../types'; -import { DEFAULT_PLUGIN_NAME, isV2KnowledgeBaseEnabled, performChecks } from '../helpers'; +import { DEFAULT_PLUGIN_NAME, performChecks } from '../helpers'; import { fetchLangSmithDataset } from './utils'; import { transformESSearchToAnonymizationFields } from '../../ai_assistant_data_clients/anonymization_fields/helpers'; import { EsAnonymizationFieldsSchema } from '../../ai_assistant_data_clients/anonymization_fields/types'; @@ -91,7 +91,6 @@ export const postEvaluateRoute = ( const actions = ctx.elasticAssistant.actions; const logger = assistantContext.logger.get('evaluate'); const abortSignal = getRequestAbortedSignal(request.events.aborted$); - const v2KnowledgeBaseEnabled = isV2KnowledgeBaseEnabled({ context: ctx, request }); // Perform license, authenticated user and evaluation FF checks const checkResponse = performChecks({ @@ -158,9 +157,7 @@ export const postEvaluateRoute = ( const conversationsDataClient = (await assistantContext.getAIAssistantConversationsDataClient()) ?? undefined; const kbDataClient = - (await assistantContext.getAIAssistantKnowledgeBaseDataClient({ - v2KnowledgeBaseEnabled, - })) ?? undefined; + (await assistantContext.getAIAssistantKnowledgeBaseDataClient()) ?? undefined; const dataClients: AssistantDataClients = { anonymizationFieldsDataClient, conversationsDataClient, @@ -246,7 +243,7 @@ export const postEvaluateRoute = ( // Check if KB is available const isEnabledKnowledgeBase = - (await dataClients.kbDataClient?.isModelDeployed()) ?? false; + (await dataClients.kbDataClient?.isInferenceEndpointExists()) ?? false; // Skeleton request from route to pass to the agents // params will be passed to the actions executor diff --git a/x-pack/plugins/elastic_assistant/server/routes/helpers.ts b/x-pack/plugins/elastic_assistant/server/routes/helpers.ts index 0c5c39f77d692..e68efd8e71f8f 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/helpers.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/helpers.ts @@ -374,8 +374,6 @@ export const langChainExecute = async ({ const assistantTools = assistantContext .getRegisteredTools(pluginName) .filter((x) => x.id !== 'attack-discovery'); // We don't (yet) support asking the assistant for NEW attack discoveries from a conversation - const v2KnowledgeBaseEnabled = - assistantContext.getRegisteredFeatures(pluginName).assistantKnowledgeBaseByDefault; // get a scoped esClient for assistant memory const esClient = context.core.elasticsearch.client.asCurrentUser; @@ -389,9 +387,7 @@ export const langChainExecute = async ({ // Create an ElasticsearchStore for KB interactions const kbDataClient = - (await assistantContext.getAIAssistantKnowledgeBaseDataClient({ - v2KnowledgeBaseEnabled, - })) ?? undefined; + (await assistantContext.getAIAssistantKnowledgeBaseDataClient()) ?? undefined; const dataClients: AssistantDataClients = { anonymizationFieldsDataClient: anonymizationFieldsDataClient ?? undefined, @@ -643,29 +639,6 @@ export const performChecks = ({ }; }; -/** - * Returns whether the v2 KB is enabled - * - * @param context - Route context - * @param request - Route KibanaRequest - - */ -export const isV2KnowledgeBaseEnabled = ({ - context, - request, -}: { - context: AwaitedProperties< - Pick - >; - request: KibanaRequest; -}): boolean => { - const pluginName = getPluginNameFromRequest({ - request, - defaultPluginName: DEFAULT_PLUGIN_NAME, - }); - return context.elasticAssistant.getRegisteredFeatures(pluginName).assistantKnowledgeBaseByDefault; -}; - /** * Telemetry function to determine whether knowledge base has been installed * @param kbDataClient @@ -674,11 +647,11 @@ export const getIsKnowledgeBaseInstalled = async ( kbDataClient?: AIAssistantKnowledgeBaseDataClient | null ): Promise => { let securityLabsDocsExist = false; - let isModelDeployed = false; + let isInferenceEndpointExists = false; if (kbDataClient != null) { try { - isModelDeployed = await kbDataClient.isModelDeployed(); - if (isModelDeployed) { + isInferenceEndpointExists = await kbDataClient.isInferenceEndpointExists(); + if (isInferenceEndpointExists) { securityLabsDocsExist = ( await kbDataClient.getKnowledgeBaseDocumentEntries({ @@ -692,5 +665,5 @@ export const getIsKnowledgeBaseInstalled = async ( } } - return isModelDeployed && securityLabsDocsExist; + return isInferenceEndpointExists && securityLabsDocsExist; }; diff --git a/x-pack/plugins/elastic_assistant/server/routes/index.ts b/x-pack/plugins/elastic_assistant/server/routes/index.ts index 928c3211faa9b..c30a62872a82d 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/index.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/index.ts @@ -13,7 +13,6 @@ export { postAttackDiscoveryRoute } from './attack_discovery/post/post_attack_di export { getAttackDiscoveryRoute } from './attack_discovery/get/get_attack_discovery'; // Knowledge Base -export { deleteKnowledgeBaseRoute } from './knowledge_base/delete_knowledge_base'; export { getKnowledgeBaseIndicesRoute } from './knowledge_base/get_knowledge_base_indices'; export { getKnowledgeBaseStatusRoute } from './knowledge_base/get_knowledge_base_status'; export { postKnowledgeBaseRoute } from './knowledge_base/post_knowledge_base'; diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/constants.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/constants.ts index 052b2cac57609..1c26c6d77b53f 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/constants.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/constants.ts @@ -5,8 +5,6 @@ * 2.0. */ -export const KNOWLEDGE_BASE_INDEX_PATTERN = '.kibana-elastic-ai-assistant-kb'; -export const KNOWLEDGE_BASE_INGEST_PIPELINE = '.kibana-elastic-ai-assistant-kb-ingest-pipeline'; // Query for determining if ESQL docs have been loaded, searches for a specific doc. Intended for the ElasticsearchStore.similaritySearch() // Note: We may want to add a tag of the resource name to the document metadata, so we can CRUD by specific resource export const ESQL_DOCS_LOADED_QUERY = diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/delete_knowledge_base.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/delete_knowledge_base.ts deleted file mode 100644 index 3e387e8a8a4d2..0000000000000 --- a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/delete_knowledge_base.ts +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IRouter, KibanaRequest } from '@kbn/core/server'; -import { transformError } from '@kbn/securitysolution-es-utils'; - -import { - ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION, - ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_URL, -} from '@kbn/elastic-assistant-common'; -import { - DeleteKnowledgeBaseRequestParams, - DeleteKnowledgeBaseResponse, -} from '@kbn/elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.gen'; -import { buildRouteValidationWithZod } from '@kbn/elastic-assistant-common/impl/schemas/common'; -import { buildResponse } from '../../lib/build_response'; -import { ElasticAssistantRequestHandlerContext } from '../../types'; -import { isV2KnowledgeBaseEnabled } from '../helpers'; - -/** - * Delete Knowledge Base index, pipeline, and resources (collection of documents) - * @param router - */ -export const deleteKnowledgeBaseRoute = ( - router: IRouter -) => { - router.versioned - .delete({ - access: 'internal', - path: ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_URL, - options: { - tags: ['access:elasticAssistant'], - }, - }) - .addVersion( - { - version: ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION, - validate: { - request: { - params: buildRouteValidationWithZod(DeleteKnowledgeBaseRequestParams), - }, - }, - }, - async (context, request: KibanaRequest, response) => { - const resp = buildResponse(response); - const ctx = await context.resolve(['core', 'elasticAssistant', 'licensing']); - const assistantContext = ctx.elasticAssistant; - const logger = ctx.elasticAssistant.logger; - - // FF Check for V2 KB - const v2KnowledgeBaseEnabled = isV2KnowledgeBaseEnabled({ context: ctx, request }); - - try { - const knowledgeBaseDataClient = - await assistantContext.getAIAssistantKnowledgeBaseDataClient({ - v2KnowledgeBaseEnabled, - }); - if (!knowledgeBaseDataClient) { - return response.custom({ body: { success: false }, statusCode: 500 }); - } - - // TODO: This delete API is likely not needed and can be replaced by the new `entries` API - const body: DeleteKnowledgeBaseResponse = { - success: false, - }; - - return response.ok({ body }); - } catch (err) { - logger.error(err); - const error = transformError(err); - - return resp.error({ - body: error.message, - statusCode: error.statusCode, - }); - } - } - ); -}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/entries/bulk_actions_route.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/entries/bulk_actions_route.ts index fc49068a09cc9..c6c5f9d94bef3 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/entries/bulk_actions_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/entries/bulk_actions_route.ts @@ -155,7 +155,6 @@ export const bulkActionKnowledgeBaseEntriesRoute = (router: ElasticAssistantPlug // Perform license, authenticated user and FF checks const checkResponse = performChecks({ - capability: 'assistantKnowledgeBaseByDefault', context: ctx, request, response, @@ -187,9 +186,7 @@ export const bulkActionKnowledgeBaseEntriesRoute = (router: ElasticAssistantPlug // subscribing to completed$, because it handles both cases when request was completed and aborted. // when route is finished by timeout, aborted$ is not getting fired request.events.completed$.subscribe(() => abortController.abort()); - const kbDataClient = await ctx.elasticAssistant.getAIAssistantKnowledgeBaseDataClient({ - v2KnowledgeBaseEnabled: true, - }); + const kbDataClient = await ctx.elasticAssistant.getAIAssistantKnowledgeBaseDataClient(); const spaceId = ctx.elasticAssistant.getSpaceId(); const authenticatedUser = checkResponse.currentUser; const userFilter = getKBUserFilter(authenticatedUser); @@ -288,8 +285,7 @@ export const bulkActionKnowledgeBaseEntriesRoute = (router: ElasticAssistantPlug global: entry.users != null && entry.users.length === 0, }) ), - getUpdateScript: (entry: UpdateKnowledgeBaseEntrySchema) => - getUpdateScript({ entry, isPatch: true }), + getUpdateScript: (entry: UpdateKnowledgeBaseEntrySchema) => getUpdateScript({ entry }), authenticatedUser, }); const created = diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/entries/create_route.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/entries/create_route.ts index d5df2d02055fd..4c1ea3851aaf5 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/entries/create_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/entries/create_route.ts @@ -47,7 +47,6 @@ export const createKnowledgeBaseEntryRoute = (router: ElasticAssistantPluginRout // Perform license, authenticated user and FF checks const checkResponse = performChecks({ - capability: 'assistantKnowledgeBaseByDefault', context: ctx, request, response, @@ -56,10 +55,7 @@ export const createKnowledgeBaseEntryRoute = (router: ElasticAssistantPluginRout return checkResponse.response; } - // Check mappings and upgrade if necessary -- this route only supports v2 KB, so always `true` - const kbDataClient = await ctx.elasticAssistant.getAIAssistantKnowledgeBaseDataClient({ - v2KnowledgeBaseEnabled: true, - }); + const kbDataClient = await ctx.elasticAssistant.getAIAssistantKnowledgeBaseDataClient(); logger.debug(() => `Creating KB Entry:\n${JSON.stringify(request.body)}`); const createResponse = await kbDataClient?.createKnowledgeBaseEntry({ diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/entries/find_route.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/entries/find_route.ts index 13334d0d829b1..e4035264a8352 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/entries/find_route.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/entries/find_route.ts @@ -58,7 +58,6 @@ export const findKnowledgeBaseEntriesRoute = (router: ElasticAssistantPluginRout // Perform license, authenticated user and FF checks const checkResponse = performChecks({ - capability: 'assistantKnowledgeBaseByDefault', context: ctx, request, response, @@ -67,9 +66,7 @@ export const findKnowledgeBaseEntriesRoute = (router: ElasticAssistantPluginRout return checkResponse.response; } - const kbDataClient = await ctx.elasticAssistant.getAIAssistantKnowledgeBaseDataClient({ - v2KnowledgeBaseEnabled: true, - }); + const kbDataClient = await ctx.elasticAssistant.getAIAssistantKnowledgeBaseDataClient(); const currentUser = checkResponse.currentUser; const userFilter = getKBUserFilter(currentUser); const systemFilter = ` AND (kb_resource:"user" OR type:"index")`; diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_status.test.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_status.test.ts index b30e5ac3653ad..a31af7596977a 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_status.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_status.test.ts @@ -35,7 +35,7 @@ describe('Get Knowledge Base Status Route', () => { }, isModelInstalled: jest.fn().mockResolvedValue(true), isSetupAvailable: jest.fn().mockResolvedValue(true), - isModelDeployed: jest.fn().mockResolvedValue(true), + isInferenceEndpointExists: jest.fn().mockResolvedValue(true), isSetupInProgress: false, isSecurityLabsDocsLoaded: jest.fn().mockResolvedValue(true), isUserDataExists: jest.fn().mockResolvedValue(true), diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_status.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_status.ts index f278cd469ac0e..4e8112b420d06 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_status.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_status.ts @@ -17,7 +17,6 @@ import { buildRouteValidationWithZod } from '@kbn/elastic-assistant-common/impl/ import { KibanaRequest } from '@kbn/core/server'; import { buildResponse } from '../../lib/build_response'; import { ElasticAssistantPluginRouter } from '../../types'; -import { isV2KnowledgeBaseEnabled } from '../helpers'; /** * Get the status of the Knowledge Base index, pipeline, and resources (collection of documents) @@ -49,12 +48,7 @@ export const getKnowledgeBaseStatusRoute = (router: ElasticAssistantPluginRouter const logger = ctx.elasticAssistant.logger; try { - // FF Check for V2 KB - const v2KnowledgeBaseEnabled = isV2KnowledgeBaseEnabled({ context: ctx, request }); - - const kbDataClient = await assistantContext.getAIAssistantKnowledgeBaseDataClient({ - v2KnowledgeBaseEnabled, - }); + const kbDataClient = await assistantContext.getAIAssistantKnowledgeBaseDataClient(); if (!kbDataClient) { return response.custom({ body: { success: false }, statusCode: 500 }); } @@ -63,7 +57,7 @@ export const getKnowledgeBaseStatusRoute = (router: ElasticAssistantPluginRouter const pipelineExists = true; // Installed at startup, always true const modelExists = await kbDataClient.isModelInstalled(); const setupAvailable = await kbDataClient.isSetupAvailable(); - const isModelDeployed = await kbDataClient.isModelDeployed(); + const isInferenceEndpointExists = await kbDataClient.isInferenceEndpointExists(); const body: ReadKnowledgeBaseResponse = { elser_exists: modelExists, @@ -73,13 +67,9 @@ export const getKnowledgeBaseStatusRoute = (router: ElasticAssistantPluginRouter pipeline_exists: pipelineExists, }; - if (indexExists && isModelDeployed) { - const securityLabsExists = v2KnowledgeBaseEnabled - ? await kbDataClient.isSecurityLabsDocsLoaded() - : true; - const userDataExists = v2KnowledgeBaseEnabled - ? await kbDataClient.isUserDataExists() - : true; + if (indexExists && isInferenceEndpointExists) { + const securityLabsExists = await kbDataClient.isSecurityLabsDocsLoaded(); + const userDataExists = await kbDataClient.isUserDataExists(); return response.ok({ body: { diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/post_knowledge_base.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/post_knowledge_base.ts index 23604886e4a52..fa7716a51033d 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/post_knowledge_base.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/post_knowledge_base.ts @@ -16,7 +16,6 @@ import { buildRouteValidationWithZod } from '@kbn/elastic-assistant-common/impl/ import { IKibanaResponse } from '@kbn/core/server'; import { buildResponse } from '../../lib/build_response'; import { ElasticAssistantPluginRouter } from '../../types'; -import { isV2KnowledgeBaseEnabled } from '../helpers'; // Since we're awaiting on ELSER setup, this could take a bit (especially if ML needs to autoscale) // Consider just returning if attempt was successful, and switch to client polling @@ -54,19 +53,12 @@ export const postKnowledgeBaseRoute = (router: ElasticAssistantPluginRouter) => const assistantContext = ctx.elasticAssistant; const core = ctx.core; const soClient = core.savedObjects.getClient(); - - // FF Check for V2 KB - const v2KnowledgeBaseEnabled = isV2KnowledgeBaseEnabled({ context: ctx, request }); - // Only allow modelId override if FF is enabled as this will re-write the ingest pipeline and break any previous KB entries - // This is only really needed for API integration tests - const modelIdOverride = v2KnowledgeBaseEnabled ? request.query.modelId : undefined; const ignoreSecurityLabs = request.query.ignoreSecurityLabs; try { const knowledgeBaseDataClient = await assistantContext.getAIAssistantKnowledgeBaseDataClient({ - modelIdOverride, - v2KnowledgeBaseEnabled, + modelIdOverride: request.query.modelId, }); if (!knowledgeBaseDataClient) { return response.custom({ body: { success: false }, statusCode: 500 }); @@ -74,7 +66,6 @@ export const postKnowledgeBaseRoute = (router: ElasticAssistantPluginRouter) => await knowledgeBaseDataClient.setupKnowledgeBase({ soClient, - v2KnowledgeBaseEnabled, ignoreSecurityLabs, }); diff --git a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts index bb217f7f5aa3a..43264a6c1f54b 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts @@ -23,9 +23,7 @@ import { buildResponse } from '../lib/build_response'; import { ElasticAssistantRequestHandlerContext, GetElser } from '../types'; import { appendAssistantMessageToConversation, - DEFAULT_PLUGIN_NAME, getIsKnowledgeBaseInstalled, - getPluginNameFromRequest, getSystemPromptFromUserConversation, langChainExecute, performChecks, @@ -159,17 +157,9 @@ export const postActionsConnectorExecuteRoute = ( if (onLlmResponse) { await onLlmResponse(error.message, {}, true); } - const pluginName = getPluginNameFromRequest({ - request, - defaultPluginName: DEFAULT_PLUGIN_NAME, - logger, - }); - const v2KnowledgeBaseEnabled = - assistantContext.getRegisteredFeatures(pluginName).assistantKnowledgeBaseByDefault; + const kbDataClient = - (await assistantContext.getAIAssistantKnowledgeBaseDataClient({ - v2KnowledgeBaseEnabled, - })) ?? undefined; + (await assistantContext.getAIAssistantKnowledgeBaseDataClient()) ?? undefined; const isKnowledgeBaseInstalled = await getIsKnowledgeBaseInstalled(kbDataClient); telemetry.reportEvent(INVOKE_ASSISTANT_ERROR_EVENT.eventType, { actionTypeId: request.body.actionTypeId, diff --git a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts index 7d97029e7252a..3f81763db49d9 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts @@ -83,33 +83,28 @@ export class RequestContextFactory implements IRequestContextFactory { telemetry: core.analytics, - // Note: Due to plugin lifecycle and feature flag registration timing, we need to pass in the feature flag here - // Remove `v2KnowledgeBaseEnabled` once 'assistantKnowledgeBaseByDefault' feature flag is removed - // Additionally, modelIdOverride is used here to enable setting up the KB using a different ELSER model, which + // Note: modelIdOverride is used here to enable setting up the KB using a different ELSER model, which // is necessary for testing purposes (`pt_tiny_elser`). - getAIAssistantKnowledgeBaseDataClient: memoize( - async ({ modelIdOverride, v2KnowledgeBaseEnabled = false }) => { - const currentUser = getCurrentUser(); - - const { securitySolutionAssistant } = await coreStart.capabilities.resolveCapabilities( - request, - { - capabilityPath: 'securitySolutionAssistant.*', - } - ); - - return this.assistantService.createAIAssistantKnowledgeBaseDataClient({ - spaceId: getSpaceId(), - logger: this.logger, - licensing: context.licensing, - currentUser, - modelIdOverride, - v2KnowledgeBaseEnabled, - manageGlobalKnowledgeBaseAIAssistant: - securitySolutionAssistant.manageGlobalKnowledgeBaseAIAssistant as boolean, - }); - } - ), + getAIAssistantKnowledgeBaseDataClient: memoize(async (params) => { + const currentUser = getCurrentUser(); + + const { securitySolutionAssistant } = await coreStart.capabilities.resolveCapabilities( + request, + { + capabilityPath: 'securitySolutionAssistant.*', + } + ); + + return this.assistantService.createAIAssistantKnowledgeBaseDataClient({ + spaceId: getSpaceId(), + logger: this.logger, + licensing: context.licensing, + currentUser, + modelIdOverride: params?.modelIdOverride, + manageGlobalKnowledgeBaseAIAssistant: + securitySolutionAssistant.manageGlobalKnowledgeBaseAIAssistant as boolean, + }); + }), getAttackDiscoveryDataClient: memoize(() => { const currentUser = getCurrentUser(); diff --git a/x-pack/plugins/elastic_assistant/server/types.ts b/x-pack/plugins/elastic_assistant/server/types.ts index 00fec0dcabc6d..b021ef5a7017d 100755 --- a/x-pack/plugins/elastic_assistant/server/types.ts +++ b/x-pack/plugins/elastic_assistant/server/types.ts @@ -126,7 +126,7 @@ export interface ElasticAssistantApiRequestHandlerContext { getCurrentUser: () => AuthenticatedUser | null; getAIAssistantConversationsDataClient: () => Promise; getAIAssistantKnowledgeBaseDataClient: ( - params: GetAIAssistantKnowledgeBaseDataClientParams + params?: GetAIAssistantKnowledgeBaseDataClientParams ) => Promise; getAttackDiscoveryDataClient: () => Promise; getAIAssistantPromptsDataClient: () => Promise; diff --git a/x-pack/plugins/entity_manager/public/lib/entity_client.test.ts b/x-pack/plugins/entity_manager/public/lib/entity_client.test.ts index dbaf1205cdf98..6679140314cb5 100644 --- a/x-pack/plugins/entity_manager/public/lib/entity_client.test.ts +++ b/x-pack/plugins/entity_manager/public/lib/entity_client.test.ts @@ -5,16 +5,17 @@ * 2.0. */ -import { EntityClient, EnitityInstance } from './entity_client'; +import { EntityClient } from './entity_client'; import { coreMock } from '@kbn/core/public/mocks'; +import type { EntityInstance } from '@kbn/entities-schema'; -const commonEntityFields: EnitityInstance = { +const commonEntityFields: EntityInstance = { entity: { last_seen_timestamp: '2023-10-09T00:00:00Z', id: '1', display_name: 'entity_name', definition_id: 'entity_definition_id', - } as EnitityInstance['entity'], + } as EntityInstance['entity'], }; describe('EntityClient', () => { @@ -26,7 +27,7 @@ describe('EntityClient', () => { describe('asKqlFilter', () => { it('should return the kql filter', () => { - const entityLatest: EnitityInstance = { + const entityLatest: EntityInstance = { entity: { ...commonEntityFields.entity, identity_fields: ['service.name', 'service.environment'], @@ -42,7 +43,7 @@ describe('EntityClient', () => { }); it('should return the kql filter when indentity_fields is composed by multiple fields', () => { - const entityLatest: EnitityInstance = { + const entityLatest: EntityInstance = { entity: { ...commonEntityFields.entity, identity_fields: ['service.name', 'service.environment'], @@ -59,7 +60,7 @@ describe('EntityClient', () => { }); it('should ignore fields that are not present in the entity', () => { - const entityLatest: EnitityInstance = { + const entityLatest: EntityInstance = { entity: { ...commonEntityFields.entity, identity_fields: ['host.name', 'foo.bar'], @@ -76,7 +77,7 @@ describe('EntityClient', () => { describe('getIdentityFieldsValue', () => { it('should return identity fields values', () => { - const entityLatest: EnitityInstance = { + const entityLatest: EntityInstance = { entity: { ...commonEntityFields.entity, identity_fields: ['service.name', 'service.environment'], @@ -93,7 +94,7 @@ describe('EntityClient', () => { }); it('should return identity fields values when indentity_fields is composed by multiple fields', () => { - const entityLatest: EnitityInstance = { + const entityLatest: EntityInstance = { entity: { ...commonEntityFields.entity, identity_fields: ['service.name', 'service.environment'], @@ -112,7 +113,7 @@ describe('EntityClient', () => { }); it('should return identity fields when field is in the root', () => { - const entityLatest: EnitityInstance = { + const entityLatest: EntityInstance = { entity: { ...commonEntityFields.entity, identity_fields: ['name'], @@ -127,7 +128,7 @@ describe('EntityClient', () => { }); it('should throw an error when identity fields are missing', () => { - const entityLatest: EnitityInstance = { + const entityLatest: EntityInstance = { ...commonEntityFields, }; diff --git a/x-pack/plugins/entity_manager/public/lib/entity_client.ts b/x-pack/plugins/entity_manager/public/lib/entity_client.ts index 08794873ba930..7132dc50330d5 100644 --- a/x-pack/plugins/entity_manager/public/lib/entity_client.ts +++ b/x-pack/plugins/entity_manager/public/lib/entity_client.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { z } from '@kbn/zod'; import { CoreSetup, CoreStart } from '@kbn/core/public'; import { ClientRequestParamsOf, @@ -14,7 +13,7 @@ import { isHttpFetchError, } from '@kbn/server-route-repository-client'; import { type KueryNode, nodeTypes, toKqlExpression } from '@kbn/es-query'; -import { entityLatestSchema } from '@kbn/entities-schema'; +import type { EntityInstance, EntityMetadata } from '@kbn/entities-schema'; import { castArray } from 'lodash'; import { DisableManagedEntityResponse, @@ -39,8 +38,6 @@ type CreateEntityDefinitionQuery = QueryParamOf< ClientRequestParamsOf >; -export type EnitityInstance = z.infer; - export class EntityClient { public readonly repositoryClient: EntityManagerRepositoryClient['fetch']; @@ -90,8 +87,12 @@ export class EntityClient { } } - asKqlFilter(entityLatest: EnitityInstance) { - const identityFieldsValue = this.getIdentityFieldsValue(entityLatest); + asKqlFilter( + entityInstance: { + entity: Pick; + } & Required + ) { + const identityFieldsValue = this.getIdentityFieldsValue(entityInstance); const nodes: KueryNode[] = Object.entries(identityFieldsValue).map(([identityField, value]) => { return nodeTypes.function.buildNode('is', identityField, value); @@ -104,8 +105,12 @@ export class EntityClient { return toKqlExpression(kqlExpression); } - getIdentityFieldsValue(entityLatest: EnitityInstance) { - const { identity_fields: identityFields } = entityLatest.entity; + getIdentityFieldsValue( + entityInstance: { + entity: Pick; + } & Required + ) { + const { identity_fields: identityFields } = entityInstance.entity; if (!identityFields) { throw new Error('Identity fields are missing'); @@ -114,7 +119,7 @@ export class EntityClient { return castArray(identityFields).reduce((acc, field) => { const value = field.split('.').reduce((obj: any, part: string) => { return obj && typeof obj === 'object' ? (obj as Record)[part] : undefined; - }, entityLatest); + }, entityInstance); if (value) { acc[field] = value; diff --git a/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts b/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts index 5e518618d98d8..422c16a726843 100644 --- a/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts +++ b/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts @@ -24,6 +24,12 @@ export const initFindFieldsMetadataRoute = ({ .addVersion( { version: '1', + security: { + authz: { + enabled: false, + reason: 'This route is opted out from authorization', + }, + }, validate: { request: { query: createValidationFunction(fieldsMetadataV1.findFieldsMetadataRequestQueryRT), diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.test.ts index 255572d57cf49..1a4f2998d49dd 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.test.ts @@ -42,6 +42,30 @@ const mockedRemoveArchiveEntries = removeArchiveEntries as jest.MockedFunction< let soClient: jest.Mocked; let esClient: jest.Mocked; +const assetsMap = new Map([ + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json', + Buffer.from('{"content": "data"}'), + ], + ['security_detection_engine-8.16.1/LICENSE.txt', Buffer.from('{"content": "data"}')], + ['security_detection_engine-8.16.1/NOTICE.txt', Buffer.from('{"content": "data"}')], + ['security_detection_engine-8.16.1/changelog.yml', Buffer.from('{"content": "data"}')], + ['security_detection_engine-8.16.1/manifest.yml', Buffer.from('{"content": "data"}')], + ['security_detection_engine-8.16.1/docs/README.md', Buffer.from('{"content": "data"}')], + [ + 'security_detection_engine-8.16.1/img/security-logo-color-64px.svg', + Buffer.from('{"content": "data"}'), + ], + [ + 'security_detection_engine-8.16.1/kibana/security_rule/000047bb-b27a-47ec-8b62-ef1a5d2c9e19_208.json', + Buffer.from('{"content": "data"}'), + ], + [ + 'security_detection_engine-8.16.1/kibana/security_rule/000047bb-b27a-47ec-8b62-ef1a5d2c9e19_209.json', + Buffer.from('{"content": "data"}'), + ], +]); + const packageInstallContext = { packageInfo: { title: 'title', @@ -56,13 +80,8 @@ const packageInstallContext = { owner: { github: 'elastic/fleet' }, } as any, paths: ['some/path/1', 'some/path/2'], - assetsMap: new Map([ - [ - 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json', - Buffer.from('{"content": "data"}'), - ], - ]), - archiveIterator: createArchiveIteratorFromMap(new Map()), + assetsMap, + archiveIterator: createArchiveIteratorFromMap(assetsMap), }; const getMockInstalledPackageSo = ( installedEs: EsAssetReference[] = [] @@ -196,6 +215,63 @@ describe('stepSaveArchiveEntries', () => { ], }); }); + + it('should save package icons, readme, and changelog but not Kibana assets with useStreaming:true ', async () => { + jest.mocked(mockedSaveArchiveEntriesFromAssetsMap).mockResolvedValue({ + saved_objects: [ + { + id: 'test', + attributes: { + package_name: 'test-package', + package_version: '1.0.0', + install_source: 'registry', + asset_path: 'some/path', + media_type: '', + data_utf8: '', + data_base64: '', + }, + type: '', + references: [], + }, + ], + }); + await stepSaveArchiveEntries({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + useStreaming: true, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + expect( + [ + ...(jest + .mocked(mockedSaveArchiveEntriesFromAssetsMap) + .mock.lastCall?.[0].assetsMap?.keys() ?? []), + ].sort() + ).toMatchInlineSnapshot(` + Array [ + "endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json", + "security_detection_engine-8.16.1/LICENSE.txt", + "security_detection_engine-8.16.1/NOTICE.txt", + "security_detection_engine-8.16.1/changelog.yml", + "security_detection_engine-8.16.1/docs/README.md", + "security_detection_engine-8.16.1/img/security-logo-color-64px.svg", + "security_detection_engine-8.16.1/manifest.yml", + ] + `); + }); }); describe('cleanupArchiveEntriesStep', () => { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts index 7db44bb243f85..f081d9a93e633 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts @@ -14,7 +14,7 @@ import { withPackageSpan } from '../../utils'; import type { InstallContext } from '../_state_machine_package_install'; import { INSTALL_STATES } from '../../../../../../common/types'; -import { MANIFEST_NAME } from '../../../archive/parse'; +import { isKibanaAssetType } from '../../../kibana/assets/install'; export async function stepSaveArchiveEntries(context: InstallContext) { const { packageInstallContext, savedObjectsClient, installSource, useStreaming } = context; @@ -28,7 +28,8 @@ export async function stepSaveArchiveEntries(context: InstallContext) { if (useStreaming) { assetsMap = new Map(); await archiveIterator.traverseEntries(async (entry) => { - if (entry.path.endsWith(MANIFEST_NAME)) { + // Skip only kibana assets type + if (!isKibanaAssetType(entry.path)) { assetsMap.set(entry.path, entry.buffer); } }); diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.test.tsx index 7d41e326372e5..ad53002a1b0cb 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.test.tsx @@ -5,20 +5,12 @@ * 2.0. */ -import React, { useEffect } from 'react'; -import { ReactWrapper } from 'enzyme'; +import React from 'react'; import { screen, within } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { EditorFrame, EditorFrameProps } from './editor_frame'; -import { - DatasourceMap, - DatasourcePublicAPI, - DatasourceSuggestion, - Visualization, - VisualizationMap, -} from '../../types'; -import { act } from '@testing-library/react'; +import { DatasourceMap, DatasourcePublicAPI, Visualization, VisualizationMap } from '../../types'; import { coreMock } from '@kbn/core/public/mocks'; import { createMockVisualization, @@ -29,30 +21,16 @@ import { renderWithReduxStore, } from '../../mocks'; import { inspectorPluginMock } from '@kbn/inspector-plugin/public/mocks'; -import { Droppable, useDragDropContext } from '@kbn/dom-drag-drop'; import { uiActionsPluginMock } from '@kbn/ui-actions-plugin/public/mocks'; import { chartPluginMock } from '@kbn/charts-plugin/public/mocks'; import { expressionsPluginMock } from '@kbn/expressions-plugin/public/mocks'; -import { mockDataPlugin, mountWithProvider } from '../../mocks'; +import { mockDataPlugin } from '../../mocks'; import { LensAppState, setState } from '../../state_management'; import { getLensInspectorService } from '../../lens_inspector_service'; import { createIndexPatternServiceMock } from '../../mocks/data_views_service_mock'; import { dataViewPluginMocks } from '@kbn/data-views-plugin/public/mocks'; import { EventAnnotationServiceType } from '@kbn/event-annotation-plugin/public'; -function generateSuggestion(state = {}): DatasourceSuggestion { - return { - state, - table: { - columns: [], - isMultiRow: true, - layerId: 'first', - changeType: 'unchanged', - }, - keptLayerIds: ['first'], - }; -} - function wrapDataViewsContract() { const dataViewsContract = dataViewPluginMocks.createStartContract(); return { @@ -437,180 +415,5 @@ describe('editor_frame', () => { }) ); }); - describe('legacy tests', () => { - let instance: ReactWrapper; - - afterEach(() => { - instance.unmount(); - }); - - it('should use the currently selected visualization if possible on field drop', async () => { - mockDatasource.getLayers.mockReturnValue(['first', 'second', 'third']); - const suggestionVisState = {}; - const props = { - ...getDefaultProps(), - visualizationMap: { - testVis: { - ...mockVisualization, - getSuggestions: () => [ - { - score: 0.2, - state: {}, - title: 'Suggestion1', - previewIcon: 'empty', - }, - { - score: 0.6, - state: suggestionVisState, - title: 'Suggestion2', - previewIcon: 'empty', - }, - ], - }, - testVis2: { - ...mockVisualization2, - getSuggestions: () => [ - { - score: 0.8, - state: {}, - title: 'Suggestion3', - previewIcon: 'empty', - }, - ], - }, - }, - datasourceMap: { - testDatasource: { - ...mockDatasource, - getDatasourceSuggestionsForField: () => [generateSuggestion()], - getDatasourceSuggestionsFromCurrentState: () => [generateSuggestion()], - getDatasourceSuggestionsForVisualizeField: () => [generateSuggestion()], - DataPanelComponent: jest.fn().mockImplementation(() =>
), - }, - }, - } as EditorFrameProps; - instance = ( - await mountWithProvider(, { - preloadedState: { - datasourceStates: { - testDatasource: { - isLoading: false, - state: { - internalState1: '', - }, - }, - }, - }, - }) - ).instance; - - instance.update(); - - act(() => { - instance.find('[data-test-subj="mockVisA"]').find(Droppable).prop('onDrop')!( - { - indexPatternId: '1', - field: {}, - id: '1', - humanData: { label: 'draggedField' }, - }, - 'field_add' - ); - }); - - expect(mockVisualization.getConfiguration).toHaveBeenCalledWith( - expect.objectContaining({ - state: suggestionVisState, - }) - ); - }); - - it('should use the highest priority suggestion available', async () => { - mockDatasource.getLayers.mockReturnValue(['first', 'second', 'third']); - const suggestionVisState = {}; - const mockVisualization3 = { - ...createMockVisualization('testVis3', ['third']), - getSuggestions: () => [ - { - score: 0.9, - state: suggestionVisState, - title: 'Suggestion3', - previewIcon: 'empty', - }, - { - score: 0.7, - state: {}, - title: 'Suggestion4', - previewIcon: 'empty', - }, - ], - }; - - const props = { - ...getDefaultProps(), - visualizationMap: { - testVis: { - ...mockVisualization, - // do not return suggestions for the currently active vis, otherwise it will be chosen - getSuggestions: () => [], - }, - testVis2: { - ...mockVisualization2, - getSuggestions: () => [], - }, - testVis3: { - ...mockVisualization3, - }, - }, - datasourceMap: { - testDatasource: { - ...mockDatasource, - getDatasourceSuggestionsForField: () => [generateSuggestion()], - getDatasourceSuggestionsFromCurrentState: () => [generateSuggestion()], - getDatasourceSuggestionsForVisualizeField: () => [generateSuggestion()], - DataPanelComponent: jest.fn().mockImplementation(() => { - const [, dndDispatch] = useDragDropContext(); - useEffect(() => { - dndDispatch({ - type: 'startDragging', - payload: { - dragging: { - id: 'draggedField', - humanData: { label: '1' }, - }, - }, - }); - }, [dndDispatch]); - return
; - }), - }, - }, - } as EditorFrameProps; - - instance = (await mountWithProvider()).instance; - - instance.update(); - - act(() => { - instance.find(Droppable).filter('[dataTestSubj="lnsWorkspace"]').prop('onDrop')!( - { - indexPatternId: '1', - field: {}, - id: '1', - humanData: { - label: 'label', - }, - }, - 'field_add' - ); - }); - - expect(mockVisualization3.getConfiguration).toHaveBeenCalledWith( - expect.objectContaining({ - state: suggestionVisState, - }) - ); - }); - }); }); }); diff --git a/x-pack/plugins/ml/public/application/components/anomalies_table/links_menu.tsx b/x-pack/plugins/ml/public/application/components/anomalies_table/links_menu.tsx index 89bb4b800eb91..1f81a94227611 100644 --- a/x-pack/plugins/ml/public/application/components/anomalies_table/links_menu.tsx +++ b/x-pack/plugins/ml/public/application/components/anomalies_table/links_menu.tsx @@ -930,7 +930,7 @@ export const LinksMenuUI = (props: LinksMenuProps) => { items.push( @@ -946,7 +946,7 @@ export const LinksMenuUI = (props: LinksMenuProps) => { items.push( { closePopover(); const additionalField = getAdditionalField(anomaly); diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/split_cards/split_cards.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/split_cards/split_cards.tsx index d09791941a379..96692ab073738 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/split_cards/split_cards.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/split_cards/split_cards.tsx @@ -97,42 +97,45 @@ export const SplitCards: FC> = memo( } return ( - - - {(fieldValues.length === 0 || numberOfDetectors === 0) && <>{children}} - {fieldValues.length > 0 && numberOfDetectors > 0 && splitField !== null && ( - - {(jobType === JOB_TYPE.MULTI_METRIC || jobType === JOB_TYPE.GEO) && ( - + <> + + + {(fieldValues.length === 0 || numberOfDetectors === 0) && <>{children}} + {fieldValues.length > 0 && numberOfDetectors > 0 && splitField !== null && ( + + {(jobType === JOB_TYPE.MULTI_METRIC || jobType === JOB_TYPE.GEO) && ( + +
+ +
+ + + )} + + {getBackPanels()} +
- + {fieldValues[0]}
- - - )} - - {getBackPanels()} - -
- {fieldValues[0]} -
- - <>{children} - - - )} - - + + <>{children} + + + )} + + + {splitField !== null ? : null} + ); } ); diff --git a/x-pack/plugins/observability_solution/apm/ftr_e2e/cypress/e2e/transaction_details/transaction_details.cy.ts b/x-pack/plugins/observability_solution/apm/ftr_e2e/cypress/e2e/transaction_details/transaction_details.cy.ts index 730e9c443854e..af23fc8a2ad7e 100644 --- a/x-pack/plugins/observability_solution/apm/ftr_e2e/cypress/e2e/transaction_details/transaction_details.cy.ts +++ b/x-pack/plugins/observability_solution/apm/ftr_e2e/cypress/e2e/transaction_details/transaction_details.cy.ts @@ -115,7 +115,10 @@ describe.skip('Transaction details', () => { ); cy.contains('Top 5 errors', { timeout: 30000 }); - cy.getByTestSubj('topErrorsForTransactionTable').contains('a', '[MockError] Foo').click(); + cy.getByTestSubj('topErrorsForTransactionTable') + .should('be.visible') + .contains('a', '[MockError] Foo', { timeout: 10000 }) + .click(); cy.url().should('include', 'opbeans-java/errors'); }); diff --git a/x-pack/plugins/observability_solution/infra/server/routes/entities/get_data_stream_types.test.ts b/x-pack/plugins/observability_solution/infra/server/routes/entities/get_data_stream_types.test.ts index c66416331e4d0..19f7e47e84fce 100644 --- a/x-pack/plugins/observability_solution/infra/server/routes/entities/get_data_stream_types.test.ts +++ b/x-pack/plugins/observability_solution/infra/server/routes/entities/get_data_stream_types.test.ts @@ -74,7 +74,7 @@ describe('getDataStreamTypes', () => { it('should return metrics and entity source_data_stream types when entityCentriExperienceEnabled is true and has entity data', async () => { (getHasMetricsData as jest.Mock).mockResolvedValue(true); (getLatestEntity as jest.Mock).mockResolvedValue({ - 'source_data_stream.type': ['logs', 'metrics'], + sourceDataStreamType: ['logs', 'metrics'], }); const params = { @@ -118,7 +118,7 @@ describe('getDataStreamTypes', () => { it('should return entity source_data_stream types when has no metrics', async () => { (getHasMetricsData as jest.Mock).mockResolvedValue(false); (getLatestEntity as jest.Mock).mockResolvedValue({ - 'source_data_stream.type': ['logs', 'traces'], + sourceDataStreamType: ['logs', 'traces'], }); const params = { diff --git a/x-pack/plugins/observability_solution/infra/server/routes/entities/get_data_stream_types.ts b/x-pack/plugins/observability_solution/infra/server/routes/entities/get_data_stream_types.ts index 3218ae257f1a2..f9b2d41bbe050 100644 --- a/x-pack/plugins/observability_solution/infra/server/routes/entities/get_data_stream_types.ts +++ b/x-pack/plugins/observability_solution/infra/server/routes/entities/get_data_stream_types.ts @@ -7,11 +7,9 @@ import { type EntityClient } from '@kbn/entityManager-plugin/server/lib/entity_client'; import { findInventoryFields } from '@kbn/metrics-data-access-plugin/common'; -import { - EntityDataStreamType, - SOURCE_DATA_STREAM_TYPE, -} from '@kbn/observability-shared-plugin/common'; +import { EntityDataStreamType } from '@kbn/observability-shared-plugin/common'; import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; +import { castArray } from 'lodash'; import { type InfraMetricsClient } from '../../lib/helpers/get_infra_metrics_client'; import { getHasMetricsData } from './get_has_metrics_data'; import { getLatestEntity } from './get_latest_entity'; @@ -45,15 +43,15 @@ export async function getDataStreamTypes({ return Array.from(sourceDataStreams); } - const entity = await getLatestEntity({ + const latestEntity = await getLatestEntity({ inventoryEsClient: obsEsClient, entityId, entityType, entityManagerClient, }); - if (entity?.[SOURCE_DATA_STREAM_TYPE]) { - [entity[SOURCE_DATA_STREAM_TYPE]].flat().forEach((item) => { + if (latestEntity) { + castArray(latestEntity.sourceDataStreamType).forEach((item) => { sourceDataStreams.add(item as EntityDataStreamType); }); } diff --git a/x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts b/x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts index 7bcce2964fd13..31e778313f939 100644 --- a/x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts +++ b/x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts @@ -7,20 +7,16 @@ import { ENTITY_LATEST, entitiesAliasPattern } from '@kbn/entities-schema'; import { type EntityClient } from '@kbn/entityManager-plugin/server/lib/entity_client'; -import { - ENTITY_TYPE, - SOURCE_DATA_STREAM_TYPE, -} from '@kbn/observability-shared-plugin/common/field_names/elasticsearch'; +import { ENTITY_TYPE, SOURCE_DATA_STREAM_TYPE } from '@kbn/observability-shared-plugin/common'; import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; -import { esqlResultToPlainObjects } from '@kbn/observability-utils/es/utils/esql_result_to_plain_objects'; const ENTITIES_LATEST_ALIAS = entitiesAliasPattern({ type: '*', dataset: ENTITY_LATEST, }); -interface Entity { - [SOURCE_DATA_STREAM_TYPE]: string | string[]; +interface EntitySourceResponse { + sourceDataStreamType?: string | string[]; } export async function getLatestEntity({ @@ -33,7 +29,7 @@ export async function getLatestEntity({ entityType: 'host' | 'container'; entityId: string; entityManagerClient: EntityClient; -}): Promise { +}): Promise { const { definitions } = await entityManagerClient.getEntityDefinitions({ builtIn: true, type: entityType, @@ -41,10 +37,12 @@ export async function getLatestEntity({ const hostOrContainerIdentityField = definitions[0]?.identityFields?.[0]?.field; if (hostOrContainerIdentityField === undefined) { - return { [SOURCE_DATA_STREAM_TYPE]: [] }; + return undefined; } - const latestEntitiesEsqlResponse = await inventoryEsClient.esql('get_latest_entities', { + const response = await inventoryEsClient.esql<{ + source_data_stream?: { type?: string | string[] }; + }>('get_latest_entities', { query: `FROM ${ENTITIES_LATEST_ALIAS} | WHERE ${ENTITY_TYPE} == ? | WHERE ${hostOrContainerIdentityField} == ? @@ -53,5 +51,5 @@ export async function getLatestEntity({ params: [entityType, entityId], }); - return esqlResultToPlainObjects(latestEntitiesEsqlResponse)[0]; + return { sourceDataStreamType: response[0].source_data_stream?.type }; } diff --git a/x-pack/plugins/observability_solution/inventory/.storybook/get_mock_inventory_context.tsx b/x-pack/plugins/observability_solution/inventory/.storybook/get_mock_inventory_context.tsx index d3d28fe040198..0188ed3143034 100644 --- a/x-pack/plugins/observability_solution/inventory/.storybook/get_mock_inventory_context.tsx +++ b/x-pack/plugins/observability_solution/inventory/.storybook/get_mock_inventory_context.tsx @@ -24,7 +24,14 @@ export function getMockInventoryContext(): InventoryKibanaContext { return { ...coreStart, - entityManager: {} as unknown as EntityManagerPublicPluginStart, + entityManager: { + entityClient: { + asKqlFilter: jest.fn(), + getIdentityFieldsValue() { + return 'entity_id'; + }, + }, + } as unknown as EntityManagerPublicPluginStart, observabilityShared: {} as unknown as ObservabilitySharedPluginStart, inference: {} as unknown as InferencePublicStart, share: { diff --git a/x-pack/plugins/observability_solution/inventory/common/entities.ts b/x-pack/plugins/observability_solution/inventory/common/entities.ts index 3a9684a38254a..65fd8a4ffbd7a 100644 --- a/x-pack/plugins/observability_solution/inventory/common/entities.ts +++ b/x-pack/plugins/observability_solution/inventory/common/entities.ts @@ -4,24 +4,15 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { z } from '@kbn/zod'; -import { ENTITY_LATEST, entitiesAliasPattern, entityLatestSchema } from '@kbn/entities-schema'; -import { - ENTITY_DEFINITION_ID, - ENTITY_DISPLAY_NAME, - ENTITY_ID, - ENTITY_IDENTITY_FIELDS, - ENTITY_LAST_SEEN, - ENTITY_TYPE, -} from '@kbn/observability-shared-plugin/common'; +import { ENTITY_LATEST, entitiesAliasPattern, type EntityMetadata } from '@kbn/entities-schema'; import { decode, encode } from '@kbn/rison'; import { isRight } from 'fp-ts/lib/Either'; import * as t from 'io-ts'; export const entityColumnIdsRt = t.union([ - t.literal(ENTITY_DISPLAY_NAME), - t.literal(ENTITY_LAST_SEEN), - t.literal(ENTITY_TYPE), + t.literal('entityDisplayName'), + t.literal('entityLastSeenTimestamp'), + t.literal('entityType'), t.literal('alertsCount'), t.literal('actions'), ]); @@ -80,23 +71,20 @@ export const ENTITIES_LATEST_ALIAS = entitiesAliasPattern({ dataset: ENTITY_LATEST, }); -export interface Entity { - [ENTITY_LAST_SEEN]: string; - [ENTITY_ID]: string; - [ENTITY_TYPE]: string; - [ENTITY_DISPLAY_NAME]: string; - [ENTITY_DEFINITION_ID]: string; - [ENTITY_IDENTITY_FIELDS]: string | string[]; - alertsCount?: number; - [key: string]: any; -} - export type EntityGroup = { count: number; } & { [key: string]: string; }; -export type InventoryEntityLatest = z.infer & { +export type InventoryEntity = { + entityId: string; + entityType: string; + entityIdentityFields: string | string[]; + entityDisplayName: string; + entityDefinitionId: string; + entityLastSeenTimestamp: string; + entityDefinitionVersion: string; + entitySchemaVersion: string; alertsCount?: number; -}; +} & EntityMetadata; diff --git a/x-pack/plugins/observability_solution/inventory/common/utils/entity_type_guards.ts b/x-pack/plugins/observability_solution/inventory/common/utils/entity_type_guards.ts new file mode 100644 index 0000000000000..dccc888abd8dc --- /dev/null +++ b/x-pack/plugins/observability_solution/inventory/common/utils/entity_type_guards.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AgentName } from '@kbn/elastic-agent-utils'; +import type { InventoryEntity } from '../entities'; + +interface BuiltinEntityMap { + host: InventoryEntity & { cloud?: { provider?: string[] } }; + container: InventoryEntity & { cloud?: { provider?: string[] } }; + service: InventoryEntity & { + agent?: { name: AgentName[] }; + service?: { environment?: string }; + }; +} + +export const isBuiltinEntityOfType = ( + type: T, + entity: InventoryEntity +): entity is BuiltinEntityMap[T] => { + return entity.entityType === type; +}; diff --git a/x-pack/plugins/observability_solution/inventory/common/utils/unflatten_entity.ts b/x-pack/plugins/observability_solution/inventory/common/utils/unflatten_entity.ts deleted file mode 100644 index 758d185a5753b..0000000000000 --- a/x-pack/plugins/observability_solution/inventory/common/utils/unflatten_entity.ts +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { unflattenObject } from '@kbn/observability-utils/object/unflatten_object'; -import type { Entity, InventoryEntityLatest } from '../entities'; - -export function unflattenEntity(entity: Entity) { - return unflattenObject(entity) as InventoryEntityLatest; -} diff --git a/x-pack/plugins/observability_solution/inventory/e2e/cypress/e2e/home.cy.ts b/x-pack/plugins/observability_solution/inventory/e2e/cypress/e2e/home.cy.ts index 9c9011609740b..17b6cf502280a 100644 --- a/x-pack/plugins/observability_solution/inventory/e2e/cypress/e2e/home.cy.ts +++ b/x-pack/plugins/observability_solution/inventory/e2e/cypress/e2e/home.cy.ts @@ -169,6 +169,7 @@ describe('Home page', () => { 'entityTypeControlGroupOptions' ); cy.intercept('GET', '/internal/inventory/entities?**').as('getEntities'); + cy.intercept('GET', '/internal/inventory/entities/types').as('getEntitiesTypes'); cy.intercept('GET', '/internal/inventory/entities/group_by/**').as('getGroups'); cy.visitKibana('/app/inventory'); cy.wait('@getEEMStatus'); @@ -181,8 +182,6 @@ describe('Home page', () => { cy.get('server1').should('not.exist'); cy.contains('synth-node-trace-logs'); cy.contains('foo').should('not.exist'); - cy.getByTestSubj('inventoryGroup_entity.type_host').should('not.exist'); - cy.getByTestSubj('inventoryGroup_entity.type_container').should('not.exist'); }); it('Filters entities by host type', () => { @@ -193,6 +192,7 @@ describe('Home page', () => { 'entityTypeControlGroupOptions' ); cy.intercept('GET', '/internal/inventory/entities?**').as('getEntities'); + cy.intercept('GET', '/internal/inventory/entities/types').as('getEntitiesTypes'); cy.intercept('GET', '/internal/inventory/entities/group_by/**').as('getGroups'); cy.visitKibana('/app/inventory'); cy.wait('@getEEMStatus'); @@ -205,8 +205,6 @@ describe('Home page', () => { cy.contains('server1'); cy.contains('synth-node-trace-logs').should('not.exist'); cy.contains('foo').should('not.exist'); - cy.getByTestSubj('inventoryGroup_entity.type_service').should('not.exist'); - cy.getByTestSubj('inventoryGroup_entity.type_container').should('not.exist'); }); it('Filters entities by container type', () => { @@ -217,6 +215,7 @@ describe('Home page', () => { 'entityTypeControlGroupOptions' ); cy.intercept('GET', '/internal/inventory/entities?**').as('getEntities'); + cy.intercept('GET', '/internal/inventory/entities/types').as('getEntitiesTypes'); cy.intercept('GET', '/internal/inventory/entities/group_by/**').as('getGroups'); cy.visitKibana('/app/inventory'); cy.wait('@getEEMStatus'); @@ -229,8 +228,6 @@ describe('Home page', () => { cy.contains('server1').should('not.exist'); cy.contains('synth-node-trace-logs').should('not.exist'); cy.contains('foo'); - cy.getByTestSubj('inventoryGroup_entity.type_host').should('not.exist'); - cy.getByTestSubj('inventoryGroup_entity.type_service').should('not.exist'); }); it('Navigates to discover with actions button in the entities list', () => { diff --git a/x-pack/plugins/observability_solution/inventory/public/components/alerts_badge/alerts_badge.test.tsx b/x-pack/plugins/observability_solution/inventory/public/components/alerts_badge/alerts_badge.test.tsx index b5244cb29f7fc..5195a35b93f4e 100644 --- a/x-pack/plugins/observability_solution/inventory/public/components/alerts_badge/alerts_badge.test.tsx +++ b/x-pack/plugins/observability_solution/inventory/public/components/alerts_badge/alerts_badge.test.tsx @@ -8,11 +8,16 @@ import React from 'react'; import { render, screen } from '@testing-library/react'; import { AlertsBadge } from './alerts_badge'; import { useKibana } from '../../hooks/use_kibana'; -import type { Entity } from '../../../common/entities'; +import type { InventoryEntity } from '../../../common/entities'; jest.mock('../../hooks/use_kibana'); const useKibanaMock = useKibana as jest.Mock; +const commonEntityFields: Partial = { + entityLastSeenTimestamp: 'foo', + entityId: '1', +}; + describe('AlertsBadge', () => { const mockAsKqlFilter = jest.fn(); @@ -40,16 +45,19 @@ describe('AlertsBadge', () => { }); it('render alerts badge for a host entity', () => { - const entity: Entity = { - 'entity.last_seen_timestamp': 'foo', - 'entity.id': '1', - 'entity.type': 'host', - 'entity.display_name': 'foo', - 'entity.identity_fields': 'host.name', - 'host.name': 'foo', - 'entity.definition_id': 'host', - 'cloud.provider': null, + const entity: InventoryEntity = { + ...(commonEntityFields as InventoryEntity), + entityType: 'host', + entityDisplayName: 'foo', + entityIdentityFields: 'host.name', + entityDefinitionId: 'host', alertsCount: 1, + host: { + name: 'foo', + }, + cloud: { + provider: null, + }, }; mockAsKqlFilter.mockReturnValue('host.name: foo'); @@ -60,16 +68,22 @@ describe('AlertsBadge', () => { expect(screen.queryByTestId('inventoryAlertsBadgeLink')?.textContent).toEqual('1'); }); it('render alerts badge for a service entity', () => { - const entity: Entity = { - 'entity.last_seen_timestamp': 'foo', - 'agent.name': 'node', - 'entity.id': '1', - 'entity.type': 'service', - 'entity.display_name': 'foo', - 'entity.identity_fields': 'service.name', - 'service.name': 'bar', - 'entity.definition_id': 'host', - 'cloud.provider': null, + const entity: InventoryEntity = { + ...(commonEntityFields as InventoryEntity), + entityType: 'service', + entityDisplayName: 'foo', + entityIdentityFields: 'service.name', + entityDefinitionId: 'service', + service: { + name: 'bar', + }, + agent: { + name: 'node', + }, + cloud: { + provider: null, + }, + alertsCount: 5, }; mockAsKqlFilter.mockReturnValue('service.name: bar'); @@ -81,17 +95,22 @@ describe('AlertsBadge', () => { expect(screen.queryByTestId('inventoryAlertsBadgeLink')?.textContent).toEqual('5'); }); it('render alerts badge for a service entity with multiple identity fields', () => { - const entity: Entity = { - 'entity.last_seen_timestamp': 'foo', - 'agent.name': 'node', - 'entity.id': '1', - 'entity.type': 'service', - 'entity.display_name': 'foo', - 'entity.identity_fields': ['service.name', 'service.environment'], - 'service.name': 'bar', - 'service.environment': 'prod', - 'entity.definition_id': 'host', - 'cloud.provider': null, + const entity: InventoryEntity = { + ...(commonEntityFields as InventoryEntity), + entityType: 'service', + entityDisplayName: 'foo', + entityIdentityFields: ['service.name', 'service.environment'], + entityDefinitionId: 'service', + service: { + name: 'bar', + environment: 'prod', + }, + agent: { + name: 'node', + }, + cloud: { + provider: null, + }, alertsCount: 2, }; diff --git a/x-pack/plugins/observability_solution/inventory/public/components/alerts_badge/alerts_badge.tsx b/x-pack/plugins/observability_solution/inventory/public/components/alerts_badge/alerts_badge.tsx index a5845a7b42dcf..ed873bdb68c21 100644 --- a/x-pack/plugins/observability_solution/inventory/public/components/alerts_badge/alerts_badge.tsx +++ b/x-pack/plugins/observability_solution/inventory/public/components/alerts_badge/alerts_badge.tsx @@ -8,11 +8,10 @@ import React from 'react'; import rison from '@kbn/rison'; import { EuiBadge, EuiToolTip } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import type { Entity } from '../../../common/entities'; -import { unflattenEntity } from '../../../common/utils/unflatten_entity'; +import type { InventoryEntity } from '../../../common/entities'; import { useKibana } from '../../hooks/use_kibana'; -export function AlertsBadge({ entity }: { entity: Entity }) { +export function AlertsBadge({ entity }: { entity: InventoryEntity }) { const { services: { http: { basePath }, @@ -22,7 +21,12 @@ export function AlertsBadge({ entity }: { entity: Entity }) { const activeAlertsHref = basePath.prepend( `/app/observability/alerts?_a=${rison.encode({ - kuery: entityManager.entityClient.asKqlFilter(unflattenEntity(entity)), + kuery: entityManager.entityClient.asKqlFilter({ + entity: { + identity_fields: entity.entityIdentityFields, + }, + ...entity, + }), status: 'active', })}` ); diff --git a/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/entities_grid.stories.tsx b/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/entities_grid.stories.tsx index a3f2834934cd8..ae80bf09ecae2 100644 --- a/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/entities_grid.stories.tsx +++ b/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/entities_grid.stories.tsx @@ -9,7 +9,7 @@ import { EuiButton, EuiDataGridSorting, EuiFlexGroup, EuiFlexItem } from '@elast import { Meta, Story } from '@storybook/react'; import { orderBy } from 'lodash'; import React, { useMemo, useState } from 'react'; -import { ENTITY_LAST_SEEN, ENTITY_TYPE } from '@kbn/observability-shared-plugin/common'; +import { ENTITY_LAST_SEEN } from '@kbn/observability-shared-plugin/common'; import { useArgs } from '@storybook/addons'; import { EntitiesGrid } from '.'; import { entitiesMock } from './mock/entities_mock'; @@ -45,7 +45,7 @@ export const Grid: Story = (args) => { const filteredAndSortedItems = useMemo( () => orderBy( - entityType ? entitiesMock.filter((mock) => mock[ENTITY_TYPE] === entityType) : entitiesMock, + entityType ? entitiesMock.filter((mock) => mock.entityType === entityType) : entitiesMock, sort.id, sort.direction ), diff --git a/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/entity_name/entity_name.test.tsx b/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/entity_name/entity_name.test.tsx index d5d08ed415a40..29a862646c4c4 100644 --- a/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/entity_name/entity_name.test.tsx +++ b/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/entity_name/entity_name.test.tsx @@ -9,28 +9,22 @@ import React from 'react'; import { render, screen } from '@testing-library/react'; import { EntityName } from '.'; import { useDetailViewRedirect } from '../../../hooks/use_detail_view_redirect'; -import { Entity } from '../../../../common/entities'; -import { - ENTITY_DEFINITION_ID, - ENTITY_DISPLAY_NAME, - ENTITY_ID, - ENTITY_IDENTITY_FIELDS, - ENTITY_LAST_SEEN, - ENTITY_TYPE, -} from '@kbn/observability-shared-plugin/common'; +import type { InventoryEntity } from '../../../../common/entities'; jest.mock('../../../hooks/use_detail_view_redirect'); const useDetailViewRedirectMock = useDetailViewRedirect as jest.Mock; describe('EntityName', () => { - const mockEntity: Entity = { - [ENTITY_LAST_SEEN]: '2023-10-09T00:00:00Z', - [ENTITY_ID]: '1', - [ENTITY_DISPLAY_NAME]: 'entity_name', - [ENTITY_DEFINITION_ID]: 'entity_definition_id', - [ENTITY_IDENTITY_FIELDS]: ['service.name', 'service.environment'], - [ENTITY_TYPE]: 'service', + const mockEntity: InventoryEntity = { + entityLastSeenTimestamp: '2023-10-09T00:00:00Z', + entityId: '1', + entityType: 'service', + entityDisplayName: 'entity_name', + entityIdentityFields: ['service.name', 'service.environment'], + entityDefinitionId: 'entity_definition_id', + entitySchemaVersion: '1', + entityDefinitionVersion: '1', }; beforeEach(() => { diff --git a/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/entity_name/index.tsx b/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/entity_name/index.tsx index e8db7013f8cb3..6117f6e428bde 100644 --- a/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/entity_name/index.tsx +++ b/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/entity_name/index.tsx @@ -7,14 +7,13 @@ import { EuiFlexGroup, EuiFlexItem, EuiLink } from '@elastic/eui'; import React, { useCallback } from 'react'; -import { ENTITY_DISPLAY_NAME } from '@kbn/observability-shared-plugin/common'; import { useKibana } from '../../../hooks/use_kibana'; -import type { Entity } from '../../../../common/entities'; +import type { InventoryEntity } from '../../../../common/entities'; import { EntityIcon } from '../../entity_icon'; import { useDetailViewRedirect } from '../../../hooks/use_detail_view_redirect'; interface EntityNameProps { - entity: Entity; + entity: InventoryEntity; } export function EntityName({ entity }: EntityNameProps) { @@ -29,7 +28,7 @@ export function EntityName({ entity }: EntityNameProps) { const handleLinkClick = useCallback(() => { telemetry.reportEntityViewClicked({ view_type: 'detail', - entity_type: entity['entity.type'], + entity_type: entity.entityType, }); }, [entity, telemetry]); @@ -40,7 +39,7 @@ export function EntityName({ entity }: EntityNameProps) { - {entity[ENTITY_DISPLAY_NAME]} + {entity.entityDisplayName} diff --git a/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/grid_columns.tsx b/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/grid_columns.tsx index d514dc9199aec..be5c50eba9c07 100644 --- a/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/grid_columns.tsx +++ b/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/grid_columns.tsx @@ -8,11 +8,6 @@ import { EuiButtonIcon, EuiDataGridColumn, EuiToolTip } from '@elastic/eui'; import React from 'react'; import { i18n } from '@kbn/i18n'; -import { - ENTITY_DISPLAY_NAME, - ENTITY_LAST_SEEN, - ENTITY_TYPE, -} from '@kbn/observability-shared-plugin/common'; const alertsLabel = i18n.translate('xpack.inventory.entitiesGrid.euiDataGrid.alertsLabel', { defaultMessage: 'Alerts', @@ -76,12 +71,12 @@ export const getColumns = ({ }: { showAlertsColumn: boolean; showActions: boolean; -}): EuiDataGridColumn[] => { +}) => { return [ ...(showAlertsColumn ? [ { - id: 'alertsCount', + id: 'alertsCount' as const, displayAsText: alertsLabel, isSortable: true, display: , @@ -91,21 +86,21 @@ export const getColumns = ({ ] : []), { - id: ENTITY_DISPLAY_NAME, + id: 'entityDisplayName' as const, // keep it for accessibility purposes displayAsText: entityNameLabel, display: , isSortable: true, }, { - id: ENTITY_TYPE, + id: 'entityType' as const, // keep it for accessibility purposes displayAsText: entityTypeLabel, display: , isSortable: true, }, { - id: ENTITY_LAST_SEEN, + id: 'entityLastSeenTimestamp' as const, // keep it for accessibility purposes displayAsText: entityLastSeenLabel, display: ( @@ -118,7 +113,7 @@ export const getColumns = ({ ...(showActions ? [ { - id: 'actions', + id: 'actions' as const, // keep it for accessibility purposes displayAsText: entityActionsLabel, display: ( @@ -128,5 +123,5 @@ export const getColumns = ({ }, ] : []), - ]; + ] satisfies EuiDataGridColumn[]; }; diff --git a/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/index.tsx b/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/index.tsx index 7ca29f7820332..ff4329955773d 100644 --- a/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/index.tsx +++ b/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/index.tsx @@ -15,13 +15,8 @@ import { i18n } from '@kbn/i18n'; import { FormattedDate, FormattedMessage, FormattedTime } from '@kbn/i18n-react'; import { last } from 'lodash'; import React, { useCallback, useMemo } from 'react'; -import { - ENTITY_DISPLAY_NAME, - ENTITY_LAST_SEEN, - ENTITY_TYPE, -} from '@kbn/observability-shared-plugin/common'; -import { EntityColumnIds } from '../../../common/entities'; -import { APIReturnType } from '../../api'; +import { ENTITY_TYPE } from '@kbn/observability-shared-plugin/common'; +import { EntityColumnIds, InventoryEntity } from '../../../common/entities'; import { BadgeFilterWithPopover } from '../badge_filter_with_popover'; import { getColumns } from './grid_columns'; import { AlertsBadge } from '../alerts_badge/alerts_badge'; @@ -29,12 +24,9 @@ import { EntityName } from './entity_name'; import { EntityActions } from '../entity_actions'; import { useDiscoverRedirect } from '../../hooks/use_discover_redirect'; -type InventoryEntitiesAPIReturnType = APIReturnType<'GET /internal/inventory/entities'>; -type LatestEntities = InventoryEntitiesAPIReturnType['entities']; - interface Props { loading: boolean; - entities: LatestEntities; + entities: InventoryEntity[]; sortDirection: 'asc' | 'desc'; sortField: string; pageIndex: number; @@ -88,16 +80,17 @@ export function EntitiesGrid({ } const columnEntityTableId = columnId as EntityColumnIds; - const entityType = entity[ENTITY_TYPE]; + const entityType = entity.entityType; const discoverUrl = getDiscoverRedirectUrl(entity); switch (columnEntityTableId) { case 'alertsCount': return entity?.alertsCount ? : null; - case ENTITY_TYPE: + case 'entityType': return ; - case ENTITY_LAST_SEEN: + + case 'entityLastSeenTimestamp': return ( ); - case ENTITY_DISPLAY_NAME: + case 'entityDisplayName': return ; case 'actions': return ( discoverUrl && ( ) ); default: - return entity[columnId as EntityColumnIds] || ''; + return null; } }, [entities, getDiscoverRedirectUrl] diff --git a/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/mock/entities_mock.ts b/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/mock/entities_mock.ts index 3b7e7afcadb99..1048b18f82e91 100644 --- a/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/mock/entities_mock.ts +++ b/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/mock/entities_mock.ts @@ -6,15 +6,7 @@ */ import { faker } from '@faker-js/faker'; -import { - ENTITY_DISPLAY_NAME, - ENTITY_TYPE, - ENTITY_ID, - ENTITY_LAST_SEEN, - AGENT_NAME, - CLOUD_PROVIDER, -} from '@kbn/observability-shared-plugin/common'; -import { Entity } from '../../../../common/entities'; +import type { InventoryEntity } from '../../../../common/entities'; const idGenerator = () => { let id = 0; @@ -33,38 +25,48 @@ function generateRandomTimestamp() { return randomDate.toISOString(); } -const getEntity = (entityType: string, customFields: Record = {}) => ({ - [ENTITY_LAST_SEEN]: generateRandomTimestamp(), - [ENTITY_TYPE]: entityType, - [ENTITY_DISPLAY_NAME]: faker.person.fullName(), - [ENTITY_ID]: generateId(), - ...customFields, +const indentityFieldsPerType: Record = { + host: ['host.name'], + container: ['container.id'], + service: ['service.name'], +}; + +const getEntityLatest = ( + entityType: string, + overrides?: Partial +): InventoryEntity => ({ + entityLastSeenTimestamp: generateRandomTimestamp(), + entityType, + entityDisplayName: faker.person.fullName(), + entityId: generateId(), + entityDefinitionId: faker.string.uuid(), + entityDefinitionVersion: '1.0.0', + entityIdentityFields: indentityFieldsPerType[entityType], + entitySchemaVersion: '1.0.0', + ...overrides, }); -const alertsMock = [ - { - ...getEntity('host'), - alertsCount: 3, - }, - { - ...getEntity('service'), +const alertsMock: InventoryEntity[] = [ + getEntityLatest('host', { + alertsCount: 1, + }), + getEntityLatest('service', { alertsCount: 3, - }, - - { - ...getEntity('host'), + }), + getEntityLatest('host', { alertsCount: 10, - }, - { - ...getEntity('host'), + }), + getEntityLatest('host', { alertsCount: 1, - }, + }), ]; -const hostsMock = Array.from({ length: 20 }, () => getEntity('host', { [CLOUD_PROVIDER]: 'gcp' })); -const containersMock = Array.from({ length: 20 }, () => getEntity('container')); +const hostsMock = Array.from({ length: 20 }, () => + getEntityLatest('host', { cloud: { provider: 'gcp' } }) +); +const containersMock = Array.from({ length: 20 }, () => getEntityLatest('container')); const servicesMock = Array.from({ length: 20 }, () => - getEntity('service', { [AGENT_NAME]: 'java' }) + getEntityLatest('service', { agent: { name: 'java' } }) ); export const entitiesMock = [ @@ -72,4 +74,4 @@ export const entitiesMock = [ ...hostsMock, ...containersMock, ...servicesMock, -] as Entity[]; +] as InventoryEntity[]; diff --git a/x-pack/plugins/observability_solution/inventory/public/components/entity_icon/index.tsx b/x-pack/plugins/observability_solution/inventory/public/components/entity_icon/index.tsx index 48b21779d2e38..4da8fd3103c41 100644 --- a/x-pack/plugins/observability_solution/inventory/public/components/entity_icon/index.tsx +++ b/x-pack/plugins/observability_solution/inventory/public/components/entity_icon/index.tsx @@ -6,36 +6,23 @@ */ import React from 'react'; -import { - AGENT_NAME, - CLOUD_PROVIDER, - ENTITY_TYPE, - ENTITY_TYPES, -} from '@kbn/observability-shared-plugin/common'; import { type CloudProvider, CloudProviderIcon, AgentIcon } from '@kbn/custom-icons'; import { EuiFlexGroup, EuiFlexItem, EuiIcon } from '@elastic/eui'; -import type { AgentName } from '@kbn/elastic-agent-utils'; import { euiThemeVars } from '@kbn/ui-theme'; -import type { Entity } from '../../../common/entities'; +import { castArray } from 'lodash'; +import type { InventoryEntity } from '../../../common/entities'; +import { isBuiltinEntityOfType } from '../../../common/utils/entity_type_guards'; interface EntityIconProps { - entity: Entity; + entity: InventoryEntity; } -type NotNullableCloudProvider = Exclude; - -const getSingleValue = (value?: T | T[] | null): T | undefined => { - return value == null ? undefined : Array.isArray(value) ? value[0] : value; -}; - export function EntityIcon({ entity }: EntityIconProps) { - const entityType = entity[ENTITY_TYPE]; const defaultIconSize = euiThemeVars.euiSizeL; - if (entityType === ENTITY_TYPES.HOST || entityType === ENTITY_TYPES.CONTAINER) { - const cloudProvider = getSingleValue( - entity[CLOUD_PROVIDER] as NotNullableCloudProvider | NotNullableCloudProvider[] - ); + if (isBuiltinEntityOfType('host', entity) || isBuiltinEntityOfType('container', entity)) { + const cloudProvider = castArray(entity.cloud?.provider)[0]; + return ( ; + if (isBuiltinEntityOfType('service', entity)) { + return ; } - if (entityType.startsWith('kubernetes')) { + if (entity.entityType.startsWith('kubernetes')) { return ; } diff --git a/x-pack/plugins/observability_solution/inventory/public/components/grouped_inventory/index.tsx b/x-pack/plugins/observability_solution/inventory/public/components/grouped_inventory/index.tsx index b939f0fa5c423..0964b7bb39465 100644 --- a/x-pack/plugins/observability_solution/inventory/public/components/grouped_inventory/index.tsx +++ b/x-pack/plugins/observability_solution/inventory/public/components/grouped_inventory/index.tsx @@ -8,6 +8,7 @@ import { EuiSpacer } from '@elastic/eui'; import { ENTITY_TYPE } from '@kbn/observability-shared-plugin/common'; import React from 'react'; import useEffectOnce from 'react-use/lib/useEffectOnce'; +import { flattenObject } from '@kbn/observability-utils/object/flatten_object'; import { useInventoryAbortableAsync } from '../../hooks/use_inventory_abortable_async'; import { useKibana } from '../../hooks/use_kibana'; import { useUnifiedSearchContext } from '../../hooks/use_unified_search_context'; @@ -52,15 +53,18 @@ export function GroupedInventory() { <> - {value.groups.map((group) => ( - - ))} + {value.groups.map((group) => { + const groupValue = flattenObject(group)[value.groupBy]; + return ( + + ); + })} ); } diff --git a/x-pack/plugins/observability_solution/inventory/public/hooks/use_detail_view_redirect.test.ts b/x-pack/plugins/observability_solution/inventory/public/hooks/use_detail_view_redirect.test.ts index cf4993f871880..233c1a1076b79 100644 --- a/x-pack/plugins/observability_solution/inventory/public/hooks/use_detail_view_redirect.test.ts +++ b/x-pack/plugins/observability_solution/inventory/public/hooks/use_detail_view_redirect.test.ts @@ -9,34 +9,24 @@ import { renderHook } from '@testing-library/react-hooks'; import { useDetailViewRedirect } from './use_detail_view_redirect'; import { useKibana } from './use_kibana'; import { - AGENT_NAME, - CLOUD_PROVIDER, CONTAINER_ID, - ENTITY_DEFINITION_ID, - ENTITY_DISPLAY_NAME, - ENTITY_ID, - ENTITY_IDENTITY_FIELDS, - ENTITY_LAST_SEEN, - ENTITY_TYPE, - HOST_NAME, ENTITY_TYPES, - SERVICE_ENVIRONMENT, + HOST_NAME, SERVICE_NAME, } from '@kbn/observability-shared-plugin/common'; -import { unflattenEntity } from '../../common/utils/unflatten_entity'; -import type { Entity } from '../../common/entities'; +import type { InventoryEntity } from '../../common/entities'; jest.mock('./use_kibana'); -jest.mock('../../common/utils/unflatten_entity'); const useKibanaMock = useKibana as jest.Mock; -const unflattenEntityMock = unflattenEntity as jest.Mock; -const commonEntityFields: Partial = { - [ENTITY_LAST_SEEN]: '2023-10-09T00:00:00Z', - [ENTITY_ID]: '1', - [ENTITY_DISPLAY_NAME]: 'entity_name', - [ENTITY_DEFINITION_ID]: 'entity_definition_id', +const commonEntityFields: Partial = { + entityLastSeenTimestamp: '2023-10-09T00:00:00Z', + entityId: '1', + entityDisplayName: 'entity_name', + entityDefinitionId: 'entity_definition_id', + entityDefinitionVersion: '1', + entitySchemaVersion: '1', }; describe('useDetailViewRedirect', () => { @@ -66,17 +56,19 @@ describe('useDetailViewRedirect', () => { }, }, }); - - unflattenEntityMock.mockImplementation((entity) => entity); }); it('getEntityRedirectUrl should return the correct URL for host entity', () => { - const entity: Entity = { - ...(commonEntityFields as Entity), - [ENTITY_IDENTITY_FIELDS]: [HOST_NAME], - [ENTITY_TYPE]: 'host', - [HOST_NAME]: 'host-1', - [CLOUD_PROVIDER]: null, + const entity: InventoryEntity = { + ...(commonEntityFields as InventoryEntity), + entityType: 'host', + entityIdentityFields: ['host.name'], + host: { + name: 'host-1', + }, + cloud: { + provider: null, + }, }; mockGetIdentityFieldsValue.mockReturnValue({ [HOST_NAME]: 'host-1' }); @@ -90,12 +82,16 @@ describe('useDetailViewRedirect', () => { }); it('getEntityRedirectUrl should return the correct URL for container entity', () => { - const entity: Entity = { - ...(commonEntityFields as Entity), - [ENTITY_IDENTITY_FIELDS]: [CONTAINER_ID], - [ENTITY_TYPE]: 'container', - [CONTAINER_ID]: 'container-1', - [CLOUD_PROVIDER]: null, + const entity: InventoryEntity = { + ...(commonEntityFields as InventoryEntity), + entityType: 'container', + entityIdentityFields: ['container.id'], + container: { + id: 'container-1', + }, + cloud: { + provider: null, + }, }; mockGetIdentityFieldsValue.mockReturnValue({ [CONTAINER_ID]: 'container-1' }); @@ -112,13 +108,17 @@ describe('useDetailViewRedirect', () => { }); it('getEntityRedirectUrl should return the correct URL for service entity', () => { - const entity: Entity = { - ...(commonEntityFields as Entity), - [ENTITY_IDENTITY_FIELDS]: [SERVICE_NAME], - [ENTITY_TYPE]: 'service', - [SERVICE_NAME]: 'service-1', - [SERVICE_ENVIRONMENT]: 'prod', - [AGENT_NAME]: 'node', + const entity: InventoryEntity = { + ...(commonEntityFields as InventoryEntity), + entityType: 'service', + entityIdentityFields: ['service.name'], + agent: { + name: 'node', + }, + service: { + name: 'service-1', + environment: 'prod', + }, }; mockGetIdentityFieldsValue.mockReturnValue({ [SERVICE_NAME]: 'service-1' }); mockGetRedirectUrl.mockReturnValue('service-overview-url'); @@ -145,10 +145,13 @@ describe('useDetailViewRedirect', () => { [ENTITY_TYPES.KUBERNETES.STATEFULSET.ecs, 'kubernetes-21694370-bcb2-11ec-b64f-7dd6e8e82013'], ].forEach(([entityType, dashboardId]) => { it(`getEntityRedirectUrl should return the correct URL for ${entityType} entity`, () => { - const entity: Entity = { - ...(commonEntityFields as Entity), - [ENTITY_IDENTITY_FIELDS]: ['some.field'], - [ENTITY_TYPE]: entityType, + const entity: InventoryEntity = { + ...(commonEntityFields as InventoryEntity), + entityType, + entityIdentityFields: ['some.field'], + some: { + field: 'some-value', + }, }; mockAsKqlFilter.mockReturnValue('kql-query'); diff --git a/x-pack/plugins/observability_solution/inventory/public/hooks/use_detail_view_redirect.ts b/x-pack/plugins/observability_solution/inventory/public/hooks/use_detail_view_redirect.ts index 23380dc3704de..4df4fa4ca1f96 100644 --- a/x-pack/plugins/observability_solution/inventory/public/hooks/use_detail_view_redirect.ts +++ b/x-pack/plugins/observability_solution/inventory/public/hooks/use_detail_view_redirect.ts @@ -6,20 +6,17 @@ */ import { ASSET_DETAILS_LOCATOR_ID, - AssetDetailsLocatorParams, - ENTITY_IDENTITY_FIELDS, - ENTITY_TYPE, ENTITY_TYPES, - SERVICE_ENVIRONMENT, SERVICE_OVERVIEW_LOCATOR_ID, - ServiceOverviewParams, + type AssetDetailsLocatorParams, + type ServiceOverviewParams, } from '@kbn/observability-shared-plugin/common'; import { useCallback } from 'react'; -import { DashboardLocatorParams } from '@kbn/dashboard-plugin/public'; +import type { DashboardLocatorParams } from '@kbn/dashboard-plugin/public'; import { DASHBOARD_APP_LOCATOR } from '@kbn/deeplinks-analytics'; import { castArray } from 'lodash'; -import type { Entity } from '../../common/entities'; -import { unflattenEntity } from '../../common/utils/unflatten_entity'; +import { isBuiltinEntityOfType } from '../../common/utils/entity_type_guards'; +import type { InventoryEntity } from '../../common/entities'; import { useKibana } from './use_kibana'; const KUBERNETES_DASHBOARDS_IDS: Record = { @@ -44,52 +41,38 @@ export const useDetailViewRedirect = () => { const dashboardLocator = locators.get(DASHBOARD_APP_LOCATOR); const serviceOverviewLocator = locators.get(SERVICE_OVERVIEW_LOCATOR_ID); - const getSingleIdentityFieldValue = useCallback( - (entity: Entity) => { - const identityFields = castArray(entity[ENTITY_IDENTITY_FIELDS]); - if (identityFields.length > 1) { - throw new Error(`Multiple identity fields are not supported for ${entity[ENTITY_TYPE]}`); - } - - const identityField = identityFields[0]; - return entityManager.entityClient.getIdentityFieldsValue(unflattenEntity(entity))[ - identityField - ]; - }, - [entityManager.entityClient] - ); - const getDetailViewRedirectUrl = useCallback( - (entity: Entity) => { - const type = entity[ENTITY_TYPE]; - const identityValue = getSingleIdentityFieldValue(entity); - - switch (type) { - case ENTITY_TYPES.HOST: - case ENTITY_TYPES.CONTAINER: - return assetDetailsLocator?.getRedirectUrl({ - assetId: identityValue, - assetType: type, - }); + (entity: InventoryEntity) => { + const identityFieldsValue = entityManager.entityClient.getIdentityFieldsValue({ + entity: { + identity_fields: entity.entityIdentityFields, + }, + ...entity, + }); + const identityFields = castArray(entity.entityIdentityFields); - case 'service': - return serviceOverviewLocator?.getRedirectUrl({ - serviceName: identityValue, - // service.environemnt is not part of entity.identityFields - // we need to manually get its value - environment: [entity[SERVICE_ENVIRONMENT] || undefined].flat()[0], - }); + if (isBuiltinEntityOfType('host', entity) || isBuiltinEntityOfType('container', entity)) { + return assetDetailsLocator?.getRedirectUrl({ + assetId: identityFieldsValue[identityFields[0]], + assetType: entity.entityType, + }); + } - default: - return undefined; + if (isBuiltinEntityOfType('service', entity)) { + return serviceOverviewLocator?.getRedirectUrl({ + serviceName: identityFieldsValue[identityFields[0]], + environment: entity.service?.environment, + }); } + + return undefined; }, - [assetDetailsLocator, getSingleIdentityFieldValue, serviceOverviewLocator] + [assetDetailsLocator, entityManager.entityClient, serviceOverviewLocator] ); const getDashboardRedirectUrl = useCallback( - (entity: Entity) => { - const type = entity[ENTITY_TYPE]; + (entity: InventoryEntity) => { + const type = entity.entityType; const dashboardId = KUBERNETES_DASHBOARDS_IDS[type]; return dashboardId @@ -97,7 +80,12 @@ export const useDetailViewRedirect = () => { dashboardId, query: { language: 'kuery', - query: entityManager.entityClient.asKqlFilter(unflattenEntity(entity)), + query: entityManager.entityClient.asKqlFilter({ + entity: { + identity_fields: entity.entityIdentityFields, + }, + ...entity, + }), }, }) : undefined; @@ -106,7 +94,8 @@ export const useDetailViewRedirect = () => { ); const getEntityRedirectUrl = useCallback( - (entity: Entity) => getDetailViewRedirectUrl(entity) ?? getDashboardRedirectUrl(entity), + (entity: InventoryEntity) => + getDetailViewRedirectUrl(entity) ?? getDashboardRedirectUrl(entity), [getDashboardRedirectUrl, getDetailViewRedirectUrl] ); diff --git a/x-pack/plugins/observability_solution/inventory/public/hooks/use_discover_redirect.ts b/x-pack/plugins/observability_solution/inventory/public/hooks/use_discover_redirect.ts index c29caca7e5b77..33758c9df449d 100644 --- a/x-pack/plugins/observability_solution/inventory/public/hooks/use_discover_redirect.ts +++ b/x-pack/plugins/observability_solution/inventory/public/hooks/use_discover_redirect.ts @@ -11,12 +11,11 @@ import { ENTITY_TYPE, } from '@kbn/observability-shared-plugin/common'; import { useCallback } from 'react'; -import type { Entity, EntityColumnIds } from '../../common/entities'; -import { unflattenEntity } from '../../common/utils/unflatten_entity'; +import type { InventoryEntity } from '../../common/entities'; import { useKibana } from './use_kibana'; import { useUnifiedSearchContext } from './use_unified_search_context'; -const ACTIVE_COLUMNS: EntityColumnIds[] = [ENTITY_DISPLAY_NAME, ENTITY_TYPE, ENTITY_LAST_SEEN]; +const ACTIVE_COLUMNS = [ENTITY_DISPLAY_NAME, ENTITY_TYPE, ENTITY_LAST_SEEN]; export const useDiscoverRedirect = () => { const { @@ -31,9 +30,14 @@ export const useDiscoverRedirect = () => { const discoverLocator = share.url.locators.get('DISCOVER_APP_LOCATOR'); const getDiscoverEntitiesRedirectUrl = useCallback( - (entity?: Entity) => { + (entity?: InventoryEntity) => { const entityKqlFilter = entity - ? entityManager.entityClient.asKqlFilter(unflattenEntity(entity)) + ? entityManager.entityClient.asKqlFilter({ + entity: { + identity_fields: entity.entityIdentityFields, + }, + ...entity, + }) : ''; const kueryWithEntityDefinitionFilters = [ @@ -65,7 +69,7 @@ export const useDiscoverRedirect = () => { ); const getDiscoverRedirectUrl = useCallback( - (entity?: Entity) => getDiscoverEntitiesRedirectUrl(entity), + (entity?: InventoryEntity) => getDiscoverEntitiesRedirectUrl(entity), [getDiscoverEntitiesRedirectUrl] ); diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts index 8c72e18bc0740..bab4af50e316e 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts @@ -7,7 +7,6 @@ import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; -import { esqlResultToPlainObjects } from '@kbn/observability-utils/es/utils/esql_result_to_plain_objects'; import { ENTITIES_LATEST_ALIAS, type EntityGroup, @@ -32,10 +31,8 @@ export async function getEntityGroupsBy({ const limit = `LIMIT ${MAX_NUMBER_OF_ENTITIES}`; const query = [from, ...where, group, sort, limit].join(' | '); - const groups = await inventoryEsClient.esql('get_entities_groups', { + return inventoryEsClient.esql('get_entities_groups', { query, filter: esQuery, }); - - return esqlResultToPlainObjects(groups); } diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_types.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_types.ts index 2dfc9b8ccfdf3..99b8829b600b2 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_types.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_types.ts @@ -7,6 +7,7 @@ import { type ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; import { ENTITY_TYPE } from '@kbn/observability-shared-plugin/common'; +import type { EntityInstance } from '@kbn/entities-schema'; import { ENTITIES_LATEST_ALIAS } from '../../../common/entities'; import { getBuiltinEntityDefinitionIdESQLWhereClause } from './query_helper'; @@ -15,12 +16,14 @@ export async function getEntityTypes({ }: { inventoryEsClient: ObservabilityElasticsearchClient; }) { - const entityTypesEsqlResponse = await inventoryEsClient.esql('get_entity_types', { + const entityTypesEsqlResponse = await inventoryEsClient.esql<{ + entity: Pick; + }>('get_entity_types', { query: `FROM ${ENTITIES_LATEST_ALIAS} | ${getBuiltinEntityDefinitionIdESQLWhereClause()} | STATS count = COUNT(${ENTITY_TYPE}) BY ${ENTITY_TYPE} `, }); - return entityTypesEsqlResponse.values.map(([_, val]) => val as string); + return entityTypesEsqlResponse.map((response) => response.entity.type); } diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_identify_fields.test.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_identify_fields.test.ts index 62d77c08fd27a..8b6b3b109352c 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_identify_fields.test.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_identify_fields.test.ts @@ -5,52 +5,60 @@ * 2.0. */ -import type { Entity } from '../../../common/entities'; -import { - ENTITY_DEFINITION_ID, - ENTITY_DISPLAY_NAME, - ENTITY_ID, - ENTITY_IDENTITY_FIELDS, - ENTITY_LAST_SEEN, -} from '@kbn/observability-shared-plugin/common'; +import type { InventoryEntity } from '../../../common/entities'; import { getIdentityFieldsPerEntityType } from './get_identity_fields_per_entity_type'; -const commonEntityFields = { - [ENTITY_LAST_SEEN]: '2023-10-09T00:00:00Z', - [ENTITY_ID]: '1', - [ENTITY_DISPLAY_NAME]: 'entity_name', - [ENTITY_DEFINITION_ID]: 'entity_definition_id', - alertCount: 3, +const commonEntityFields: Partial = { + entityLastSeenTimestamp: '2023-10-09T00:00:00Z', + entityId: '1', + entityDisplayName: 'entity_name', + entityDefinitionId: 'entity_definition_id', + alertsCount: 3, }; + describe('getIdentityFields', () => { it('should return an empty Map when no entities are provided', () => { const result = getIdentityFieldsPerEntityType([]); expect(result.size).toBe(0); }); it('should return a Map with unique entity types and their respective identity fields', () => { - const serviceEntity: Entity = { - 'agent.name': 'node', - [ENTITY_IDENTITY_FIELDS]: ['service.name', 'service.environment'], - 'service.name': 'my-service', - 'entity.type': 'service', - ...commonEntityFields, + const serviceEntity: InventoryEntity = { + ...(commonEntityFields as InventoryEntity), + entityIdentityFields: ['service.name', 'service.environment'], + entityType: 'service', + agent: { + name: 'node', + }, + service: { + name: 'my-service', + }, }; - const hostEntity: Entity = { - [ENTITY_IDENTITY_FIELDS]: ['host.name'], - 'host.name': 'my-host', - 'entity.type': 'host', - 'cloud.provider': null, - ...commonEntityFields, + const hostEntity: InventoryEntity = { + ...(commonEntityFields as InventoryEntity), + entityIdentityFields: ['host.name'], + entityType: 'host', + cloud: { + provider: null, + }, + host: { + name: 'my-host', + }, }; - const containerEntity: Entity = { - [ENTITY_IDENTITY_FIELDS]: 'container.id', - 'host.name': 'my-host', - 'entity.type': 'container', - 'cloud.provider': null, - 'container.id': '123', - ...commonEntityFields, + const containerEntity: InventoryEntity = { + ...(commonEntityFields as InventoryEntity), + entityIdentityFields: ['container.id'], + entityType: 'container', + host: { + name: 'my-host', + }, + cloud: { + provider: null, + }, + container: { + id: '123', + }, }; const mockEntities = [serviceEntity, hostEntity, containerEntity]; diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_identity_fields_per_entity_type.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_identity_fields_per_entity_type.ts index f54dc8a7f121f..06070b66bad1f 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_identity_fields_per_entity_type.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_identity_fields_per_entity_type.ts @@ -5,16 +5,16 @@ * 2.0. */ -import { ENTITY_IDENTITY_FIELDS, ENTITY_TYPE } from '@kbn/observability-shared-plugin/common'; -import { Entity } from '../../../common/entities'; +import { castArray } from 'lodash'; +import type { InventoryEntity } from '../../../common/entities'; export type IdentityFieldsPerEntityType = Map; -export const getIdentityFieldsPerEntityType = (entities: Entity[]) => { - const identityFieldsPerEntityType: IdentityFieldsPerEntityType = new Map(); +export const getIdentityFieldsPerEntityType = (latestEntities: InventoryEntity[]) => { + const identityFieldsPerEntityType = new Map(); - entities.forEach((entity) => - identityFieldsPerEntityType.set(entity[ENTITY_TYPE], [entity[ENTITY_IDENTITY_FIELDS]].flat()) + latestEntities.forEach((entity) => + identityFieldsPerEntityType.set(entity.entityType, castArray(entity.entityIdentityFields)) ); return identityFieldsPerEntityType; diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts index 402d11720a9da..7a65ac5039615 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts @@ -5,18 +5,32 @@ * 2.0. */ +import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; +import { + ENTITY_LAST_SEEN, + ENTITY_TYPE, + ENTITY_DISPLAY_NAME, +} from '@kbn/observability-shared-plugin/common'; import type { QueryDslQueryContainer, ScalarValue } from '@elastic/elasticsearch/lib/api/types'; -import { ENTITY_LAST_SEEN, ENTITY_TYPE } from '@kbn/observability-shared-plugin/common'; -import { type ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; -import { esqlResultToPlainObjects } from '@kbn/observability-utils/es/utils/esql_result_to_plain_objects'; +import type { EntityInstance } from '@kbn/entities-schema'; import { ENTITIES_LATEST_ALIAS, MAX_NUMBER_OF_ENTITIES, - type Entity, type EntityColumnIds, + InventoryEntity, } from '../../../common/entities'; import { getBuiltinEntityDefinitionIdESQLWhereClause } from './query_helper'; +type EntitySortableColumnIds = Extract< + EntityColumnIds, + 'entityLastSeenTimestamp' | 'entityDisplayName' | 'entityType' +>; +const SORT_FIELDS_TO_ES_FIELDS: Record = { + entityLastSeenTimestamp: ENTITY_LAST_SEEN, + entityDisplayName: ENTITY_DISPLAY_NAME, + entityType: ENTITY_TYPE, +} as const; + export async function getLatestEntities({ inventoryEsClient, sortDirection, @@ -29,9 +43,10 @@ export async function getLatestEntities({ sortField: EntityColumnIds; esQuery?: QueryDslQueryContainer; entityTypes?: string[]; -}) { +}): Promise { // alertsCount doesn't exist in entities index. Ignore it and sort by entity.lastSeenTimestamp by default. - const entitiesSortField = sortField === 'alertsCount' ? ENTITY_LAST_SEEN : sortField; + const entitiesSortField = + SORT_FIELDS_TO_ES_FIELDS[sortField as EntitySortableColumnIds] ?? ENTITY_LAST_SEEN; const from = `FROM ${ENTITIES_LATEST_ALIAS}`; const where: string[] = [getBuiltinEntityDefinitionIdESQLWhereClause()]; @@ -47,11 +62,28 @@ export async function getLatestEntities({ const query = [from, ...where, sort, limit].join(' | '); - const latestEntitiesEsqlResponse = await inventoryEsClient.esql('get_latest_entities', { - query, - filter: esQuery, - params, - }); + const latestEntitiesEsqlResponse = await inventoryEsClient.esql( + 'get_latest_entities', + { + query, + filter: esQuery, + params, + } + ); - return esqlResultToPlainObjects(latestEntitiesEsqlResponse); + return latestEntitiesEsqlResponse.map((lastestEntity) => { + const { entity, ...metadata } = lastestEntity; + + return { + entityId: entity.id, + entityType: entity.type, + entityDefinitionId: entity.definition_id, + entityDisplayName: entity.display_name, + entityIdentityFields: entity.identity_fields, + entityLastSeenTimestamp: entity.last_seen_timestamp, + entityDefinitionVersion: entity.definition_version, + entitySchemaVersion: entity.schema_version, + ...metadata, + }; + }); } diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities_alerts.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities_alerts.ts index 8126c69de6922..0f1ace0407233 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities_alerts.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities_alerts.ts @@ -6,7 +6,6 @@ */ import { termQuery } from '@kbn/observability-plugin/server'; -import { ENTITY_TYPE } from '@kbn/observability-shared-plugin/common'; import { ALERT_STATUS, ALERT_STATUS_ACTIVE } from '@kbn/rule-data-utils'; import { AlertsClient } from '../../lib/create_alerts_client.ts/create_alerts_client'; import { getGroupByTermsAgg } from './get_group_by_terms_agg'; @@ -25,7 +24,7 @@ export async function getLatestEntitiesAlerts({ }: { alertsClient: AlertsClient; identityFieldsPerEntityType: IdentityFieldsPerEntityType; -}): Promise> { +}): Promise> { if (identityFieldsPerEntityType.size === 0) { return []; } @@ -54,7 +53,7 @@ export async function getLatestEntitiesAlerts({ return buckets.map((bucket: Bucket) => ({ alertsCount: bucket.doc_count, - [ENTITY_TYPE]: entityType, + entityType, ...bucket.key, })); }); diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/route.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/route.ts index ae99713375b19..bdf0b1f59af01 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/route.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/route.ts @@ -11,7 +11,7 @@ import { ENTITY_TYPE } from '@kbn/observability-shared-plugin/common'; import * as t from 'io-ts'; import { orderBy } from 'lodash'; import { joinByKey } from '@kbn/observability-utils/array/join_by_key'; -import { entityColumnIdsRt, Entity } from '../../../common/entities'; +import { entityColumnIdsRt, InventoryEntity } from '../../../common/entities'; import { createInventoryServerRoute } from '../create_inventory_server_route'; import { getEntityTypes } from './get_entity_types'; import { getLatestEntities } from './get_latest_entities'; @@ -61,7 +61,7 @@ export const listLatestEntitiesRoute = createInventoryServerRoute({ logger, plugins, request, - }): Promise<{ entities: Entity[] }> => { + }): Promise<{ entities: InventoryEntity[] }> => { const coreContext = await context.core; const inventoryEsClient = createObservabilityEsClient({ client: coreContext.elasticsearch.client.asCurrentUser, @@ -90,16 +90,16 @@ export const listLatestEntitiesRoute = createInventoryServerRoute({ }); const joined = joinByKey( - [...latestEntities, ...alerts], + [...latestEntities, ...alerts] as InventoryEntity[], [...identityFieldsPerEntityType.values()].flat() - ).filter((entity) => entity['entity.id']) as Entity[]; + ).filter((latestEntity) => latestEntity.entityId); return { entities: sortField === 'alertsCount' ? orderBy( joined, - [(item: Entity) => item?.alertsCount === undefined, sortField], + [(item: InventoryEntity) => item?.alertsCount === undefined, sortField], ['asc', sortDirection] // push entities without alertsCount to the end ) : joined, diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts b/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts index c1e4a82c343b0..d328a4f3b8d6f 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts @@ -5,7 +5,6 @@ * 2.0. */ import type { Logger } from '@kbn/core/server'; -import { esqlResultToPlainObjects } from '@kbn/observability-utils/es/utils/esql_result_to_plain_objects'; import { type ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; import { getBuiltinEntityDefinitionIdESQLWhereClause } from '../entities/query_helper'; import { ENTITIES_LATEST_ALIAS } from '../../../common/entities'; @@ -18,14 +17,15 @@ export async function getHasData({ logger: Logger; }) { try { - const esqlResults = await inventoryEsClient.esql('get_has_data', { + const esqlResults = await inventoryEsClient.esql<{ _count: number }>('get_has_data', { query: `FROM ${ENTITIES_LATEST_ALIAS} | ${getBuiltinEntityDefinitionIdESQLWhereClause()} | STATS _count = COUNT(*) | LIMIT 1`, }); - const totalCount = esqlResultToPlainObjects(esqlResults)?.[0]._count ?? 0; + const totalCount = esqlResults[0]._count; + return { hasData: totalCount > 0 }; } catch (e) { logger.error(e); diff --git a/x-pack/plugins/observability_solution/inventory/tsconfig.json b/x-pack/plugins/observability_solution/inventory/tsconfig.json index 5cb95e8ac6de5..e9949e60201c8 100644 --- a/x-pack/plugins/observability_solution/inventory/tsconfig.json +++ b/x-pack/plugins/observability_solution/inventory/tsconfig.json @@ -53,7 +53,6 @@ "@kbn/spaces-plugin", "@kbn/cloud-plugin", "@kbn/storybook", - "@kbn/zod", "@kbn/dashboard-plugin", "@kbn/deeplinks-analytics", "@kbn/controls-plugin", diff --git a/x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/synthetics_common/field_selector.tsx b/x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/synthetics_common/field_selector.tsx index 5bc77b186f960..1fea21a322c72 100644 --- a/x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/synthetics_common/field_selector.tsx +++ b/x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/synthetics_common/field_selector.tsx @@ -120,7 +120,6 @@ export function FieldSelector({ ? (field.value as Array<{ value: string; label: string }>).map((value) => ({ value: value.value, label: value.label, - 'data-test-subj': `${dataTestSubj}SelectedValue`, })) : [] } diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts index 1bf3bfb5e2445..2d722e7d5076a 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts @@ -555,10 +555,15 @@ export const RuleExceptionList = z.object({ namespace_type: z.enum(['agnostic', 'single']), }); +export type AlertSuppressionDurationUnit = z.infer; +export const AlertSuppressionDurationUnit = z.enum(['s', 'm', 'h']); +export type AlertSuppressionDurationUnitEnum = typeof AlertSuppressionDurationUnit.enum; +export const AlertSuppressionDurationUnitEnum = AlertSuppressionDurationUnit.enum; + export type AlertSuppressionDuration = z.infer; export const AlertSuppressionDuration = z.object({ value: z.number().int().min(1), - unit: z.enum(['s', 'm', 'h']), + unit: AlertSuppressionDurationUnit, }); /** diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml index 088153cca4885..029a71b9e0a1c 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml @@ -581,6 +581,13 @@ components: - type - namespace_type + AlertSuppressionDurationUnit: + type: string + enum: + - s + - m + - h + AlertSuppressionDuration: type: object properties: @@ -588,11 +595,7 @@ components: type: integer minimum: 1 unit: - type: string - enum: - - s - - m - - h + $ref: '#/components/schemas/AlertSuppressionDurationUnit' required: - value - unit diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index 892b0a0226639..dc6495e1d9737 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -108,11 +108,6 @@ export const allowedExperimentalValues = Object.freeze({ */ assistantModelEvaluation: false, - /** - * Enables new Knowledge Base Entries features, introduced in `8.15.0`. - */ - assistantKnowledgeBaseByDefault: true, - /** * Enables the Managed User section inside the new user details flyout. */ diff --git a/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml b/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml index ebd4c93280090..7e8d7a61bff2c 100644 --- a/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml @@ -1560,17 +1560,19 @@ components: type: object properties: unit: - enum: - - s - - m - - h - type: string + $ref: '#/components/schemas/AlertSuppressionDurationUnit' value: minimum: 1 type: integer required: - value - unit + AlertSuppressionDurationUnit: + enum: + - s + - m + - h + type: string AlertSuppressionGroupBy: items: type: string diff --git a/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml b/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml index 30a7ae3ea343e..58456e71140a0 100644 --- a/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml @@ -850,17 +850,19 @@ components: type: object properties: unit: - enum: - - s - - m - - h - type: string + $ref: '#/components/schemas/AlertSuppressionDurationUnit' value: minimum: 1 type: integer required: - value - unit + AlertSuppressionDurationUnit: + enum: + - s + - m + - h + type: string AlertSuppressionGroupBy: items: type: string diff --git a/x-pack/plugins/security_solution/public/cloud_security_posture/components/csp_details/misconfiguration_findings_details_table.tsx b/x-pack/plugins/security_solution/public/cloud_security_posture/components/csp_details/misconfiguration_findings_details_table.tsx index 8d4088b19f9b6..69912c58e4e15 100644 --- a/x-pack/plugins/security_solution/public/cloud_security_posture/components/csp_details/misconfiguration_findings_details_table.tsx +++ b/x-pack/plugins/security_solution/public/cloud_security_posture/components/csp_details/misconfiguration_findings_details_table.tsx @@ -126,11 +126,14 @@ export const MisconfigurationFindingsDetailsTable = memo( return getNavUrlParams({ [queryField]: name }, 'configurations', ['rule.name']); }; + const linkWidth = 40; + const resultWidth = 74; + const columns: Array> = [ { field: 'rule', name: '', - width: '5%', + width: `${linkWidth}`, render: (rule: CspBenchmarkRuleMetadata, finding: MisconfigurationFindingDetailFields) => ( { const currentPlugins = nonStatefulUiPlugins.map((plugin) => plugin.name); const insightPluginWithLicense = insightMarkdownPlugin.plugin({ diff --git a/x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/insight/index.test.tsx b/x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/insight/index.test.tsx index 37d4e004edf54..026e070f320df 100644 --- a/x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/insight/index.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/insight/index.test.tsx @@ -135,7 +135,7 @@ describe('plugin', () => { }); it('show investigate message when insightsUpsellingMessage is not provided', () => { - const result = plugin({ insightsUpsellingMessage: null }); + const result = plugin({ insightsUpsellingMessage: undefined }); expect(result.button.label).toEqual('Investigate'); }); diff --git a/x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/insight/index.tsx b/x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/insight/index.tsx index e8fa43d0a189e..794b17a219208 100644 --- a/x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/insight/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/insight/index.tsx @@ -541,11 +541,7 @@ const exampleInsight = `${insightPrefix}{ ] }}`; -export const plugin = ({ - insightsUpsellingMessage, -}: { - insightsUpsellingMessage: string | null; -}) => { +export const plugin = ({ insightsUpsellingMessage }: { insightsUpsellingMessage?: string }) => { return { name: 'insights', button: { diff --git a/x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/osquery/plugin.tsx b/x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/osquery/plugin.tsx index 6a37280f9ef23..67b3f9e2b4f8a 100644 --- a/x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/osquery/plugin.tsx +++ b/x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/osquery/plugin.tsx @@ -161,7 +161,7 @@ const OsqueryEditor = React.memo(OsqueryEditorComponent); export const plugin = ({ interactionsUpsellingMessage, }: { - interactionsUpsellingMessage: string | null; + interactionsUpsellingMessage?: string; }) => { return { name: 'osquery', diff --git a/x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/timeline/plugin.tsx b/x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/timeline/plugin.tsx index 40b2fba4b84d0..4d5b2e14e0d95 100644 --- a/x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/timeline/plugin.tsx +++ b/x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/timeline/plugin.tsx @@ -78,7 +78,7 @@ const TimelineEditor = memo(TimelineEditorComponent); export const plugin = ({ interactionsUpsellingMessage, }: { - interactionsUpsellingMessage: string | null; + interactionsUpsellingMessage?: string; }): EuiMarkdownEditorUiPlugin => { return { name: ID, diff --git a/x-pack/plugins/security_solution/public/common/hooks/use_upselling.test.tsx b/x-pack/plugins/security_solution/public/common/hooks/use_upselling.test.tsx index ee783bcd19e3d..c18a6282eb373 100644 --- a/x-pack/plugins/security_solution/public/common/hooks/use_upselling.test.tsx +++ b/x-pack/plugins/security_solution/public/common/hooks/use_upselling.test.tsx @@ -71,7 +71,7 @@ describe('use_upselling', () => { expect(result.all.length).toBe(1); // assert that it should not cause unnecessary re-renders }); - test('useUpsellingMessage returns null when upsellingMessageId not found', () => { + test('useUpsellingMessage returns undefined when upsellingMessageId not found', () => { const emptyMessages = {}; mockUpselling.setPages(emptyMessages); @@ -81,6 +81,6 @@ describe('use_upselling', () => { wrapper: RenderWrapper, } ); - expect(result.current).toBe(null); + expect(result.current).toBeUndefined(); }); }); diff --git a/x-pack/plugins/security_solution/public/common/hooks/use_upselling.ts b/x-pack/plugins/security_solution/public/common/hooks/use_upselling.ts index 8ef01b5b56a25..9f452bb4f2872 100644 --- a/x-pack/plugins/security_solution/public/common/hooks/use_upselling.ts +++ b/x-pack/plugins/security_solution/public/common/hooks/use_upselling.ts @@ -22,11 +22,11 @@ export const useUpsellingComponent = (id: UpsellingSectionId): React.ComponentTy return useMemo(() => upsellingSections?.get(id) ?? null, [id, upsellingSections]); }; -export const useUpsellingMessage = (id: UpsellingMessageId): string | null => { +export const useUpsellingMessage = (id: UpsellingMessageId): string | undefined => { const upselling = useUpsellingService(); const upsellingMessages = useObservable(upselling.messages$, upselling.getMessagesValue()); - return useMemo(() => upsellingMessages?.get(id) ?? null, [id, upsellingMessages]); + return useMemo(() => upsellingMessages?.get(id), [id, upsellingMessages]); }; export const useUpsellingPage = (pageName: SecurityPageName): React.ComponentType | null => { diff --git a/x-pack/plugins/security_solution/public/common/test/eui/combobox.ts b/x-pack/plugins/security_solution/public/common/test/eui/combobox.ts new file mode 100644 index 0000000000000..dad99a3c426d4 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/test/eui/combobox.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { act, fireEvent, waitFor } from '@testing-library/react'; + +export function showEuiComboBoxOptions(comboBoxToggleButton: HTMLElement): Promise { + fireEvent.click(comboBoxToggleButton); + + return waitFor(() => { + const listWithOptionsElement = document.querySelector('[role="listbox"]'); + const emptyListElement = document.querySelector('.euiComboBoxOptionsList__empty'); + + expect(listWithOptionsElement || emptyListElement).toBeInTheDocument(); + }); +} + +type SelectEuiComboBoxOptionParameters = + | { + comboBoxToggleButton: HTMLElement; + optionIndex: number; + optionText?: undefined; + } + | { + comboBoxToggleButton: HTMLElement; + optionText: string; + optionIndex?: undefined; + }; + +export function selectEuiComboBoxOption({ + comboBoxToggleButton, + optionIndex, + optionText, +}: SelectEuiComboBoxOptionParameters): Promise { + return act(async () => { + await showEuiComboBoxOptions(comboBoxToggleButton); + + const options = Array.from( + document.querySelectorAll('[data-test-subj*="comboBoxOptionsList"] [role="option"]') + ); + + if (typeof optionText === 'string') { + const optionToSelect = options.find((option) => option.textContent === optionText); + + if (optionToSelect) { + fireEvent.click(optionToSelect); + } else { + throw new Error( + `Could not find option with text "${optionText}". Available options: ${options + .map((option) => option.textContent) + .join(', ')}` + ); + } + } else { + fireEvent.click(options[optionIndex]); + } + }); +} + +export function selectFirstEuiComboBoxOption({ + comboBoxToggleButton, +}: { + comboBoxToggleButton: HTMLElement; +}): Promise { + return selectEuiComboBoxOption({ comboBoxToggleButton, optionIndex: 0 }); +} + +export function clearEuiComboBoxSelection({ + clearButton, +}: { + clearButton: HTMLElement; +}): Promise { + return act(async () => { + fireEvent.click(clearButton); + }); +} diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/components/alert_suppression_edit.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/components/alert_suppression_edit.tsx new file mode 100644 index 0000000000000..5c6099529e920 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/components/alert_suppression_edit.tsx @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { memo } from 'react'; +import { EuiPanel, EuiText, EuiToolTip } from '@elastic/eui'; +import type { DataViewFieldBase } from '@kbn/es-query'; +import { useFormData } from '../../../../../shared_imports'; +import { MissingFieldsStrategySelector } from './missing_fields_strategy_selector'; +import { SuppressionDurationSelector } from './suppression_duration_selector'; +import { SuppressionFieldsSelector } from './suppression_fields_selector'; +import { ALERT_SUPPRESSION_FIELDS_FIELD_NAME } from '../constants/fields'; + +interface AlertSuppressionEditProps { + suppressibleFields: DataViewFieldBase[]; + labelAppend?: React.ReactNode; + disabled?: boolean; + disabledText?: string; + warningText?: string; +} + +export const AlertSuppressionEdit = memo(function AlertSuppressionEdit({ + suppressibleFields, + labelAppend, + disabled, + disabledText, + warningText, +}: AlertSuppressionEditProps): JSX.Element { + const [{ [ALERT_SUPPRESSION_FIELDS_FIELD_NAME]: suppressionFields }] = useFormData<{ + [ALERT_SUPPRESSION_FIELDS_FIELD_NAME]: string[]; + }>({ + watch: ALERT_SUPPRESSION_FIELDS_FIELD_NAME, + }); + const hasSelectedFields = suppressionFields?.length > 0; + const content = ( + <> + + {warningText && ( + + {warningText} + + )} + + + + + + ); + + return disabled && disabledText ? ( + + {content} + + ) : ( + content + ); +}); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/components/missing_fields_strategy_selector.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/components/missing_fields_strategy_selector.tsx new file mode 100644 index 0000000000000..a7b38843a61ce --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/components/missing_fields_strategy_selector.tsx @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useMemo } from 'react'; +import type { EuiFormRowProps, EuiRadioGroupOption, EuiRadioGroupProps } from '@elastic/eui'; +import { RadioGroupField } from '@kbn/es-ui-shared-plugin/static/forms/components'; +import { AlertSuppressionMissingFieldsStrategyEnum } from '../../../../../../common/api/detection_engine'; +import { UseField } from '../../../../../shared_imports'; +import { SuppressionInfoIcon } from './suppression_info_icon'; +import { ALERT_SUPPRESSION_MISSING_FIELDS_FIELD_NAME } from '../constants/fields'; +import * as i18n from './translations'; + +interface MissingFieldsStrategySelectorProps { + disabled?: boolean; +} + +export function MissingFieldsStrategySelector({ + disabled, +}: MissingFieldsStrategySelectorProps): JSX.Element { + const radioFieldProps: Partial = useMemo( + () => ({ + options: ALERT_SUPPRESSION_MISSING_FIELDS_STRATEGY_OPTIONS, + 'data-test-subj': 'suppressionMissingFieldsOptions', + disabled, + }), + [disabled] + ); + + return ( + + ); +} + +const EUI_FORM_ROW_PROPS: Partial = { + label: ( + + {i18n.ALERT_SUPPRESSION_MISSING_FIELDS_LABEL} + + ), + 'data-test-subj': 'alertSuppressionMissingFields', +}; + +const ALERT_SUPPRESSION_MISSING_FIELDS_STRATEGY_OPTIONS: EuiRadioGroupOption[] = [ + { + id: AlertSuppressionMissingFieldsStrategyEnum.suppress, + label: i18n.ALERT_SUPPRESSION_MISSING_FIELDS_SUPPRESS_OPTION, + }, + { + id: AlertSuppressionMissingFieldsStrategyEnum.doNotSuppress, + label: i18n.ALERT_SUPPRESSION_MISSING_FIELDS_DO_NOT_SUPPRESS_OPTION, + }, +]; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/components/suppression_duration_selector.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/components/suppression_duration_selector.tsx new file mode 100644 index 0000000000000..7cf5eeb3018b1 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/components/suppression_duration_selector.tsx @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { memo, useEffect } from 'react'; +import { EuiFormRow, EuiRadioGroup, EuiToolTip, useEuiTheme } from '@elastic/eui'; +import { css } from '@emotion/css'; +import type { FieldHook } from '../../../../../shared_imports'; +import { UseMultiFields } from '../../../../../shared_imports'; +import { AlertSuppressionDurationType } from '../../../../../detections/pages/detection_engine/rules/types'; +import { DurationInput } from '../../duration_input'; +import { + ALERT_SUPPRESSION_DURATION_FIELD_NAME, + ALERT_SUPPRESSION_DURATION_TYPE_FIELD_NAME, + ALERT_SUPPRESSION_DURATION_UNIT_FIELD_NAME, + ALERT_SUPPRESSION_DURATION_VALUE_FIELD_NAME, +} from '../constants/fields'; +import * as i18n from './translations'; + +interface AlertSuppressionDurationProps { + onlyPerTimePeriod?: boolean; + onlyPerTimePeriodReasonMessage?: string; + disabled?: boolean; +} + +export function SuppressionDurationSelector({ + onlyPerTimePeriod, + onlyPerTimePeriodReasonMessage, + disabled, +}: AlertSuppressionDurationProps): JSX.Element { + return ( + + + fields={{ + suppressionDurationSelector: { + path: ALERT_SUPPRESSION_DURATION_TYPE_FIELD_NAME, + }, + suppressionDurationValue: { + path: `${ALERT_SUPPRESSION_DURATION_FIELD_NAME}.${ALERT_SUPPRESSION_DURATION_VALUE_FIELD_NAME}`, + }, + suppressionDurationUnit: { + path: `${ALERT_SUPPRESSION_DURATION_FIELD_NAME}.${ALERT_SUPPRESSION_DURATION_UNIT_FIELD_NAME}`, + }, + }} + > + {({ suppressionDurationSelector, suppressionDurationValue, suppressionDurationUnit }) => ( + + )} + + + ); +} + +interface SuppressionDurationSelectorFieldsProps { + suppressionDurationSelectorField: FieldHook; + suppressionDurationValueField: FieldHook; + suppressionDurationUnitField: FieldHook; + onlyPerTimePeriod?: boolean; + onlyPerTimePeriodReasonMessage?: string; + disabled?: boolean; +} + +const SuppressionDurationSelectorFields = memo(function SuppressionDurationSelectorFields({ + suppressionDurationSelectorField, + suppressionDurationValueField, + suppressionDurationUnitField, + onlyPerTimePeriod = false, + onlyPerTimePeriodReasonMessage, + disabled, +}: SuppressionDurationSelectorFieldsProps): JSX.Element { + const { euiTheme } = useEuiTheme(); + const { value: durationType, setValue: setDurationType } = suppressionDurationSelectorField; + + useEffect(() => { + if (onlyPerTimePeriod && durationType !== AlertSuppressionDurationType.PerTimePeriod) { + setDurationType(AlertSuppressionDurationType.PerTimePeriod); + } + }, [onlyPerTimePeriod, durationType, setDurationType]); + + return ( + <> + + <> {i18n.ALERT_SUPPRESSION_DURATION_PER_RULE_EXECUTION_OPTION} + + ) : ( + i18n.ALERT_SUPPRESSION_DURATION_PER_RULE_EXECUTION_OPTION + ), + disabled: onlyPerTimePeriod ? true : disabled, + }, + { + id: AlertSuppressionDurationType.PerTimePeriod, + disabled, + label: i18n.ALERT_SUPPRESSION_DURATION_PER_TIME_PERIOD_OPTION, + }, + ]} + onChange={(id) => { + suppressionDurationSelectorField.setValue(id); + }} + data-test-subj="alertSuppressionDurationOptions" + /> +
+ +
+ + ); +}); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/components/suppression_fields_selector.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/components/suppression_fields_selector.tsx new file mode 100644 index 0000000000000..72eea027288f0 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/components/suppression_fields_selector.tsx @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiFormRow } from '@elastic/eui'; +import type { DataViewFieldBase } from '@kbn/es-query'; +import { UseField } from '../../../../../shared_imports'; +import { MultiSelectFieldsAutocomplete } from '../../../../rule_creation_ui/components/multi_select_fields'; +import { ALERT_SUPPRESSION_FIELDS_FIELD_NAME } from '../constants/fields'; +import * as i18n from './translations'; + +interface SuppressionFieldsSelectorProps { + suppressibleFields: DataViewFieldBase[]; + labelAppend?: React.ReactNode; + disabled?: boolean; +} + +export function SuppressionFieldsSelector({ + suppressibleFields, + labelAppend, + disabled, +}: SuppressionFieldsSelectorProps): JSX.Element { + return ( + + <> + + + + ); +} diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/suppression_info_icon/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/components/suppression_info_icon.tsx similarity index 80% rename from x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/suppression_info_icon/index.tsx rename to x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/components/suppression_info_icon.tsx index bb3b0db1ccdab..466600dd394da 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/suppression_info_icon/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/components/suppression_info_icon.tsx @@ -5,28 +5,25 @@ * 2.0. */ -import React, { useState } from 'react'; +import React from 'react'; import { EuiLink, EuiPopover, EuiText, EuiButtonIcon } from '@elastic/eui'; +import { useBoolean } from '@kbn/react-hooks'; import { FormattedMessage } from '@kbn/i18n-react'; - -import { useKibana } from '../../../../common/lib/kibana'; +import { useKibana } from '../../../../../common/lib/kibana'; const POPOVER_WIDTH = 320; /** * Icon and popover that gives hint to users how suppression for missing fields work */ -const SuppressionInfoIconComponent = () => { - const [isPopoverOpen, setIsPopoverOpen] = useState(false); +export function SuppressionInfoIcon(): JSX.Element { + const [isPopoverOpen, { off: closePopover, toggle: togglePopover }] = useBoolean(false); const { docLinks } = useKibana().services; - const onButtonClick = () => setIsPopoverOpen(!isPopoverOpen); - const closePopover = () => setIsPopoverOpen(false); - const button = ( ); @@ -59,8 +56,4 @@ const SuppressionInfoIconComponent = () => { ); -}; - -export const SuppressionInfoIcon = React.memo(SuppressionInfoIconComponent); - -SuppressionInfoIcon.displayName = 'SuppressionInfoIcon'; +} diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/components/translations.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/components/translations.ts new file mode 100644 index 0000000000000..8da7d435adfeb --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/components/translations.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +export const ALERT_SUPPRESSION_SUPPRESS_BY_FIELD_LABEL = i18n.translate( + 'xpack.securitySolution.ruleManagement.ruleFields.alertSuppression.fieldsSelector.label', + { + defaultMessage: 'Suppress alerts by', + } +); + +export const ALERT_SUPPRESSION_SUPPRESS_BY_FIELD_HELP_TEXT = i18n.translate( + 'xpack.securitySolution.ruleManagement.ruleFields.alertSuppression.suppressByFields.helpText', + { + defaultMessage: 'Select field(s) to use for suppressing extra alerts', + } +); + +export const ALERT_SUPPRESSION_DURATION_PER_RULE_EXECUTION_OPTION = i18n.translate( + 'xpack.securitySolution.ruleManagement.ruleFields.alertSuppression.suppressionDuration.perRuleExecutionOption', + { + defaultMessage: 'Per rule execution', + } +); + +export const ALERT_SUPPRESSION_DURATION_PER_TIME_PERIOD_OPTION = i18n.translate( + 'xpack.securitySolution.ruleManagement.ruleFields.alertSuppression.suppressionDuration.perTimePeriodOption', + { + defaultMessage: 'Per time period', + } +); + +export const ALERT_SUPPRESSION_MISSING_FIELDS_LABEL = i18n.translate( + 'xpack.securitySolution.ruleManagement.ruleFields.alertSuppression.missingFields.label', + { + defaultMessage: 'If a suppression field is missing', + } +); + +export const ALERT_SUPPRESSION_MISSING_FIELDS_SUPPRESS_OPTION = i18n.translate( + 'xpack.securitySolution.ruleManagement.ruleFields.alertSuppression.missingFields.suppressOption', + { + defaultMessage: 'Suppress and group alerts for events with missing fields', + } +); + +export const ALERT_SUPPRESSION_MISSING_FIELDS_DO_NOT_SUPPRESS_OPTION = i18n.translate( + 'xpack.securitySolution.ruleManagement.ruleFields.alertSuppression.missingFields.doNotSuppressOption', + { + defaultMessage: 'Do not suppress alerts for events with missing fields', + } +); + +export const ALERT_SUPPRESSION_NOT_SUPPORTED_FOR_EQL_SEQUENCE = i18n.translate( + 'xpack.securitySolution.ruleManagement.ruleFields.alertSuppression.notSupportedForEqlSequence', + { + defaultMessage: 'Suppression is not supported for EQL sequence queries', + } +); + +export const MACHINE_LEARNING_SUPPRESSION_FIELDS_LOADING = i18n.translate( + 'xpack.securitySolution.ruleManagement.ruleFields.alertSuppression.machineLearningSuppressionFieldsLoading', + { + defaultMessage: 'Machine Learning suppression fields are loading', + } +); + +export const MACHINE_LEARNING_NO_SUPPRESSION_FIELDS = i18n.translate( + 'xpack.securitySolution.ruleManagement.ruleFields.alertSuppression.machineLearningNoSuppressionFields', + { + defaultMessage: + 'Unable to load machine Learning suppression fields, start relevant Machine Learning jobs.', + } +); + +export const ESQL_SUPPRESSION_FIELDS_LOADING = i18n.translate( + 'xpack.securitySolution.ruleManagement.ruleFields.alertSuppression.esqlFieldsLoading', + { + defaultMessage: 'ES|QL suppression fields are loading', + } +); + +export const MACHINE_LEARNING_SUPPRESSION_INCOMPLETE_LABEL = i18n.translate( + 'xpack.securitySolution.ruleManagement.ruleFields.alertSuppression.machineLearningSuppressionIncomplete', + { + defaultMessage: + 'This list of fields might be incomplete as some Machine Learning jobs are not running. Start all relevant jobs for a complete list.', + } +); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/constants/default_duration.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/constants/default_duration.ts new file mode 100644 index 0000000000000..6e06d0d67031a --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/constants/default_duration.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AlertSuppressionDurationUnitEnum } from '../../../../../../common/api/detection_engine'; +import { + ALERT_SUPPRESSION_DURATION_VALUE_FIELD_NAME, + ALERT_SUPPRESSION_DURATION_UNIT_FIELD_NAME, +} from './fields'; + +export const ALERT_SUPPRESSION_DEFAULT_DURATION = { + [ALERT_SUPPRESSION_DURATION_VALUE_FIELD_NAME]: 5, + [ALERT_SUPPRESSION_DURATION_UNIT_FIELD_NAME]: AlertSuppressionDurationUnitEnum.m, +}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/constants/fields.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/constants/fields.ts new file mode 100644 index 0000000000000..42a0583e90512 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/constants/fields.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const ALERT_SUPPRESSION_FIELDS_FIELD_NAME = 'alertSuppressionFields' as const; +export const ALERT_SUPPRESSION_DURATION_TYPE_FIELD_NAME = 'alertSuppressionDurationType' as const; +export const ALERT_SUPPRESSION_DURATION_FIELD_NAME = 'alertSuppressionDuration' as const; +export const ALERT_SUPPRESSION_DURATION_VALUE_FIELD_NAME = 'value' as const; +export const ALERT_SUPPRESSION_DURATION_UNIT_FIELD_NAME = 'unit' as const; +export const ALERT_SUPPRESSION_MISSING_FIELDS_FIELD_NAME = 'alertSuppressionMissingFields' as const; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/index.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/index.ts new file mode 100644 index 0000000000000..a97e74726e3c4 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './components/alert_suppression_edit'; +export * from './components/suppression_duration_selector'; +export * from './constants/fields'; +export * from './constants/default_duration'; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/test_helpers.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/test_helpers.ts new file mode 100644 index 0000000000000..b7d6c4003e934 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/alert_suppression_edit/test_helpers.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// eslint-disable-next-line import/no-extraneous-dependencies +import { act, fireEvent, waitFor, within, screen } from '@testing-library/react'; +import type { AlertSuppressionDurationUnit } from '../../../../../common/api/detection_engine'; +import { selectEuiComboBoxOption } from '../../../../common/test/eui/combobox'; + +const COMBO_BOX_TOGGLE_BUTTON_TEST_ID = 'comboBoxToggleListButton'; + +export async function setSuppressionFields(fieldNames: string[]): Promise { + const getAlertSuppressionFieldsComboBoxToggleButton = () => + within(screen.getByTestId('alertSuppressionInput')).getByTestId( + COMBO_BOX_TOGGLE_BUTTON_TEST_ID + ); + + await waitFor(() => { + expect(getAlertSuppressionFieldsComboBoxToggleButton()).toBeInTheDocument(); + }); + + for (const fieldName of fieldNames) { + await selectEuiComboBoxOption({ + comboBoxToggleButton: getAlertSuppressionFieldsComboBoxToggleButton(), + optionText: fieldName, + }); + } +} + +export function expectSuppressionFields(fieldNames: string[]): void { + for (const fieldName of fieldNames) { + expect( + within(screen.getByTestId('alertSuppressionInput')).getByTitle(fieldName) + ).toBeInTheDocument(); + } +} + +export function setDurationType(value: 'Per rule execution' | 'Per time period'): void { + act(() => { + fireEvent.click(within(screen.getByTestId('alertSuppressionDuration')).getByLabelText(value)); + }); +} + +export function setDuration(value: number, unit: AlertSuppressionDurationUnit): void { + act(() => { + fireEvent.input( + within(screen.getByTestId('alertSuppressionDuration')).getByTestId('interval'), + { + target: { value: value.toString() }, + } + ); + + fireEvent.change( + within(screen.getByTestId('alertSuppressionDuration')).getByTestId('timeType'), + { + target: { value: unit }, + } + ); + }); +} + +export function expectDuration(value: number, unit: AlertSuppressionDurationUnit): void { + expect( + within(screen.getByTestId('alertSuppressionDuration')).getByTestId('interval') + ).toHaveValue(value); + expect( + within(screen.getByTestId('alertSuppressionDuration')).getByTestId('timeType') + ).toHaveValue(unit); +} diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/duration_input/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/duration_input/index.tsx similarity index 68% rename from x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/duration_input/index.tsx rename to x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/duration_input/index.tsx index 99222756bcf26..b203cdea8f737 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/duration_input/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/duration_input/index.tsx @@ -5,10 +5,9 @@ * 2.0. */ -import { EuiFieldNumber, EuiFormRow, EuiSelect, transparentize } from '@elastic/eui'; -import React, { useCallback } from 'react'; -import styled from 'styled-components'; - +import React, { memo, useCallback } from 'react'; +import { css } from '@emotion/css'; +import { EuiFieldNumber, EuiFormRow, EuiSelect, transparentize, useEuiTheme } from '@elastic/eui'; import type { FieldHook } from '../../../../shared_imports'; import { getFieldValidityAndErrorMessage } from '../../../../shared_imports'; import * as I18n from './translations'; @@ -21,39 +20,10 @@ interface DurationInputProps { durationUnitOptions?: Array<{ value: 's' | 'm' | 'h' | 'd'; text: string }>; } -const getNumberFromUserInput = (input: string, minimumValue = 0): number | undefined => { - const number = parseInt(input, 10); - if (Number.isNaN(number)) { - return minimumValue; - } else { - return Math.max(minimumValue, Math.min(number, Number.MAX_SAFE_INTEGER)); - } -}; - -const StyledEuiFormRow = styled(EuiFormRow)` - max-width: 235px; - - .euiFormControlLayout__append { - padding-inline: 0 !important; - } - - .euiFormControlLayoutIcons { - color: ${({ theme }) => theme.eui.euiColorPrimary}; - } -`; - -const MyEuiSelect = styled(EuiSelect)` - min-width: 106px; // Preserve layout when disabled & dropdown arrow is not rendered - box-shadow: none; - background: ${({ theme }) => - transparentize(theme.eui.euiColorPrimary, 0.1)} !important; // Override focus states etc. - color: ${({ theme }) => theme.eui.euiColorPrimary}; -`; - // This component is similar to the ScheduleItem component, but instead of combining the value // and unit into a single string it keeps them separate. This makes the component simpler and // allows for easier validation of values and units in APIs as well. -const DurationInputComponent: React.FC = ({ +export const DurationInput = memo(function DurationInputComponent({ durationValueField, durationUnitField, minimumValue = 0, @@ -64,7 +34,8 @@ const DurationInputComponent: React.FC = ({ { value: 'h', text: I18n.HOURS }, ], ...props -}: DurationInputProps) => { +}: DurationInputProps): JSX.Element { + const { euiTheme } = useEuiTheme(); const { isInvalid, errorMessage } = getFieldValidityAndErrorMessage(durationValueField); const { value: durationValue, setValue: setDurationValue } = durationValueField; const { value: durationUnit, setValue: setDurationUnit } = durationUnitField; @@ -84,17 +55,40 @@ const DurationInputComponent: React.FC = ({ [minimumValue, setDurationValue] ); + const durationFormRowStyle = css` + max-width: 235px; + + .euiFormControlLayout__append { + padding-inline: 0 !important; + } + + .euiFormControlLayoutIcons { + color: ${euiTheme.colors.primary}; + } + `; + const durationUnitSelectStyle = css` + min-width: 106px; // Preserve layout when disabled & dropdown arrow is not rendered + box-shadow: none; + background: ${transparentize( + euiTheme.colors.primary, + 0.1 + )} !important; // Override focus states etc. + color: ${euiTheme.colors.primary}; + `; + // EUI missing some props const rest = { disabled: isDisabled, ...props }; return ( - + @@ -106,8 +100,16 @@ const DurationInputComponent: React.FC = ({ data-test-subj="interval" {...rest} /> - + ); -}; +}); + +function getNumberFromUserInput(input: string, minimumValue = 0): number | undefined { + const number = parseInt(input, 10); -export const DurationInput = React.memo(DurationInputComponent); + if (Number.isNaN(number)) { + return minimumValue; + } else { + return Math.max(minimumValue, Math.min(number, Number.MAX_SAFE_INTEGER)); + } +} diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/duration_input/translations.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/duration_input/translations.ts similarity index 82% rename from x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/duration_input/translations.ts rename to x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/duration_input/translations.ts index c460d2f7198b3..51d659210c52b 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/duration_input/translations.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/duration_input/translations.ts @@ -34,3 +34,10 @@ export const DAYS = i18n.translate( defaultMessage: 'Days', } ); + +export const DURATION_UNIT_SELECTOR = i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepScheduleRuleForm.durationUnitSelector', + { + defaultMessage: 'Duration unit selector', + } +); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/related_integrations/related_integrations.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/related_integrations/related_integrations.test.tsx index 960df4c7de5b9..31e139a335bee 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/related_integrations/related_integrations.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/related_integrations/related_integrations.test.tsx @@ -6,19 +6,24 @@ */ import React from 'react'; -import { - screen, - render, - act, - fireEvent, - waitFor, - waitForElementToBeRemoved, -} from '@testing-library/react'; +import { screen, render, act, fireEvent, waitFor } from '@testing-library/react'; import type { RelatedIntegration } from '../../../../../common/api/detection_engine'; import { FIELD_TYPES, Form, useForm } from '../../../../shared_imports'; import { createReactQueryWrapper } from '../../../../common/mock'; import { fleetIntegrationsApi } from '../../../fleet_integrations/api/__mocks__'; import { RelatedIntegrations } from './related_integrations'; +import { + clearEuiComboBoxSelection, + selectEuiComboBoxOption, + selectFirstEuiComboBoxOption, + showEuiComboBoxOptions, +} from '../../../../common/test/eui/combobox'; +import { + addRelatedIntegrationRow, + removeLastRelatedIntegrationRow, + setVersion, + waitForIntegrationsToBeLoaded, +} from './test_helpers'; // must match to the import in rules/related_integrations/use_integrations.tsx jest.mock('../../../fleet_integrations/api'); @@ -41,7 +46,6 @@ const COMBO_BOX_TOGGLE_BUTTON_TEST_ID = 'comboBoxToggleListButton'; const COMBO_BOX_SELECTION_TEST_ID = 'euiComboBoxPill'; const COMBO_BOX_CLEAR_BUTTON_TEST_ID = 'comboBoxClearButton'; const VERSION_INPUT_TEST_ID = 'relatedIntegrationVersionDependency'; -const REMOVE_INTEGRATION_ROW_BUTTON_TEST_ID = 'relatedIntegrationRemove'; describe('RelatedIntegrations form part', () => { beforeEach(() => { @@ -708,72 +712,6 @@ function TestForm({ initialState, onSubmit }: TestFormProps): JSX.Element { ); } -function waitForIntegrationsToBeLoaded(): Promise { - return waitForElementToBeRemoved(screen.queryAllByRole('progressbar')); -} - -function addRelatedIntegrationRow(): Promise { - return act(async () => { - fireEvent.click(screen.getByText('Add integration')); - }); -} - -function removeLastRelatedIntegrationRow(): Promise { - return act(async () => { - const lastRemoveButton = screen.getAllByTestId(REMOVE_INTEGRATION_ROW_BUTTON_TEST_ID).at(-1); - - if (!lastRemoveButton) { - throw new Error(`There are no "${REMOVE_INTEGRATION_ROW_BUTTON_TEST_ID}" found`); - } - - fireEvent.click(lastRemoveButton); - }); -} - -function showEuiComboBoxOptions(comboBoxToggleButton: HTMLElement): Promise { - fireEvent.click(comboBoxToggleButton); - - return waitFor(() => { - expect(screen.getByRole('listbox')).toBeInTheDocument(); - }); -} - -function selectEuiComboBoxOption({ - comboBoxToggleButton, - optionIndex, -}: { - comboBoxToggleButton: HTMLElement; - optionIndex: number; -}): Promise { - return act(async () => { - await showEuiComboBoxOptions(comboBoxToggleButton); - - fireEvent.click(screen.getAllByRole('option')[optionIndex]); - }); -} - -function clearEuiComboBoxSelection({ clearButton }: { clearButton: HTMLElement }): Promise { - return act(async () => { - fireEvent.click(clearButton); - }); -} - -function selectFirstEuiComboBoxOption({ - comboBoxToggleButton, -}: { - comboBoxToggleButton: HTMLElement; -}): Promise { - return selectEuiComboBoxOption({ comboBoxToggleButton, optionIndex: 0 }); -} - -function setVersion({ input, value }: { input: HTMLInputElement; value: string }): Promise { - return act(async () => { - fireEvent.input(input, { - target: { value }, - }); - }); -} - function submitForm(): Promise { return act(async () => { fireEvent.click(screen.getByText('Submit')); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/related_integrations/test_helpers.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/related_integrations/test_helpers.ts new file mode 100644 index 0000000000000..b8c51fd594e13 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/related_integrations/test_helpers.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// eslint-disable-next-line import/no-extraneous-dependencies +import { act, fireEvent, waitForElementToBeRemoved, screen } from '@testing-library/react'; + +const REMOVE_INTEGRATION_ROW_BUTTON_TEST_ID = 'relatedIntegrationRemove'; + +export function waitForIntegrationsToBeLoaded(): Promise { + return waitForElementToBeRemoved(screen.queryAllByRole('progressbar')); +} + +export function addRelatedIntegrationRow(): Promise { + return act(async () => { + fireEvent.click(screen.getByText('Add integration')); + }); +} + +export function removeLastRelatedIntegrationRow(): Promise { + return act(async () => { + const lastRemoveButton = screen.getAllByTestId(REMOVE_INTEGRATION_ROW_BUTTON_TEST_ID).at(-1); + + if (!lastRemoveButton) { + throw new Error(`There are no "${REMOVE_INTEGRATION_ROW_BUTTON_TEST_ID}" found`); + } + + fireEvent.click(lastRemoveButton); + }); +} + +export function setVersion({ + input, + value, +}: { + input: HTMLInputElement; + value: string; +}): Promise { + return act(async () => { + fireEvent.input(input, { + target: { value }, + }); + }); +} diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/threshold_alert_suppression_edit/fields.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/threshold_alert_suppression_edit/fields.ts new file mode 100644 index 0000000000000..4956a2555bc9c --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/threshold_alert_suppression_edit/fields.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const THRESHOLD_ALERT_SUPPRESSION_ENABLED = 'thresholdAlertSuppressionEnabled' as const; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/threshold_alert_suppression_edit/index.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/threshold_alert_suppression_edit/index.ts new file mode 100644 index 0000000000000..67848fbd5e3b5 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/threshold_alert_suppression_edit/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './threshold_alert_suppression_edit'; +export * from './fields'; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/threshold_alert_suppression_edit/threshold_alert_suppression_edit.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/threshold_alert_suppression_edit/threshold_alert_suppression_edit.tsx new file mode 100644 index 0000000000000..a832bff648e8a --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/threshold_alert_suppression_edit/threshold_alert_suppression_edit.tsx @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { memo } from 'react'; +import { EuiPanel, EuiToolTip } from '@elastic/eui'; +import { CheckBoxField } from '@kbn/es-ui-shared-plugin/static/forms/components'; +import { UseField, useFormData } from '../../../../shared_imports'; +import { THRESHOLD_ALERT_SUPPRESSION_ENABLED } from './fields'; +import { SuppressionDurationSelector } from '../alert_suppression_edit'; +import * as i18n from './translations'; + +interface ThresholdAlertSuppressionEditProps { + suppressionFieldNames: string[] | undefined; + disabled?: boolean; + disabledText?: string; +} + +export const ThresholdAlertSuppressionEdit = memo(function ThresholdAlertSuppressionEdit({ + suppressionFieldNames, + disabled, + disabledText, +}: ThresholdAlertSuppressionEditProps): JSX.Element { + const [{ [THRESHOLD_ALERT_SUPPRESSION_ENABLED]: suppressionEnabled }] = useFormData({ + watch: THRESHOLD_ALERT_SUPPRESSION_ENABLED, + }); + const content = ( + <> + + + + + + ); + + return disabled && disabledText ? ( + + {content} + + ) : ( + content + ); +}); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/threshold_alert_suppression_edit/translations.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/threshold_alert_suppression_edit/translations.tsx new file mode 100644 index 0000000000000..25b7158610b34 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/threshold_alert_suppression_edit/translations.tsx @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; + +export const enableSuppressionForFields = (fields: string[]) => ( + {fields.join(', ')} }} + /> +); + +export const SUPPRESS_ALERTS = i18n.translate( + 'xpack.securitySolution.ruleManagement.ruleFields.thresholdAlertSuppression.enable', + { + defaultMessage: 'Suppress alerts', + } +); + +export const THRESHOLD_SUPPRESSION_PER_RULE_EXECUTION_WARNING = i18n.translate( + 'xpack.securitySolution.ruleManagement.ruleFields.thresholdAlertSuppression.perRuleExecutionWarning', + { + defaultMessage: 'Per rule execution option is not available for Threshold rule type', + } +); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/data_view_selector_field/__mocks__/use_data_views.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/data_view_selector_field/__mocks__/use_data_view_list_items.ts similarity index 81% rename from x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/data_view_selector_field/__mocks__/use_data_views.ts rename to x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/data_view_selector_field/__mocks__/use_data_view_list_items.ts index 248729f1f46e7..3d2ba5d1c3724 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/data_view_selector_field/__mocks__/use_data_views.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/data_view_selector_field/__mocks__/use_data_view_list_items.ts @@ -5,7 +5,7 @@ * 2.0. */ -export const useDataViews = jest.fn().mockReturnValue({ +export const useDataViewListItems = jest.fn().mockReturnValue({ data: [], isFetching: false, }); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/data_view_selector_field/data_view_selector_field.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/data_view_selector_field/data_view_selector_field.test.tsx index 6cfdf060434b8..8648ade5164e6 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/data_view_selector_field/data_view_selector_field.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/data_view_selector_field/data_view_selector_field.test.tsx @@ -9,14 +9,14 @@ import React from 'react'; import { screen, render } from '@testing-library/react'; import { TestProviders, useFormFieldMock } from '../../../../common/mock'; import { DataViewSelectorField } from './data_view_selector_field'; -import { useDataViews } from './use_data_views'; +import { useDataViewListItems } from './use_data_view_list_items'; jest.mock('../../../../common/lib/kibana'); -jest.mock('./use_data_views'); +jest.mock('./use_data_view_list_items'); describe('data_view_selector', () => { it('renders correctly', () => { - (useDataViews as jest.Mock).mockReturnValue({ data: [], isFetching: false }); + (useDataViewListItems as jest.Mock).mockReturnValue({ data: [], isFetching: false }); render( { }); it('disables the combobox while data views are fetching', () => { - (useDataViews as jest.Mock).mockReturnValue({ data: [], isFetching: true }); + (useDataViewListItems as jest.Mock).mockReturnValue({ data: [], isFetching: true }); render( { title: 'logs-*', }, ]; - (useDataViews as jest.Mock).mockReturnValue({ data: dataViews, isFetching: false }); + (useDataViewListItems as jest.Mock).mockReturnValue({ data: dataViews, isFetching: false }); render( { title: 'logs-*', }, ]; - (useDataViews as jest.Mock).mockReturnValue({ data: dataViews, isFetching: false }); + (useDataViewListItems as jest.Mock).mockReturnValue({ data: dataViews, isFetching: false }); render( { }); it('displays warning on missing data view', () => { - (useDataViews as jest.Mock).mockReturnValue({ data: [], isFetching: false }); + (useDataViewListItems as jest.Mock).mockReturnValue({ data: [], isFetching: false }); render( { @@ -615,11 +615,11 @@ export const buildAlertSuppressionDescription = ( export const buildAlertSuppressionWindowDescription = ( label: string, value: Duration, - groupByRadioSelection: GroupByOptions, + alertSuppressionDuration: AlertSuppressionDurationType, ruleType: Type ): ListItems[] => { const description = - groupByRadioSelection === GroupByOptions.PerTimePeriod + alertSuppressionDuration === AlertSuppressionDurationType.PerTimePeriod ? `${value.value}${value.unit}` : i18n.ALERT_SUPPRESSION_PER_RULE_EXECUTION; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx index f5a7e39634359..de46d09065f4e 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx @@ -30,6 +30,15 @@ import { schema } from '../step_about_rule/schema'; import type { ListItems } from './types'; import type { AboutStepRule } from '../../../../detections/pages/detection_engine/rules/types'; import { createLicenseServiceMock } from '../../../../../common/license/mocks'; +import { + ALERT_SUPPRESSION_DURATION_FIELD_NAME, + ALERT_SUPPRESSION_DURATION_TYPE_FIELD_NAME, + ALERT_SUPPRESSION_DURATION_UNIT_FIELD_NAME, + ALERT_SUPPRESSION_DURATION_VALUE_FIELD_NAME, + ALERT_SUPPRESSION_FIELDS_FIELD_NAME, + ALERT_SUPPRESSION_MISSING_FIELDS_FIELD_NAME, +} from '../../../rule_creation/components/alert_suppression_edit'; +import { THRESHOLD_ALERT_SUPPRESSION_ENABLED } from '../../../rule_creation/components/threshold_alert_suppression_edit'; jest.mock('../../../../common/lib/kibana'); @@ -575,25 +584,25 @@ describe('description_step', () => { describe('alert suppression', () => { const suppressionFields = { - groupByDuration: { - unit: 'm', - value: 50, + [ALERT_SUPPRESSION_DURATION_FIELD_NAME]: { + [ALERT_SUPPRESSION_DURATION_VALUE_FIELD_NAME]: 50, + [ALERT_SUPPRESSION_DURATION_UNIT_FIELD_NAME]: 'm', }, - groupByRadioSelection: 'per-time-period', - enableThresholdSuppression: true, - groupByFields: ['agent.name'], - suppressionMissingFields: 'suppress', + [ALERT_SUPPRESSION_DURATION_TYPE_FIELD_NAME]: 'per-time-period', + [THRESHOLD_ALERT_SUPPRESSION_ENABLED]: true, + [ALERT_SUPPRESSION_FIELDS_FIELD_NAME]: ['agent.name'], + [ALERT_SUPPRESSION_MISSING_FIELDS_FIELD_NAME]: 'suppress', }; - describe('groupByDuration', () => { + describe(ALERT_SUPPRESSION_DURATION_FIELD_NAME, () => { ['query', 'saved_query'].forEach((ruleType) => { - test(`should be empty if groupByFields empty for ${ruleType} rule`, () => { + test(`should be empty if ${ALERT_SUPPRESSION_FIELDS_FIELD_NAME} empty for ${ruleType} rule`, () => { const result: ListItems[] = getDescriptionItem( - 'groupByDuration', + ALERT_SUPPRESSION_DURATION_FIELD_NAME, 'label', { ruleType: 'query', ...suppressionFields, - groupByFields: [], + [ALERT_SUPPRESSION_FIELDS_FIELD_NAME]: [], }, mockFilterManager, mockLicenseService @@ -604,7 +613,7 @@ describe('description_step', () => { test(`should return item for ${ruleType} rule`, () => { const result: ListItems[] = getDescriptionItem( - 'groupByDuration', + ALERT_SUPPRESSION_DURATION_FIELD_NAME, 'label', { ruleType: 'query', @@ -620,7 +629,7 @@ describe('description_step', () => { test('should return item for threshold rule', () => { const result: ListItems[] = getDescriptionItem( - 'groupByDuration', + ALERT_SUPPRESSION_DURATION_FIELD_NAME, 'label', { ruleType: 'threshold', @@ -633,14 +642,14 @@ describe('description_step', () => { expect(result[0].description).toBe('50m'); }); - test('should return item for threshold rule if groupByFields empty', () => { + test(`should return item for threshold rule if ${ALERT_SUPPRESSION_FIELDS_FIELD_NAME} empty`, () => { const result: ListItems[] = getDescriptionItem( - 'groupByDuration', + ALERT_SUPPRESSION_DURATION_FIELD_NAME, 'label', { ruleType: 'threshold', ...suppressionFields, - groupByFields: [], + [ALERT_SUPPRESSION_FIELDS_FIELD_NAME]: [], }, mockFilterManager, mockLicenseService @@ -651,12 +660,12 @@ describe('description_step', () => { test('should be empty for threshold rule if suppression not enabled', () => { const result: ListItems[] = getDescriptionItem( - 'groupByDuration', + ALERT_SUPPRESSION_DURATION_FIELD_NAME, 'label', { ruleType: 'threshold', ...suppressionFields, - enableThresholdSuppression: false, + [THRESHOLD_ALERT_SUPPRESSION_ENABLED]: false, }, mockFilterManager, mockLicenseService @@ -666,10 +675,10 @@ describe('description_step', () => { }); }); - describe('groupByFields', () => { + describe(ALERT_SUPPRESSION_FIELDS_FIELD_NAME, () => { test(`should be empty if rule type is 'threshold'`, () => { const result: ListItems[] = getDescriptionItem( - 'groupByFields', + ALERT_SUPPRESSION_FIELDS_FIELD_NAME, 'label', { ruleType: 'threshold', @@ -685,7 +694,7 @@ describe('description_step', () => { ['query', 'saved_query'].forEach((ruleType) => { test(`should return item for ${ruleType} rule`, () => { const result: ListItems[] = getDescriptionItem( - 'groupByFields', + ALERT_SUPPRESSION_FIELDS_FIELD_NAME, 'label', { ruleType, @@ -699,10 +708,10 @@ describe('description_step', () => { }); }); - describe('suppressionMissingFields', () => { + describe(ALERT_SUPPRESSION_MISSING_FIELDS_FIELD_NAME, () => { test(`should be empty if rule type is 'threshold'`, () => { const result: ListItems[] = getDescriptionItem( - 'suppressionMissingFields', + ALERT_SUPPRESSION_MISSING_FIELDS_FIELD_NAME, 'label', { ruleType: 'threshold', @@ -718,7 +727,7 @@ describe('description_step', () => { ['query', 'saved_query'].forEach((ruleType) => { test(`should return item for ${ruleType} rule`, () => { const result: ListItems[] = getDescriptionItem( - 'suppressionMissingFields', + ALERT_SUPPRESSION_MISSING_FIELDS_FIELD_NAME, 'label', { ruleType, @@ -730,14 +739,14 @@ describe('description_step', () => { expect(result[0].description).toContain('Suppress'); }); - test(`should be empty if groupByFields empty for ${ruleType} rule`, () => { + test(`should be empty if ${ALERT_SUPPRESSION_FIELDS_FIELD_NAME} empty for ${ruleType} rule`, () => { const result: ListItems[] = getDescriptionItem( - 'suppressionMissingFields', + ALERT_SUPPRESSION_MISSING_FIELDS_FIELD_NAME, 'label', { ruleType: 'query', ...suppressionFields, - groupByFields: [], + [ALERT_SUPPRESSION_FIELDS_FIELD_NAME]: [], }, mockFilterManager, mockLicenseService diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx index 4676f065f4af8..657f592fe47c4 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx @@ -65,6 +65,13 @@ import { isSuppressionRuleConfiguredWithGroupBy, isSuppressionRuleConfiguredWithDuration, } from '../../../../../common/detection_engine/utils'; +import { + ALERT_SUPPRESSION_DURATION_FIELD_NAME, + ALERT_SUPPRESSION_DURATION_TYPE_FIELD_NAME, + ALERT_SUPPRESSION_FIELDS_FIELD_NAME, + ALERT_SUPPRESSION_MISSING_FIELDS_FIELD_NAME, +} from '../../../rule_creation/components/alert_suppression_edit'; +import { THRESHOLD_ALERT_SUPPRESSION_ENABLED } from '../../../rule_creation/components/threshold_alert_suppression_edit'; const DescriptionListContainer = styled(EuiDescriptionList)` max-width: 600px; @@ -217,7 +224,7 @@ export const getDescriptionItem = ( }); } else if (field === 'responseActions') { return []; - } else if (field === 'groupByFields') { + } else if (field === ALERT_SUPPRESSION_FIELDS_FIELD_NAME) { const ruleType: Type = get('ruleType', data); const ruleCanHaveGroupByFields = isSuppressionRuleConfiguredWithGroupBy(ruleType); @@ -226,9 +233,9 @@ export const getDescriptionItem = ( } const values: string[] = get(field, data); return buildAlertSuppressionDescription(label, values, ruleType); - } else if (field === 'groupByRadioSelection') { + } else if (field === ALERT_SUPPRESSION_DURATION_TYPE_FIELD_NAME) { return []; - } else if (field === 'groupByDuration') { + } else if (field === ALERT_SUPPRESSION_DURATION_FIELD_NAME) { const ruleType: Type = get('ruleType', data); const ruleCanHaveDuration = isSuppressionRuleConfiguredWithDuration(ruleType); @@ -239,21 +246,21 @@ export const getDescriptionItem = ( // threshold rule has suppression duration without grouping fields, but suppression should be explicitly enabled by user // query rule have suppression duration only if group by fields selected const showDuration = isThresholdRule(ruleType) - ? get('enableThresholdSuppression', data) === true - : get('groupByFields', data).length > 0; + ? get(THRESHOLD_ALERT_SUPPRESSION_ENABLED, data) === true + : get(ALERT_SUPPRESSION_FIELDS_FIELD_NAME, data).length > 0; if (showDuration) { const value: Duration = get(field, data); return buildAlertSuppressionWindowDescription( label, value, - get('groupByRadioSelection', data), + get(ALERT_SUPPRESSION_DURATION_TYPE_FIELD_NAME, data), ruleType ); } else { return []; } - } else if (field === 'suppressionMissingFields') { + } else if (field === ALERT_SUPPRESSION_MISSING_FIELDS_FIELD_NAME) { const ruleType: Type = get('ruleType', data); const ruleCanHaveSuppressionMissingFields = isSuppressionRuleConfiguredWithMissingFields(ruleType); @@ -261,7 +268,7 @@ export const getDescriptionItem = ( if (!ruleCanHaveSuppressionMissingFields) { return []; } - if (get('groupByFields', data).length > 0) { + if (get(ALERT_SUPPRESSION_FIELDS_FIELD_NAME, data).length > 0) { const value = get(field, data); return buildAlertSuppressionMissingFieldsDescription(label, value, ruleType); } else { diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/translations.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/translations.ts index 27dfec9818eb9..5c43b9181adcb 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/translations.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/translations.ts @@ -182,8 +182,8 @@ export const BUILDING_BLOCK_DESCRIPTION = i18n.translate( } ); -export const GROUP_BY_LABEL = i18n.translate( - 'xpack.securitySolution.detectionEngine.ruleDescription.groupByFieldsLabel', +export const ALERT_SUPPRESSION_LABEL = i18n.translate( + 'xpack.securitySolution.detectionEngine.ruleDescription.alertSuppressionFieldsLabel', { defaultMessage: 'Suppress alerts by', } diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/multi_select_fields/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/multi_select_fields/index.tsx index d38af219fe858..8a27d2f668094 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/multi_select_fields/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/multi_select_fields/index.tsx @@ -6,11 +6,9 @@ */ import React, { useMemo } from 'react'; - -import { EuiToolTip } from '@elastic/eui'; import type { DataViewFieldBase } from '@kbn/es-query'; +import { ComboBoxField } from '@kbn/es-ui-shared-plugin/static/forms/components'; import type { FieldHook } from '../../../../shared_imports'; -import { Field } from '../../../../shared_imports'; import { FIELD_PLACEHOLDER } from './translations'; interface MultiSelectAutocompleteProps { @@ -18,7 +16,6 @@ interface MultiSelectAutocompleteProps { isDisabled: boolean; field: FieldHook; fullWidth?: boolean; - disabledText?: string; dataTestSubj?: string; } @@ -28,7 +25,6 @@ const fieldDescribedByIds = 'detectionEngineMultiSelectAutocompleteField'; export const MultiSelectAutocompleteComponent: React.FC = ({ browserFields, - disabledText, isDisabled, field, fullWidth = false, @@ -46,21 +42,15 @@ export const MultiSelectAutocompleteComponent: React.FC ); - return isDisabled ? ( - - {fieldComponent} - - ) : ( - fieldComponent - ); }; export const MultiSelectFieldsAutocomplete = React.memo(MultiSelectAutocompleteComponent); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.test.tsx index bdbc01ada58ff..cc303731b26e3 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.test.tsx @@ -23,7 +23,7 @@ import type { } from '../../../../detections/pages/detection_engine/rules/types'; import { DataSourceType, - GroupByOptions, + AlertSuppressionDurationType, } from '../../../../detections/pages/detection_engine/rules/types'; import { fillEmptySeverityMappings } from '../../../../detections/pages/detection_engine/rules/helpers'; import { TestProviders } from '../../../../common/mock'; @@ -36,6 +36,16 @@ import { import type { FormHook } from '../../../../shared_imports'; import { useKibana as mockUseKibana } from '../../../../common/lib/kibana/__mocks__'; import { useKibana } from '../../../../common/lib/kibana'; +import { + ALERT_SUPPRESSION_DURATION_FIELD_NAME, + ALERT_SUPPRESSION_DURATION_TYPE_FIELD_NAME, + ALERT_SUPPRESSION_DURATION_UNIT_FIELD_NAME, + ALERT_SUPPRESSION_DURATION_VALUE_FIELD_NAME, + ALERT_SUPPRESSION_FIELDS_FIELD_NAME, + ALERT_SUPPRESSION_MISSING_FIELDS_FIELD_NAME, +} from '../../../rule_creation/components/alert_suppression_edit'; +import { THRESHOLD_ALERT_SUPPRESSION_ENABLED } from '../../../rule_creation/components/threshold_alert_suppression_edit'; +import { AlertSuppressionMissingFieldsStrategyEnum } from '../../../../../common/api/detection_engine'; jest.mock('../../../../common/lib/kibana'); jest.mock('../../../../common/containers/source'); @@ -69,16 +79,17 @@ export const stepDefineStepMLRule: DefineStepRule = { timeline: { id: null, title: null }, eqlOptions: {}, dataSourceType: DataSourceType.IndexPatterns, - groupByFields: ['host.name'], - groupByRadioSelection: GroupByOptions.PerRuleExecution, - groupByDuration: { - unit: 'm', - value: 5, + [ALERT_SUPPRESSION_FIELDS_FIELD_NAME]: ['host.name'], + [ALERT_SUPPRESSION_DURATION_TYPE_FIELD_NAME]: AlertSuppressionDurationType.PerRuleExecution, + [ALERT_SUPPRESSION_DURATION_FIELD_NAME]: { + [ALERT_SUPPRESSION_DURATION_VALUE_FIELD_NAME]: 5, + [ALERT_SUPPRESSION_DURATION_UNIT_FIELD_NAME]: 'm', }, + [ALERT_SUPPRESSION_MISSING_FIELDS_FIELD_NAME]: AlertSuppressionMissingFieldsStrategyEnum.suppress, + [THRESHOLD_ALERT_SUPPRESSION_ENABLED]: false, newTermsFields: ['host.ip'], historyWindowSize: '7d', shouldLoadQueryDynamically: false, - enableThresholdSuppression: false, }; describe('StepAboutRuleComponent', () => { diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.test.tsx index f1dcfc74e7923..50264fffabfb8 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.test.tsx @@ -9,7 +9,8 @@ import React, { useEffect, useState } from 'react'; import { screen, fireEvent, render, within, act, waitFor } from '@testing-library/react'; import type { Type as RuleType } from '@kbn/securitysolution-io-ts-alerting-types'; import type { DataViewBase } from '@kbn/es-query'; -import { StepDefineRule, aggregatableFields } from '.'; +import type { FieldSpec } from '@kbn/data-plugin/common'; +import { StepDefineRule } from '.'; import type { StepDefineRuleProps } from '.'; import { mockBrowserFields } from '../../../../common/containers/source/mock'; import { useRuleFromTimeline } from '../../../../detections/containers/detection_engine/rules/use_rule_from_timeline'; @@ -25,6 +26,22 @@ import { createIndexPatternField, getSelectToggleButtonForName, } from '../../../rule_creation/components/required_fields/required_fields.test'; +import { ALERT_SUPPRESSION_FIELDS_FIELD_NAME } from '../../../rule_creation/components/alert_suppression_edit'; +import { + expectDuration, + expectSuppressionFields, + setDuration, + setDurationType, + setSuppressionFields, +} from '../../../rule_creation/components/alert_suppression_edit/test_helpers'; +import { + selectEuiComboBoxOption, + selectFirstEuiComboBoxOption, +} from '../../../../common/test/eui/combobox'; +import { + addRelatedIntegrationRow, + setVersion, +} from '../../../rule_creation/components/related_integrations/test_helpers'; // Mocks integrations jest.mock('../../../fleet_integrations/api'); @@ -48,7 +65,13 @@ jest.mock('../ai_assistant', () => { }; }); -jest.mock('../data_view_selector_field/use_data_views'); +jest.mock('../data_view_selector_field/use_data_view_list_items'); + +jest.mock('../../../../common/hooks/use_license', () => ({ + useLicense: jest.fn().mockReturnValue({ + isAtLeast: jest.fn().mockReturnValue(true), + }), +})); const mockRedirectLegacyUrl = jest.fn(); const mockGetLegacyUrlConflict = jest.fn(); @@ -149,53 +172,6 @@ jest.mock('react-redux', () => { jest.mock('../../../../detections/containers/detection_engine/rules/use_rule_from_timeline'); -test('aggregatableFields', function () { - expect( - aggregatableFields([ - { - name: 'error.message', - type: 'string', - esTypes: ['text'], - searchable: true, - aggregatable: false, - readFromDocValues: false, - }, - ]) - ).toEqual([]); -}); - -test('aggregatableFields with aggregatable: true', function () { - expect( - aggregatableFields([ - { - name: 'error.message', - type: 'string', - esTypes: ['text'], - searchable: true, - aggregatable: false, - readFromDocValues: false, - }, - { - name: 'file.path', - type: 'string', - esTypes: ['keyword'], - searchable: true, - aggregatable: true, - readFromDocValues: false, - }, - ]) - ).toEqual([ - { - name: 'file.path', - type: 'string', - esTypes: ['keyword'], - searchable: true, - aggregatable: true, - readFromDocValues: false, - }, - ]); -}); - const mockUseRuleFromTimeline = useRuleFromTimeline as jest.Mock; const onOpenTimeline = jest.fn(); @@ -218,6 +194,62 @@ describe.skip('StepDefineRule', () => { expect(screen.getByTestId('stepDefineRule')).toBeDefined(); }); + describe('alert suppression', () => { + it('persists state when switching between custom query and threshold rule types', async () => { + const mockFields: FieldSpec[] = [ + { + name: 'test-field', + type: 'string', + searchable: false, + aggregatable: true, + }, + ]; + + const { rerender } = render( + , + { + wrapper: TestProviders, + } + ); + + await setSuppressionFields(['test-field']); + setDurationType('Per time period'); + setDuration(10, 'h'); + + // switch to threshold rule type + rerender( + + ); + + expectDuration(10, 'h'); + + // switch back to custom query rule type + rerender( + + ); + + expectSuppressionFields(['test-field']); + expectDuration(10, 'h'); + }); + }); + describe('related integrations', () => { beforeEach(() => { fleetIntegrationsApi.fetchAllIntegrations.mockResolvedValue({ @@ -631,13 +663,12 @@ function TestForm({ ruleType={ruleType} index={stepDefineDefaultValue.index} threatIndex={stepDefineDefaultValue.threatIndex} - groupByFields={stepDefineDefaultValue.groupByFields} + alertSuppressionFields={stepDefineDefaultValue[ALERT_SUPPRESSION_FIELDS_FIELD_NAME]} dataSourceType={stepDefineDefaultValue.dataSourceType} shouldLoadQueryDynamically={stepDefineDefaultValue.shouldLoadQueryDynamically} queryBarTitle="" queryBarSavedId="" thresholdFields={[]} - enableThresholdSuppression={false} {...formProps} />