Merge branch '8.11' into backport/8.11/pr-170499

.buildkite/pipelines/security_solution/api_integration.yml

@@ -0,0 +1,67 @@
+steps:
+  - label: Running exception_workflows:runner:serverless
+    command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh exception_workflows:qa:serverless
+    key: exception_workflows:runner:serverless
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 2
+
+  - label: Running exception_operators_date_numeric_types:runner:serverless
+    command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh exception_operators_date_numeric_types:qa:serverless
+    key: exception_operators_date_numeric_types:runner:serverless
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 120  
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 2
+
+  - label: Running exception_operators_keyword_text_long:runner:serverless
+    command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh exception_operators_keyword_text_long:qa:serverless
+    key: exception_operators_keyword_text_long:runner:serverless
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 2
+
+  - label: Running exception_operators_ips_text_array:runner:serverless
+    command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh exception_operators_ips_text_array:qa:serverless
+    key: exception_operators_ips_text_array:runner:serverless
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '1'
+          limit: 2
+
+  - label: Running rule_creation:runner:serverless
+    command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh rule_creation:qa:serverless
+    key: rule_creation:runner:serverless
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '1'
+          limit: 2
+
+  - label: Running actions:qa:serverless
+    command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh actions:qa:serverless
+    key: actions:qa:serverless
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '1'
+          limit: 2
+

.buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+if [ -z "$1" ]
+  then
+    echo "No target script from the package.json file, is supplied"
+    exit 1
+fi
+
+source .buildkite/scripts/common/util.sh
+.buildkite/scripts/bootstrap.sh
+
+buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true"
+
+echo "--- Serverless Security Second Quality Gate"
+cd x-pack/test/security_solution_api_integration
+set +e
+
+QA_API_KEY=$(retry 5 5 vault read -field=qa_api_key secret/kibana-issues/dev/security-solution-qg-enc-key)
+
+# Generate a random 5-digit number
+random_number=$((10000 + $RANDOM % 90000))
+ENVIRONMENT_DETAILS=$(curl --location 'https://global.qa.cld.elstc.co/api/v1/serverless/projects/security' \
+    --header "Authorization: ApiKey $QA_API_KEY" \
+    --header 'Content-Type: application/json' \
+    --data '{
+        "name": "ftr-integration-tests-'$random_number'",
+        "region_id": "aws-eu-west-1"}' | jq '.')
+NAME=$(echo $ENVIRONMENT_DETAILS | jq -r '.name')
+ID=$(echo $ENVIRONMENT_DETAILS | jq -r '.id')
+ES_URL=$(echo $ENVIRONMENT_DETAILS | jq -r '.endpoints.elasticsearch')
+KB_URL=$(echo $ENVIRONMENT_DETAILS | jq -r '.endpoints.kibana')
+
+# Wait five seconds for the project to appear
+sleep 5
+
+# Resetting the credentials of the elastic user in the project
+CREDS_BODY=$(curl -s --location --request POST "https://global.qa.cld.elstc.co/api/v1/serverless/projects/security/$ID/_reset-credentials" \
+  --header "Authorization: ApiKey $QA_API_KEY" \
+  --header 'Content-Type: application/json' | jq '.')
+USERNAME=$(echo $CREDS_BODY | jq -r '.username')
+PASSWORD=$(echo $CREDS_BODY | jq -r '.password')
+AUTH=$(echo "$USERNAME:$PASSWORD")
+
+# Checking if Elasticsearch has status green
+while : ; do
+  STATUS=$(curl -u $AUTH --location "$ES_URL:443/_cluster/health?wait_for_status=green&timeout=50s" | jq -r '.status')
+  if [ "$STATUS" != "green" ]; then
+    echo "Sleeping for 40s to wait for ES status to be green..."
+    sleep 40
+  else
+    echo "Elasticsearch has status green." 
+    break
+  fi
+done
+
+# Checking if Kibana is available
+while : ; do
+  STATUS=$(curl -u $AUTH --location "$KB_URL:443/api/status" | jq -r '.status.overall.level')
+  if [ "$STATUS" != "available" ]; then
+    echo "Sleeping for 15s to wait for Kibana to be available..."
+    sleep 15
+  else
+    echo "Kibana is available." 
+    break
+  fi
+done
+
+# Removing the https:// part of the url provided in order to use it in the command below.
+FORMATTED_ES_URL="${ES_URL/https:\/\//}"    
+FORMATTED_KB_URL="${KB_URL/https:\/\//}"
+
+# Find a way to remove this in the future
+# This is used in order to wait for the environment to be ready. 
+sleep 150
+
+TEST_CLOUD=1 TEST_ES_URL="https://elastic:$PASSWORD@$FORMATTED_ES_URL:443" TEST_KIBANA_URL="https://elastic:$PASSWORD@$FORMATTED_KB_URL:443" yarn run $1
+cmd_status=$?
+echo "Exit code with status: $cmd_status"
+
+curl --location --request DELETE "https://global.qa.cld.elstc.co/api/v1/serverless/projects/security/$ID" \
+  --header "Authorization: ApiKey $QA_API_KEY"
+
+exit $cmd_status

.buildkite/scripts/steps/artifacts/docker_context.sh

@@ -11,7 +11,7 @@ KIBANA_DOCKER_CONTEXT="${KIBANA_DOCKER_CONTEXT:="default"}"
 
 echo "--- Create contexts"
 mkdir -p target
-node scripts/build --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact
+node scripts/build --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact "${BUILD_ARGS[@]}"
 
 echo "--- Setup context"
 DOCKER_BUILD_FOLDER=$(mktemp -d)

docs/CHANGELOG.asciidoc

@@ -132,6 +132,7 @@ Alerting::
 * Adds support for the new ES|QL language for {es} query rules ({kibana-pull}165973[#165973]).
 * Elasticsearch query rule can select multiple group-by terms ({kibana-pull}166146[#166146]).
 * Adds a Log tab to the Observability Rules page ({kibana-pull}165115[#165115]).
+* Adds a new Observability Rules type called Custom threshold under technical preview ({kibana-pull}167782[#167782]).
 APM::
 * Adds bulk action to untrack selected alerts ({kibana-pull}167579[#167579]).
 * Introduce custom dashboards tab in service overview ({kibana-pull}166789[#166789]).

package.json

@@ -101,7 +101,7 @@
     "@elastic/charts": "60.0.0",
     "@elastic/datemath": "5.0.3",
     "@elastic/elasticsearch": "npm:@elastic/[email protected]",
-    "@elastic/ems-client": "8.4.0",
+    "@elastic/ems-client": "8.4.1",
     "@elastic/eui": "88.5.0",
     "@elastic/filesaver": "1.1.2",
     "@elastic/node-crypto": "1.2.1",

src/dev/license_checker/config.ts

@@ -84,7 +84,7 @@ export const PER_PACKAGE_ALLOWED_LICENSES = {
 export const LICENSE_OVERRIDES = {
   '[email protected]': ['Eclipse Distribution License - v 1.0'], // cf. https://github.com/bjornharrtell/jsts
   '@mapbox/[email protected]': ['MIT'], // license in readme https://github.com/tmcw/jsonlint
-  '@elastic/[email protected].0': ['Elastic License 2.0'],
+  '@elastic/[email protected].1': ['Elastic License 2.0'],
   '@elastic/[email protected]': ['SSPL-1.0 OR Elastic License 2.0'],
   '[email protected]': ['CC-BY-4.0'], // retired ODC‑By license https://github.com/mattcg/language-subtag-registry
   '[email protected]': ['MIT'], // license in importing module https://www.npmjs.com/package/binary

x-pack/plugins/security_solution/common/search_strategy/security_solution/risk_score/all/index.ts

@@ -74,6 +74,8 @@ export interface RiskScoreItem {
   [RiskScoreFields.hostName]: Maybe<string>;
   [RiskScoreFields.userName]: Maybe<string>;
 
+  [RiskScoreFields.timestamp]: Maybe<string>;
+
   [RiskScoreFields.hostRisk]: Maybe<RiskSeverity>;
   [RiskScoreFields.userRisk]: Maybe<RiskSeverity>;
 

x-pack/plugins/security_solution/common/search_strategy/security_solution/users/common/index.ts

@@ -11,6 +11,7 @@ import type { CommonFields, Maybe, RiskScoreFields, RiskSeverity, SortField } fr
 export interface UserRiskScoreItem {
   _id?: Maybe<string>;
   [RiskScoreFields.userName]: Maybe<string>;
+  [RiskScoreFields.timestamp]: Maybe<string>;
   [RiskScoreFields.userRisk]: Maybe<RiskSeverity>;
   [RiskScoreFields.userRiskScore]: Maybe<number>;
 }

x-pack/plugins/security_solution/public/explore/components/risk_score/risk_score_onboarding/risk_score_no_data_detected.tsx

@@ -14,7 +14,6 @@ import * as i18n from './translations';
 import { RiskScoreHeaderTitle } from './risk_score_header_title';
 import { RiskScoreRestartButton } from './risk_score_restart_button';
 import type { inputsModel } from '../../../../common/store';
-import * as overviewI18n from '../../../../overview/components/entity_analytics/common/translations';
 import { useIsNewRiskScoreModuleInstalled } from '../../../../entity_analytics/api/hooks/use_risk_engine_status';
 
 const RiskScoresNoDataDetectedComponent = ({
@@ -37,15 +36,7 @@ const RiskScoresNoDataDetectedComponent = ({
 
   return (
     <EuiPanel data-test-subj={`${entityType}-risk-score-no-data-detected`} hasBorder>
-      <HeaderSection
-        title={<RiskScoreHeaderTitle riskScoreEntity={entityType} />}
-        titleSize="s"
-        tooltip={
-          entityType === RiskScoreEntity.user
-            ? overviewI18n.USER_RISK_TABLE_TOOLTIP
-            : overviewI18n.HOST_RISK_TABLE_TOOLTIP
-        }
-      />
+      <HeaderSection title={<RiskScoreHeaderTitle riskScoreEntity={entityType} />} titleSize="s" />
       <EuiEmptyPrompt
         title={<h2>{translations.title}</h2>}
         body={translations.body}

x-pack/plugins/security_solution/public/explore/containers/risk_score/all/index.tsx

@@ -125,13 +125,6 @@ export const useRiskScore = <T extends RiskScoreEntity.host | RiskScoreEntity.us
     }
   }, [defaultIndex, refetch, refetchDeprecated]);
 
-  // since query does not take timerange arg, we need to manually refetch when time range updates
-  // the results can be different if the user has run the ML for the first time since pressing refresh
-  useEffect(() => {
-    refetchAll();
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [timerange?.to, timerange?.from]);
-
   const riskScoreResponse = useMemo(
     () => ({
       data: response.data,
@@ -168,7 +161,7 @@ export const useRiskScore = <T extends RiskScoreEntity.host | RiskScoreEntity.us
                   }
                 : undefined,
             sort,
-            timerange: onlyLatest ? undefined : requestTimerange,
+            timerange: requestTimerange,
             alertsTimerange: includeAlertsCount ? requestTimerange : undefined,
           }
         : null,
@@ -180,7 +173,6 @@ export const useRiskScore = <T extends RiskScoreEntity.host | RiskScoreEntity.us
       querySize,
       sort,
       requestTimerange,
-      onlyLatest,
       riskEntity,
       includeAlertsCount,
     ]

x-pack/plugins/security_solution/public/explore/containers/risk_score/kpi/index.tsx

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { useEffect, useMemo } from 'react';
+import { useCallback, useEffect, useMemo } from 'react';
 
 import {
   getHostRiskIndex,
@@ -20,12 +20,12 @@ import { isIndexNotFoundError } from '../../../../common/utils/exceptions';
 import type { ESQuery } from '../../../../../common/typed_json';
 import type { SeverityCount } from '../../../components/risk_score/severity/types';
 import { useSpaceId } from '../../../../common/hooks/use_space_id';
-import { useMlCapabilities } from '../../../../common/components/ml/hooks/use_ml_capabilities';
 import { useSearchStrategy } from '../../../../common/containers/use_search_strategy';
 import type { InspectResponse } from '../../../../types';
 import type { inputsModel } from '../../../../common/store';
 import { useAppToasts } from '../../../../common/hooks/use_app_toasts';
 import { useIsNewRiskScoreModuleInstalled } from '../../../../entity_analytics/api/hooks/use_risk_engine_status';
+import { useRiskScoreFeatureStatus } from '../feature_status';
 
 interface RiskScoreKpi {
   error: unknown;
@@ -52,14 +52,22 @@ export const useRiskScoreKpi = ({
 }: UseRiskScoreKpiProps): RiskScoreKpi => {
   const { addError } = useAppToasts();
   const spaceId = useSpaceId();
-  const featureEnabled = useMlCapabilities().isPlatinumOrTrialLicense;
+
   const isNewRiskScoreModuleInstalled = useIsNewRiskScoreModuleInstalled();
   const defaultIndex = spaceId
     ? riskEntity === RiskScoreEntity.host
       ? getHostRiskIndex(spaceId, true, isNewRiskScoreModuleInstalled)
       : getUserRiskIndex(spaceId, true, isNewRiskScoreModuleInstalled)
     : undefined;
 
+  const {
+    isDeprecated,
+    isEnabled,
+    isAuthorized,
+    isLoading: isDeprecatedLoading,
+    refetch: refetchFeatureStatus,
+  } = useRiskScoreFeatureStatus(riskEntity, defaultIndex);
+
   const { loading, result, search, refetch, inspect, error } =
     useSearchStrategy<RiskQueries.kpiRiskScore>({
       factoryQueryType: RiskQueries.kpiRiskScore,
@@ -72,21 +80,46 @@ export const useRiskScoreKpi = ({
 
   const isModuleDisabled = !!error && isIndexNotFoundError(error);
 
+  const requestTimerange = useMemo(
+    () => (timerange ? { to: timerange.to, from: timerange.from, interval: '' } : undefined),
+    [timerange]
+  );
+
   useEffect(() => {
-    if (!skip && defaultIndex && featureEnabled) {
+    if (
+      !skip &&
+      !isDeprecatedLoading &&
+      defaultIndex &&
+      isAuthorized &&
+      isEnabled &&
+      !isDeprecated
+    ) {
       search({
         filterQuery,
         defaultIndex: [defaultIndex],
         entity: riskEntity,
+        timerange: requestTimerange,
       });
     }
-  }, [defaultIndex, search, filterQuery, skip, riskEntity, featureEnabled]);
+  }, [
+    defaultIndex,
+    search,
+    filterQuery,
+    skip,
+    riskEntity,
+    requestTimerange,
+    isEnabled,
+    isDeprecated,
+    isDeprecatedLoading,
+    isAuthorized,
+  ]);
 
-  // since query does not take timerange arg, we need to manually refetch when time range updates
-  useEffect(() => {
-    refetch();
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [timerange?.to, timerange?.from]);
+  const refetchAll = useCallback(() => {
+    if (defaultIndex) {
+      refetchFeatureStatus(defaultIndex);
+      refetch();
+    }
+  }, [defaultIndex, refetch, refetchFeatureStatus]);
 
   useEffect(() => {
     if (error) {
@@ -110,5 +143,5 @@ export const useRiskScoreKpi = ({
     };
   }, [result, loading, error]);
 
-  return { error, severityCount, loading, isModuleDisabled, refetch, inspect };
+  return { error, severityCount, loading, isModuleDisabled, refetch: refetchAll, inspect };
 };

x-pack/plugins/security_solution/public/explore/hosts/components/host_risk_score_table/columns.test.tsx

@@ -17,7 +17,7 @@ describe('getHostRiskScoreColumns', () => {
     });
 
     const riskScore = 10.11111111;
-    const riskScoreColumn = columns[1];
+    const riskScoreColumn = columns[2];
     const renderedColumn = riskScoreColumn.render!(riskScore, null);
 
     const { queryByTestId } = render(<TestProviders>{renderedColumn}</TestProviders>);