Merge branch 'main' into slo/check-source-index-privileges

elastic · Nov 12, 2024 · ee0dd91 · ee0dd91
2 parents 6e344d9 + eea4a20
commit ee0dd91
Show file tree

Hide file tree

Showing 374 changed files with 5,219 additions and 6,025 deletions.
diff --git a/.eslintrc.js b/.eslintrc.js
@@ -1995,9 +1995,6 @@ module.exports = {
         // logsShared depends on o11y/private plugins, but platform plugins depend on it
         'x-pack/plugins/observability_solution/logs_shared/**',
 
-        // this plugin depends on visTypeTimeseries plugin (for TSVB viz) which is platform/private ATM
-        'x-pack/plugins/observability_solution/infra/**',
-
         // TODO @kibana/operations
         'scripts/create_observability_rules.js', // is importing "@kbn/observability-alerting-test-data" (observability/private)
         'src/cli_setup/**', // is importing "@kbn/interactive-setup-plugin" (platform/private)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -324,7 +324,7 @@ packages/kbn-custom-icons @elastic/obs-ux-logs-team
 packages/kbn-custom-integrations @elastic/obs-ux-logs-team
 packages/kbn-cypress-config @elastic/kibana-operations
 packages/kbn-data-service @elastic/kibana-visualizations @elastic/kibana-data-discovery
-packages/kbn-data-stream-adapter @elastic/security-threat-hunting-explore
+packages/kbn-data-stream-adapter @elastic/security-threat-hunting
 packages/kbn-data-view-utils @elastic/kibana-data-discovery
 packages/kbn-datemath @elastic/kibana-data-discovery
 packages/kbn-dev-cli-errors @elastic/kibana-operations
@@ -380,6 +380,7 @@ packages/kbn-i18n @elastic/kibana-core
 packages/kbn-i18n-react @elastic/kibana-core
 packages/kbn-import-locator @elastic/kibana-operations
 packages/kbn-import-resolver @elastic/kibana-operations
+packages/kbn-index-adapter @elastic/security-threat-hunting
 packages/kbn-interpreter @elastic/kibana-visualizations
 packages/kbn-investigation-shared @elastic/obs-ux-management-team
 packages/kbn-io-ts-utils @elastic/obs-knowledge-team

diff --git a/api_docs/kbn_elastic_assistant_common.devdocs.json b/api_docs/kbn_elastic_assistant_common.devdocs.json
@@ -997,7 +997,7 @@
           "\nInterface for features available to the elastic assistant"
         ],
         "signature": [
-          "{ readonly assistantKnowledgeBaseByDefault: boolean; readonly assistantModelEvaluation: boolean; }"
+          "{ readonly assistantModelEvaluation: boolean; }"
         ],
         "path": "x-pack/packages/kbn-elastic-assistant-common/impl/capabilities/index.ts",
         "deprecated": false,
@@ -2772,7 +2772,7 @@
         "label": "GetCapabilitiesResponse",
         "description": [],
         "signature": [
-          "{ assistantKnowledgeBaseByDefault: boolean; assistantModelEvaluation: boolean; }"
+          "{ assistantModelEvaluation: boolean; }"
         ],
         "path": "x-pack/packages/kbn-elastic-assistant-common/impl/schemas/capabilities/get_capabilities_route.gen.ts",
         "deprecated": false,
@@ -4767,7 +4767,7 @@
           "\nDefault features available to the elastic assistant"
         ],
         "signature": [
-          "{ readonly assistantKnowledgeBaseByDefault: true; readonly assistantModelEvaluation: false; }"
+          "{ readonly assistantModelEvaluation: false; }"
         ],
         "path": "x-pack/packages/kbn-elastic-assistant-common/impl/capabilities/index.ts",
         "deprecated": false,
@@ -5232,7 +5232,7 @@
         "label": "GetCapabilitiesResponse",
         "description": [],
         "signature": [
-          "Zod.ZodObject<{ assistantKnowledgeBaseByDefault: Zod.ZodBoolean; assistantModelEvaluation: Zod.ZodBoolean; }, \"strip\", Zod.ZodTypeAny, { assistantKnowledgeBaseByDefault: boolean; assistantModelEvaluation: boolean; }, { assistantKnowledgeBaseByDefault: boolean; assistantModelEvaluation: boolean; }>"
+          "Zod.ZodObject<{ assistantModelEvaluation: Zod.ZodBoolean; }, \"strip\", Zod.ZodTypeAny, { assistantModelEvaluation: boolean; }, { assistantModelEvaluation: boolean; }>"
         ],
         "path": "x-pack/packages/kbn-elastic-assistant-common/impl/schemas/capabilities/get_capabilities_route.gen.ts",
         "deprecated": false,
@@ -6201,4 +6201,4 @@
       }
     ]
   }
-}
+}
diff --git a/api_docs/security_solution.devdocs.json b/api_docs/security_solution.devdocs.json
diff --git a/docs/management/action-types.asciidoc b/docs/management/action-types.asciidoc
@@ -8,6 +8,12 @@ Actions are instantiations of a connector that are linked to rules and run as ba
 [cols="2"]
 |===
 
+// ifeval::["featureAIConnector"=="true"]
+// a| <<inference-action-type,{infer}>>
+
+// | Send a request to {infer}.
+// endif::[]
+
 a| <<bedrock-action-type,{bedrock}>>
 
 | Send a request to {bedrock}.
@@ -28,10 +34,6 @@ a| <<gemini-action-type,{gemini}>>
 
 | Send a request to {gemini}.
 
-a| <<inference-action-type,{infer}>>
-
-| Send a request to {infer}.
-
 a| <<email-action-type,Email>>
 
 | Send email from your server.

diff --git a/docs/management/connectors/action-types/inference.asciidoc b/docs/management/connectors/action-types/inference.asciidoc
@@ -1,13 +1,14 @@
 [[inference-action-type]]
-== {infer-cap} connector and action
+== AI connector and action
 ++++
-<titleabbrev>{infer-cap}</titleabbrev>
+<titleabbrev>AI</titleabbrev>
 ++++
 :frontmatter-description: Add a connector that can send requests to {inference}.
 :frontmatter-tags-products: [kibana] 
 :frontmatter-tags-content-type: [how-to] 
 :frontmatter-tags-user-goals: [configure]
 
+coming::[]
 
 The {infer} connector uses the {es} client to send requests to an {infer} service.
 The connector uses the <<execute-connector-api,run connector API>> to send the request.
@@ -16,6 +17,7 @@ The connector uses the <<execute-connector-api,run connector API>> to send the r
 [[define-inference-ui]]
 === Create connectors in {kib}
 
+// TBD After you set the `xpack.stack_connectors.enableExperimental` to include `inferenceConnectorOn`,
 You can create connectors in *{stack-manage-app} > {connectors-ui}*. For example:
 
 [role="screenshot"]

diff --git a/docs/management/connectors/index.asciidoc b/docs/management/connectors/index.asciidoc
@@ -4,7 +4,9 @@ include::action-types/crowdstrike.asciidoc[leveloffset=+1]
 include::action-types/d3security.asciidoc[leveloffset=+1]
 include::action-types/email.asciidoc[leveloffset=+1]
 include::action-types/gemini.asciidoc[leveloffset=+1]
-include::action-types/inference.asciidoc[leveloffset=+1]
+// ifeval::["featureAIConnector"=="true"]
+// include::action-types/inference.asciidoc[leveloffset=+1]
+// endif::[]
 include::action-types/resilient.asciidoc[leveloffset=+1]
 include::action-types/index.asciidoc[leveloffset=+1]
 include::action-types/jira.asciidoc[leveloffset=+1]

diff --git a/docs/settings/alert-action-settings.asciidoc b/docs/settings/alert-action-settings.asciidoc
@@ -269,15 +269,16 @@ A configuration URL that varies by connector:
 --
 * For an <<bedrock-action-type,{bedrock} connector>>, specifies the {bedrock} request URL.
 * For an <<gemini-action-type,{gemini} connector>>, specifies the {gemini} request URL.
-* For an <<inference-action-type,{inference} connector>>, specifies the Elastic {inference} request.
 * For a <<openai-action-type,OpenAI connector>>, specifies the OpenAI request URL.
 * For a <<resilient-action-type,{ibm-r} connector>>, specifies the {ibm-r} instance URL.
 * For a <<jira-action-type,Jira connector>>, specifies the Jira instance URL.
 * For an <<opsgenie-action-type,{opsgenie} connector>>, specifies the {opsgenie} URL. For example, `https://api.opsgenie.com` or `https://api.eu.opsgenie.com`.
 * For a <<pagerduty-action-type,PagerDuty connector>>, specifies the PagerDuty event URL. Defaults to `https://events.pagerduty.com/v2/enqueue`.
 * For a <<servicenow-action-type,{sn-itsm}>>, <<servicenow-sir-action-type,{sn-sir}>>, or <<servicenow-itom-action-type,{sn-itom} connector>> specifies the ServiceNow instance URL.
 * For a <<swimlane-action-type,{swimlane} connector>>, specifies the {swimlane} instance URL.
-
+// ifeval::["featureAIConnector"=="true"]
+// * For an <<inference-action-type,AI connector>>, specifies the Elastic {inference} request.
+// endif::[]
 NOTE: If you are using the `xpack.actions.allowedHosts` setting, make sure the hostname in the URL is added to the allowed hosts.
 --
 

diff --git a/oas_docs/examples/get_connector_types_generativeai_response.yaml b/oas_docs/examples/get_connector_types_generativeai_response.yaml
@@ -31,14 +31,3 @@ value:
     supported_feature_ids:
       - generativeAIForSecurity
     is_system_action_type: false
-  - id: .inference
-    name: Inference API
-    enabled: true
-    enabled_in_config: true
-    enabled_in_license: true
-    minimum_license_required: enterprise
-    supported_feature_ids:
-      - generativeAIForSecurity
-      - generativeAIForObservability
-      - generativeAIForSearchPlayground
-    is_system_action_type: false
diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml
@@ -39976,17 +39976,20 @@ components:
       type: object
       properties:
         unit:
-          enum:
-            - s
-            - m
-            - h
-          type: string
+          $ref: >-
+            #/components/schemas/Security_Detections_API_AlertSuppressionDurationUnit
         value:
           minimum: 1
           type: integer
       required:
         - value
         - unit
+    Security_Detections_API_AlertSuppressionDurationUnit:
+      enum:
+        - s
+        - m
+        - h
+      type: string
     Security_Detections_API_AlertSuppressionGroupBy:
       items:
         type: string

diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml
@@ -48220,17 +48220,20 @@ components:
       type: object
       properties:
         unit:
-          enum:
-            - s
-            - m
-            - h
-          type: string
+          $ref: >-
+            #/components/schemas/Security_Detections_API_AlertSuppressionDurationUnit
         value:
           minimum: 1
           type: integer
       required:
         - value
         - unit
+    Security_Detections_API_AlertSuppressionDurationUnit:
+      enum:
+        - s
+        - m
+        - h
+      type: string
     Security_Detections_API_AlertSuppressionGroupBy:
       items:
         type: string

diff --git a/oas_docs/overlays/connectors.overlays.yaml b/oas_docs/overlays/connectors.overlays.yaml
@@ -140,6 +140,8 @@ actions:
                   default: {}
                   description: The connector configuration details.
                   oneOf:
+                    # AI (.inference) TBD
+                    # - $ref: '../../x-pack/plugins/actions/docs/openapi/components/schemas/inference_config.yaml'
                     # Bedrock (.bedrock)
                     - $ref: '../../x-pack/plugins/actions/docs/openapi/components/schemas/bedrock_config.yaml'
                     # Crowdstrike (.crowdstrike)
@@ -194,6 +196,8 @@ actions:
                   additionalProperties: {}
                   default: {}
                   oneOf:
+                    # AI (.inference)
+                    # - $ref: '../../x-pack/plugins/actions/docs/openapi/components/schemas/inference_secrets.yaml'
                     # Bedrock (.bedrock)
                     - $ref: '../../x-pack/plugins/actions/docs/openapi/components/schemas/bedrock_secrets.yaml'
                     # Crowdstrike (.crowdstrike)
@@ -253,6 +257,8 @@ actions:
                   default: {}
                   description: The connector configuration details.
                   oneOf:
+                    # AI (.inference)
+                    # - $ref: '../../x-pack/plugins/actions/docs/openapi/components/schemas/inference_config.yaml'
                     # Bedrock (.bedrock)
                     - $ref: '../../x-pack/plugins/actions/docs/openapi/components/schemas/bedrock_config.yaml'
                     # Crowdstrike (.crowdstrike)
@@ -307,6 +313,8 @@ actions:
                   additionalProperties: {}
                   default: {}
                   oneOf:
+                    # AI (.inference)
+                    # - $ref: '../../x-pack/plugins/actions/docs/openapi/components/schemas/inference_secrets.yaml'
                     # Bedrock (.bedrock)
                     - $ref: '../../x-pack/plugins/actions/docs/openapi/components/schemas/bedrock_secrets.yaml'
                     # Crowdstrike (.crowdstrike)

diff --git a/package.json b/package.json
@@ -567,6 +567,7 @@
     "@kbn/i18n-react": "link:packages/kbn-i18n-react",
     "@kbn/iframe-embedded-plugin": "link:x-pack/test/functional_embedded/plugins/iframe_embedded",
     "@kbn/image-embeddable-plugin": "link:src/plugins/image_embeddable",
+    "@kbn/index-adapter": "link:packages/kbn-index-adapter",
     "@kbn/index-lifecycle-management-common-shared": "link:x-pack/packages/index-lifecycle-management/index_lifecycle_management_common_shared",
     "@kbn/index-lifecycle-management-plugin": "link:x-pack/plugins/index_lifecycle_management",
     "@kbn/index-management-plugin": "link:x-pack/plugins/index_management",

diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts
@@ -58,6 +58,8 @@ export class K8sEntity extends Serializable<EntityFields> {
       'entity.definition_id': `builtin_${entityTypeWithSchema}`,
       'entity.identity_fields': identityFields,
       'entity.display_name': getDisplayName({ identityFields, fields }),
+      'entity.definition_version': '1.0.0',
+      'entity.schema_version': '1.0',
     });
   }
 }

diff --git a/packages/kbn-data-stream-adapter/index.ts b/packages/kbn-data-stream-adapter/index.ts
@@ -9,13 +9,13 @@
 
 export { DataStreamAdapter } from './src/data_stream_adapter';
 export { DataStreamSpacesAdapter } from './src/data_stream_spaces_adapter';
-export { retryTransientEsErrors } from './src/retry_transient_es_errors';
-export { ecsFieldMap, type EcsFieldMap } from './src/field_maps/ecs_field_map';
 
+export { retryTransientEsErrors, ecsFieldMap } from '@kbn/index-adapter';
 export type {
-  DataStreamAdapterParams,
   SetComponentTemplateParams,
   SetIndexTemplateParams,
   InstallParams,
-} from './src/data_stream_adapter';
-export * from './src/field_maps/types';
+  EcsFieldMap,
+} from '@kbn/index-adapter';
+
+export * from '@kbn/index-adapter/src/field_maps/types';
diff --git a/packages/kbn-data-stream-adapter/kibana.jsonc b/packages/kbn-data-stream-adapter/kibana.jsonc
@@ -1,5 +1,6 @@
 {
-  "type": "shared-common",
+  "type": "shared-server",
   "id": "@kbn/data-stream-adapter",
-  "owner": "@elastic/security-threat-hunting-explore"
+  "owner": "@elastic/security-threat-hunting",
+  "visibility": "shared"
 }
diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.test.ts b/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.test.ts
@@ -136,10 +136,11 @@ describe('createOrUpdateDataStream', () => {
   it(`should create data stream if not exists`, async () => {
     esClient.indices.getDataStream.mockResolvedValueOnce({ data_streams: [] });
 
-    await createDataStream({
+    await createOrUpdateDataStream({
       esClient,
       logger,
       name,
+      totalFieldsLimit,
     });
 
     expect(esClient.indices.createDataStream).toHaveBeenCalledWith({ name });

diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts b/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts
@@ -11,7 +11,7 @@ import type { IndicesDataStream } from '@elastic/elasticsearch/lib/api/types';
 import type { IndicesSimulateIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import type { Logger, ElasticsearchClient } from '@kbn/core/server';
 import { get } from 'lodash';
-import { retryTransientEsErrors } from './retry_transient_es_errors';
+import { retryTransientEsErrors } from '@kbn/index-adapter';
 
 interface UpdateIndexMappingsOpts {
   logger: Logger;
@@ -168,7 +168,7 @@ export async function createDataStream({
   esClient,
   name,
 }: CreateDataStreamParams): Promise<void> {
-  logger.info(`Creating data stream - ${name}`);
+  logger.debug(`Checking data stream exists - ${name}`);
 
   // check if data stream exists
   let dataStreamExists = false;
@@ -189,6 +189,7 @@ export async function createDataStream({
   if (dataStreamExists) {
     return;
   }
+  logger.info(`Installing data stream - ${name}`);
 
   try {
     await retryTransientEsErrors(() => esClient.indices.createDataStream({ name }), { logger });