Merge branch 'main' into renovate/main-minify

.buildkite/scripts/common/env.sh

@@ -146,7 +146,7 @@ if [[ "${KBN_ENABLE_FIPS:-}" == "true" ]] || is_pr_with_label "ci:enable-fips-ag
   fi
 
   if [[ -f "$KIBANA_DIR/config/kibana.yml" ]]; then
-    echo -e '\nxpack.security.experimental.fipsMode.enabled: true' >>"$KIBANA_DIR/config/kibana.yml"
+    echo -e '\nxpack.security.fipsMode.enabled: true' >>"$KIBANA_DIR/config/kibana.yml"
   fi
 fi
 

.devcontainer/scripts/env.sh

@@ -9,7 +9,7 @@ setup_fips() {
   fi
 
   if [ -n "$FIPS" ] && [ "$FIPS" = "1" ]; then
-    sed -i '/xpack.security.experimental.fipsMode.enabled:/ {s/.*/xpack.security.experimental.fipsMode.enabled: true/; t}; $a\xpack.security.experimental.fipsMode.enabled: true' "$KBN_CONFIG_FILE"
+    sed -i '/xpack.security.fipsMode.enabled:/ {s/.*/xpack.security.fipsMode.enabled: true/; t}; $a\xpack.security.fipsMode.enabled: true' "$KBN_CONFIG_FILE"
 
     # Patch node_modules so we can start Kibana in dev mode
     sed -i 's/hashType = hashType || '\''md5'\'';/hashType = hashType || '\''sha1'\'';/g' "${KBN_DIR}/node_modules/file-loader/node_modules/loader-utils/lib/getHashDigest.js"
@@ -21,7 +21,7 @@ setup_fips() {
     echo "FIPS mode enabled"
     echo "If manually bootstrapping in FIPS mode use: NODE_OPTIONS='' yarn kbn bootstrap"
   else
-    sed -i '/xpack.security.experimental.fipsMode.enabled:/ {s/.*/xpack.security.experimental.fipsMode.enabled: false/; t}; $a\xpack.security.experimental.fipsMode.enabled: false' "$KBN_CONFIG_FILE"
+    sed -i '/xpack.security.fipsMode.enabled:/ {s/.*/xpack.security.fipsMode.enabled: false/; t}; $a\xpack.security.fipsMode.enabled: false' "$KBN_CONFIG_FILE"
   fi
 }
 

docs/user/security/fips-140-2.asciidoc

@@ -29,7 +29,7 @@ For {kib}, adherence to FIPS 140-2 is ensured by:
 
 ==== Configuring {kib} for FIPS 140-2
 
-Apart from setting `xpack.security.experimental.fipsMode.enabled` to `true` in your {kib} config, a number of security related 
+Apart from setting `xpack.security.fipsMode.enabled` to `true` in your {kib} config, a number of security related 
 settings need to be reviewed and configured in order to run {kib} successfully in a FIPS 140-2 compliant Node.js 
 environment.
 
@@ -56,8 +56,3 @@ As an example, avoid PKCS#12 specific settings such as:
 * `server.ssl.truststore.path`
 * `elasticsearch.ssl.keystore.path`
 * `elasticsearch.ssl.truststore.path`
-
-===== Limitations
-
-Configuring {kib} to run in FIPS mode is still considered to be experimental. Not all features are guaranteed to
-function as expected.

package.json

@@ -104,6 +104,7 @@
     "@appland/sql-parser": "^1.5.1",
     "@aws-crypto/sha256-js": "^5.2.0",
     "@aws-crypto/util": "^5.2.0",
+    "@aws-sdk/client-bedrock-runtime": "^3.687.0",
     "@babel/runtime": "^7.24.7",
     "@dagrejs/dagre": "^1.1.4",
     "@dnd-kit/core": "^6.1.0",
@@ -1019,7 +1020,8 @@
     "@kbn/xstate-utils": "link:packages/kbn-xstate-utils",
     "@kbn/zod": "link:packages/kbn-zod",
     "@kbn/zod-helpers": "link:packages/kbn-zod-helpers",
-    "@langchain/community": "0.3.11",
+    "@langchain/aws": "^0.1.2",
+    "@langchain/community": "0.3.14",
     "@langchain/core": "^0.3.16",
     "@langchain/google-common": "^0.1.1",
     "@langchain/google-genai": "^0.1.2",
@@ -1054,7 +1056,9 @@
     "@slack/webhook": "^7.0.1",
     "@smithy/eventstream-codec": "^3.1.1",
     "@smithy/eventstream-serde-node": "^3.0.3",
-    "@smithy/protocol-http": "^4.0.2",
+    "@smithy/middleware-stack": "^3.0.10",
+    "@smithy/node-http-handler": "^3.3.1",
+    "@smithy/protocol-http": "^4.1.7",
     "@smithy/signature-v4": "^3.1.1",
     "@smithy/types": "^3.2.0",
     "@smithy/util-utf8": "^3.0.0",

packages/core/security/core-security-server-internal/src/fips/fips.test.ts

@@ -25,26 +25,26 @@ import { loggingSystemMock } from '@kbn/core-logging-server-mocks';
 describe('fips', () => {
   let securityConfig: SecurityServiceConfigType;
   describe('#isFipsEnabled', () => {
-    it('should return `true` if config.experimental.fipsMode.enabled is `true`', () => {
-      securityConfig = { experimental: { fipsMode: { enabled: true } } };
+    it('should return `true` if config.fipsMode.enabled is `true`', () => {
+      securityConfig = { fipsMode: { enabled: true } };
 
       expect(isFipsEnabled(securityConfig)).toBe(true);
     });
 
-    it('should return `false` if config.experimental.fipsMode.enabled is `false`', () => {
-      securityConfig = { experimental: { fipsMode: { enabled: false } } };
+    it('should return `false` if config.fipsMode.enabled is `false`', () => {
+      securityConfig = { fipsMode: { enabled: false } };
 
       expect(isFipsEnabled(securityConfig)).toBe(false);
     });
 
-    it('should return `false` if config.experimental.fipsMode.enabled is `undefined`', () => {
+    it('should return `false` if config.fipsMode.enabled is `undefined`', () => {
       expect(isFipsEnabled(securityConfig)).toBe(false);
     });
   });
 
   describe('checkFipsConfig', () => {
-    it('should log an error message if FIPS mode is misconfigured - xpack.security.experimental.fipsMode.enabled true, Nodejs FIPS mode false', async () => {
-      securityConfig = { experimental: { fipsMode: { enabled: true } } };
+    it('should log an error message if FIPS mode is misconfigured - xpack.security.fipsMode.enabled true, Nodejs FIPS mode false', async () => {
+      securityConfig = { fipsMode: { enabled: true } };
       const logger = loggingSystemMock.create().get();
       let fipsException: undefined | CriticalError;
       try {
@@ -56,16 +56,16 @@ describe('fips', () => {
       expect(fipsException).toBeInstanceOf(CriticalError);
       expect(fipsException!.processExitCode).toBe(78);
       expect(fipsException!.message).toEqual(
-        'Configuration mismatch error. xpack.security.experimental.fipsMode.enabled is set to true and the configured Node.js environment has FIPS disabled'
+        'Configuration mismatch error. xpack.security.fipsMode.enabled is set to true and the configured Node.js environment has FIPS disabled'
       );
     });
 
-    it('should log an error message if FIPS mode is misconfigured - xpack.security.experimental.fipsMode.enabled false, Nodejs FIPS mode true', async () => {
+    it('should log an error message if FIPS mode is misconfigured - xpack.security.fipsMode.enabled false, Nodejs FIPS mode true', async () => {
       mockGetFipsFn.mockImplementationOnce(() => {
         return 1;
       });
 
-      securityConfig = { experimental: { fipsMode: { enabled: false } } };
+      securityConfig = { fipsMode: { enabled: false } };
       const logger = loggingSystemMock.create().get();
 
       let fipsException: undefined | CriticalError;
@@ -77,16 +77,16 @@ describe('fips', () => {
       expect(fipsException).toBeInstanceOf(CriticalError);
       expect(fipsException!.processExitCode).toBe(78);
       expect(fipsException!.message).toEqual(
-        'Configuration mismatch error. xpack.security.experimental.fipsMode.enabled is set to false and the configured Node.js environment has FIPS enabled'
+        'Configuration mismatch error. xpack.security.fipsMode.enabled is set to false and the configured Node.js environment has FIPS enabled'
       );
     });
 
-    it('should log an info message if FIPS mode is properly configured - xpack.security.experimental.fipsMode.enabled true, Nodejs FIPS mode true', async () => {
+    it('should log an info message if FIPS mode is properly configured - xpack.security.fipsMode.enabled true, Nodejs FIPS mode true', async () => {
       mockGetFipsFn.mockImplementationOnce(() => {
         return 1;
       });
 
-      securityConfig = { experimental: { fipsMode: { enabled: true } } };
+      securityConfig = { fipsMode: { enabled: true } };
       const logger = loggingSystemMock.create().get();
 
       try {
@@ -113,7 +113,7 @@ describe('fips', () => {
           return 1;
         });
 
-        securityConfig = { experimental: { fipsMode: { enabled: true } } };
+        securityConfig = { fipsMode: { enabled: true } };
       });
 
       afterEach(function () {

packages/core/security/core-security-server-internal/src/fips/fips.ts

@@ -12,7 +12,7 @@ import { getFips } from 'crypto';
 import { CriticalError } from '@kbn/core-base-server-internal';
 import { PKCS12ConfigType, SecurityServiceConfigType } from '../utils';
 export function isFipsEnabled(config: SecurityServiceConfigType): boolean {
-  return config?.experimental?.fipsMode?.enabled ?? false;
+  return config?.fipsMode?.enabled ?? false;
 }
 
 export function checkFipsConfig(
@@ -33,7 +33,7 @@ export function checkFipsConfig(
     // FIPS must be enabled on both, or, log/error an exit Kibana
     if (isFipsConfigEnabled !== isNodeRunningWithFipsEnabled) {
       throw new CriticalError(
-        `Configuration mismatch error. xpack.security.experimental.fipsMode.enabled is set to ${isFipsConfigEnabled} and the configured Node.js environment has FIPS ${
+        `Configuration mismatch error. xpack.security.fipsMode.enabled is set to ${isFipsConfigEnabled} and the configured Node.js environment has FIPS ${
           isNodeRunningWithFipsEnabled ? 'enabled' : 'disabled'
         }`,
         'invalidConfig',

packages/core/security/core-security-server-internal/src/security_service.test.ts

@@ -32,10 +32,8 @@ describe('SecurityService', function () {
     const mockConfig = {
       xpack: {
         security: {
-          experimental: {
-            fipsMode: {
-              enabled: !!getFips(),
-            },
+          fipsMode: {
+            enabled: !!getFips(),
           },
         },
       },

packages/core/security/core-security-server-internal/src/utils/index.ts

@@ -11,10 +11,8 @@ export { convertSecurityApi } from './convert_security_api';
 export { getDefaultSecurityImplementation } from './default_implementation';
 
 export interface SecurityServiceConfigType {
-  experimental?: {
-    fipsMode?: {
-      enabled: boolean;
-    };
+  fipsMode?: {
+    enabled: boolean;
   };
 }
 

packages/core/test-helpers/core-test-helpers-kbn-server/src/create_root.ts

@@ -83,7 +83,7 @@ export function createRootWithSettings(
    */
   let oss = true;
   if (getFips() === 1) {
-    set(settings, 'xpack.security.experimental.fipsMode.enabled', true);
+    set(settings, 'xpack.security.fipsMode.enabled', true);
     oss = false;
     delete cliArgs.oss;
   }

src/core/server/integration_tests/node/migrator.test.ts

@@ -44,7 +44,7 @@ describe('migrator-only node', () => {
           '--no-optimizer',
           '--no-base-path',
           '--no-watch',
-          isFipsEnabled ? '--xpack.security.experimental.fipsMode.enabled=true' : '--oss',
+          isFipsEnabled ? '--xpack.security.fipsMode.enabled=true' : '--oss',
         ],
         { stdio: ['pipe', 'pipe', 'pipe'] }
       );

src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker

@@ -386,7 +386,7 @@ kibana_vars=(
     xpack.security.authc.selector.enabled
     xpack.security.cookieName
     xpack.security.encryptionKey
-    xpack.security.experimental.fipsMode.enabled
+    xpack.security.fipsMode.enabled
     xpack.security.loginAssistanceMessage
     xpack.security.loginHelp
     xpack.security.sameSiteCookies

src/dev/build/tasks/os_packages/docker_generator/templates/base/Dockerfile

@@ -59,7 +59,7 @@ RUN set -e ; \
  make install > /dev/null ; \
  rm -rf  "/usr/share/kibana/openssl-${OPENSSL_VERSION}" ; \
  chown -R 1000:0 "${OPENSSL_PATH}";
- 
+
 {{/fips}}
 # Ensure that group permissions are the same as user permissions.
 # This will help when relying on GID-0 to run Kibana, rather than UID-1000.
@@ -156,7 +156,7 @@ RUN /bin/echo -e '\n--enable-fips' >> config/node.options
 RUN echo '--openssl-config=/usr/share/kibana/config/nodejs.cnf' >> config/node.options
 COPY --chown=1000:0 openssl/nodejs.cnf "/usr/share/kibana/config/nodejs.cnf"
 ENV OPENSSL_MODULES=/usr/share/kibana/openssl/lib/ossl-modules
-ENV XPACK_SECURITY_EXPERIMENTAL_FIPSMODE_ENABLED=true
+ENV XPACK_SECURITY_FIPSMODE_ENABLED=true
 
 {{/fips}}
 RUN ln -s /usr/share/kibana /opt/kibana

x-pack/packages/kbn-langchain/server/index.ts

@@ -11,6 +11,7 @@ import { ActionsClientLlm } from './language_models/llm';
 import { ActionsClientSimpleChatModel } from './language_models/simple_chat_model';
 import { ActionsClientGeminiChatModel } from './language_models/gemini_chat';
 import { ActionsClientChatVertexAI } from './language_models/chat_vertex';
+import { ActionsClientChatBedrockConverse } from './language_models/chat_bedrock_converse';
 import { parseBedrockStream } from './utils/bedrock';
 import { parseGeminiResponse } from './utils/gemini';
 import { getDefaultArguments } from './language_models/constants';
@@ -25,4 +26,5 @@ export {
   ActionsClientGeminiChatModel,
   ActionsClientLlm,
   ActionsClientSimpleChatModel,
+  ActionsClientChatBedrockConverse,
 };

x-pack/packages/kbn-langchain/server/language_models/bedrock_chat.ts

@@ -9,11 +9,8 @@ import { BedrockChat as _BedrockChat } from '@langchain/community/chat_models/be
 import type { ActionsClient } from '@kbn/actions-plugin/server';
 import { BaseChatModelParams } from '@langchain/core/language_models/chat_models';
 import { Logger } from '@kbn/logging';
-import { Readable } from 'stream';
 import { PublicMethodsOf } from '@kbn/utility-types';
-
-export const DEFAULT_BEDROCK_MODEL = 'anthropic.claude-3-5-sonnet-20240620-v1:0';
-export const DEFAULT_BEDROCK_REGION = 'us-east-1';
+import { prepareMessages, DEFAULT_BEDROCK_MODEL, DEFAULT_BEDROCK_REGION } from '../utils/bedrock';
 
 export interface CustomChatModelInput extends BaseChatModelParams {
   actionsClient: PublicMethodsOf<ActionsClient>;
@@ -25,6 +22,11 @@ export interface CustomChatModelInput extends BaseChatModelParams {
   maxTokens?: number;
 }
 
+/**
+ * @deprecated Use the ActionsClientChatBedrockConverse chat model instead.
+ * ActionsClientBedrockChatModel chat model supports non-streaming only the Bedrock Invoke API.
+ * The LangChain team will support only the Bedrock Converse API in the future.
+ */
 export class ActionsClientBedrockChatModel extends _BedrockChat {
   constructor({ actionsClient, connectorId, logger, ...params }: CustomChatModelInput) {
     super({
@@ -36,32 +38,10 @@ export class ActionsClientBedrockChatModel extends _BedrockChat {
       fetchFn: async (url, options) => {
         const inputBody = JSON.parse(options?.body as string);
 
-        if (this.streaming && !inputBody.tools?.length) {
-          const data = (await actionsClient.execute({
-            actionId: connectorId,
-            params: {
-              subAction: 'invokeStream',
-              subActionParams: {
-                messages: inputBody.messages,
-                temperature: params.temperature ?? inputBody.temperature,
-                stopSequences: inputBody.stop_sequences,
-                system: inputBody.system,
-                maxTokens: params.maxTokens ?? inputBody.max_tokens,
-                tools: inputBody.tools,
-                anthropicVersion: inputBody.anthropic_version,
-              },
-            },
-          })) as { data: Readable; status: string; message?: string; serviceMessage?: string };
-
-          if (data.status === 'error') {
-            throw new Error(
-              `ActionsClientBedrockChat: action result status is error: ${data?.message} - ${data?.serviceMessage}`
-            );
-          }
-
-          return {
-            body: Readable.toWeb(data.data),
-          } as unknown as Response;
+        if (this.streaming) {
+          throw new Error(
+            `ActionsClientBedrockChat does not support streaming, use ActionsClientChatBedrockConverse instead`
+          );
         }
 
         const data = (await actionsClient.execute({
@@ -84,7 +64,6 @@ export class ActionsClientBedrockChatModel extends _BedrockChat {
           message?: string;
           serviceMessage?: string;
         };
-
         if (data.status === 'error') {
           throw new Error(
             `ActionsClientBedrockChat: action result status is error: ${data?.message} - ${data?.serviceMessage}`
@@ -99,20 +78,3 @@ export class ActionsClientBedrockChatModel extends _BedrockChat {
     });
   }
 }
-
-const prepareMessages = (messages: Array<{ role: string; content: string[] }>) =>
-  messages.reduce((acc, { role, content }) => {
-    const lastMessage = acc[acc.length - 1];
-
-    if (!lastMessage || lastMessage.role !== role) {
-      acc.push({ role, content });
-      return acc;
-    }
-
-    if (lastMessage.role === role) {
-      acc[acc.length - 1].content = lastMessage.content.concat(content);
-      return acc;
-    }
-
-    return acc;
-  }, [] as Array<{ role: string; content: string[] }>);

x-pack/packages/kbn-langchain/server/language_models/chat_bedrock_converse/bedrock_runtime_client.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  BedrockRuntimeClient as _BedrockRuntimeClient,
+  BedrockRuntimeClientConfig,
+} from '@aws-sdk/client-bedrock-runtime';
+import { constructStack } from '@smithy/middleware-stack';
+import { PublicMethodsOf } from '@kbn/utility-types';
+import type { ActionsClient } from '@kbn/actions-plugin/server';
+
+import { NodeHttpHandler } from './node_http_handler';
+
+export interface CustomChatModelInput extends BedrockRuntimeClientConfig {
+  actionsClient: PublicMethodsOf<ActionsClient>;
+  connectorId: string;
+  streaming?: boolean;
+}
+
+export class BedrockRuntimeClient extends _BedrockRuntimeClient {
+  middlewareStack: _BedrockRuntimeClient['middlewareStack'];
+
+  constructor({ actionsClient, connectorId, ...fields }: CustomChatModelInput) {
+    super(fields ?? {});
+    this.config.requestHandler = new NodeHttpHandler({
+      streaming: fields.streaming ?? true,
+      actionsClient,
+      connectorId,
+    });
+    // eliminate middleware steps that handle auth as Kibana connector handles auth
+    this.middlewareStack = constructStack() as _BedrockRuntimeClient['middlewareStack'];
+  }
+}