Merge branch 'elastic:main' into main

.buildkite/scripts/common/env.sh

@@ -8,6 +8,7 @@ KIBANA_DIR=$(pwd)
 export KIBANA_DIR
 export XPACK_DIR="$KIBANA_DIR/x-pack"
 
+export XDG_CACHE_HOME="$HOME/.cache"
 export CACHE_DIR="$HOME/.kibana"
 export ES_CACHE_DIR="$HOME/.es-snapshot-cache"
 PARENT_DIR="$(cd "$KIBANA_DIR/.."; pwd)"
@@ -110,7 +111,6 @@ export TEST_CORS_SERVER_PORT=6105
 if [[ "$(which google-chrome-stable)" || "$(which google-chrome)" ]]; then
   echo "Chrome detected, setting DETECT_CHROMEDRIVER_VERSION=true"
   export DETECT_CHROMEDRIVER_VERSION=true
-  export CHROMEDRIVER_FORCE_DOWNLOAD=true
 else
   echo "Chrome not detected, installing default chromedriver binary for the package version"
 fi

.github/CODEOWNERS

@@ -597,6 +597,7 @@ packages/kbn-management/settings/types @elastic/kibana-management
 packages/kbn-management/settings/utilities @elastic/kibana-management
 packages/kbn-management/storybook/config @elastic/kibana-management
 test/plugin_functional/plugins/management_test_plugin @elastic/kibana-management
+packages/kbn-manifest @elastic/kibana-core
 packages/kbn-mapbox-gl @elastic/kibana-presentation
 x-pack/examples/third_party_maps_source_example @elastic/kibana-presentation
 src/plugins/maps_ems @elastic/kibana-presentation
@@ -929,9 +930,9 @@ packages/kbn-test-eui-helpers @elastic/kibana-visualizations
 x-pack/test/licensing_plugin/plugins/test_feature_usage @elastic/kibana-security
 packages/kbn-test-jest-helpers @elastic/kibana-operations @elastic/appex-qa
 packages/kbn-test-subj-selector @elastic/kibana-operations @elastic/appex-qa
-x-pack/test_serverless 
-test 
-x-pack/test 
+x-pack/test_serverless
+test
+x-pack/test
 x-pack/performance @elastic/appex-qa
 x-pack/examples/testing_embedded_lens @elastic/kibana-visualizations
 x-pack/examples/third_party_lens_navigation_prompt @elastic/kibana-visualizations

dev_docs/key_concepts/api_authorization.mdx

@@ -0,0 +1,319 @@
+---
+id: kibDevDocsSecurityAPIAuthorization
+slug: /kibana-dev-docs/key-concepts/security-api-authorization
+title: Kibana API authorization
+description: This guide provides an overview of API authorization in Kibana.
+date: 2024-10-04
+tags: ['kibana', 'dev', 'contributor', 'security']
+---
+
+Authorization is an important aspect of API design. It must be considered for all endpoints, even those marked as `internal`. This guide explains how and when to apply authorization to your endpoints
+
+Table of contents:
+1.  [API authorization](#api-authorization)
+2.  [[Deprecated] Adding API authorization with `access` tags](#deprecated-adding-api-authorization-with-access-tags)
+    - [Why not add `access` tags to all routes by default?](#why-not-add-access-tags-to-all-routes-by-default)
+3.  [Adding API authorization with `security` configuration](#adding-api-authorization-with-security-configuration)
+    - [Key features](#key-features)
+    - [Configuring authorization on routes](#configuring-authorization-on-routes)
+    - [Opting out of authorization for specific routes](#opting-out-of-authorization-for-specific-routes)
+    - [Classic router security configuration examples](#classic-router-security-configuration-examples)
+    - [Versioned router security configuration examples](#versioned-router-security-configuration-examples)
+4. [Authorization response available in route handlers](#authorization-response-available-in-route-handlers)
+5. [OpenAPI specification (OAS) documentation](#openapi-specification-oas-documentation)
+6. [Migrating from `access` tags to `security` configuration](#migrating-from-access-tags-to-security-configuration)
+7. [Questions?](#questions)
+
+## API authorization
+Kibana API routes do not have any authorization checks applied by default. This means that your APIs are accessible to anyone with valid credentials, regardless of their permissions. This includes users with no roles assigned.
+This on its own is insufficient, and care must be taken to ensure that only authorized users can invoke your endpoints.
+
+Kibana leverages <DocLink id="kibDevDocsSavedObjectsIntro" text="Saved Objects" /> for a majority of its persistence. The Saved Objects Service performs its own authorization checks, so if your API route is primarily a CRUD interface to Saved Objects, then your authorization needs are likely already met.
+This is also true for derivatives of the Saved Objects Service, such as the Alerting and Cases services.
+
+If your endpoint is not a CRUD interface to Saved Objects, or if your endpoint bypasses our built-in Saved Objects authorization checks, then you must ensure that only authorized users can invoke your endpoint. 
+This is **especially** important if your route does any of the following:
+1. Performs non-insignificant processing, causing load on the Elasticsearch cluster or the Kibana server.
+2. Calls Elasticsearch APIs using the internal `kibana_system` user.
+3. Calls a third-party service.
+4. Exposes any non-public information to the caller, such as system configuration or state, as part of the successful or even error response.
+
+## [Deprecated] Adding API authorization with `access` tags
+**Note**: `access` tags were deprecated in favour of `security` configuration.
+
+`access` tags are used to restrict access to API routes. They are used to ensure that only users with the required privileges can access the route.
+
+Example configuration:
+```ts
+router.get({
+  path: '/api/path',
+  options: {
+    tags: ['access:<privilege_1>', 'access:<privilege_2>'],
+  },
+  ...
+}, handler);
+```
+
+More information on adding `access` tags to your routes can be found temporarily in the [legacy documentation](https://www.elastic.co/guide/en/kibana/current/development-security.html#development-plugin-feature-registration)
+
+### Why not add `access` tags to all routes by default?
+Each authorization check that we perform involves a round-trip to Elasticsearch, so they are not as cheap as we'd like. Therefore, we want to keep the number of authorization checks we perform within a single route to a minimum.
+Adding an `access` tag to routes that leverage the Saved Objects Service would be redundant in most cases, since the Saved Objects Service will be performing authorization anyway.
+
+
+## Adding API authorization with `security` configuration
+`KibanaRouteOptions` provides a security configuration at the route definition level, offering robust security configurations for both **Classic** and **Versioned** routes.
+
+### Key features:
+1. **Fine-grained control**: 
+  - Define the exact privileges required to access the route.
+  - Use `requiredPrivileges` to specify privileges with support for complex rules:
+    - **AND rules** using `allRequired`: Requires all specified privileges for access.
+    - **OR rules** using `anyRequired`: Allows access if any one of the specified privileges is met.
+    - **Complex Nested Rules**: Combine both `allRequired` and `anyRequired` for advanced access rules.
+2. **Explicit Opt-out**: Provide a reason for opting out of authorization to maintain transparency.
+3. **Versioned Routes**: Define security configurations for different versions of the same route.
+4. **Improved Documentation with OpenAPI (OAS)**: Automatically generated OAS documentation with the required privileges for each route.
+5. **AuthzResult Object in Route Handlers**: Access the authorization response in route handlers to see which privileges were met.
+
+
+### Configuring authorization on routes
+**Before migration:**
+```ts
+router.get({
+  path: '/api/path',
+  options: {
+    tags: ['access:<privilege_1>', 'access:<privilege_2>'],
+  },
+  ...
+}, handler);
+```
+
+**After migration:**
+```ts
+router.get({
+  path: '/api/path',
+  security: {
+    authz: {
+      requiredPrivileges: ['<privilege_1>', '<privilege_2>'],
+    },
+  },
+  ...
+}, handler);
+```
+
+### Opting out of authorization for specific routes
+**Before migration:**
+```ts
+router.get({
+  path: '/api/path',
+  ...
+}, handler);
+```
+
+**After migration:**
+```ts
+router.get({
+  path: '/api/path',
+  security: {
+    authz: {
+      enabled: false,
+      reason: 'This route is opted out from authorization because ...',
+    },
+  },
+  ...
+}, handler);
+```
+
+### Classic router security configuration examples
+
+**Example 1: All privileges required.**
+Requires `<privilege_1>` AND `<privilege_2>` to access the route.
+```ts
+router.get({
+  path: '/api/path',
+  security: {
+    authz: {
+      requiredPrivileges: ['<privilege_1>', '<privilege_2>'],
+    },
+  },
+  ...
+}, handler);
+```
+
+**Example 2: Any privileges required.**
+Requires `<privilege_1>` OR `<privilege_2>` to access the route.
+```ts
+router.get({
+  path: '/api/path',
+  security: {
+    authz: {
+      requiredPrivileges: [{ anyRequired: ['<privilege_1>', '<privilege_2>'] }],
+    },
+  },
+  ...
+}, handler);
+```
+
+**Example 3: Complex configuration.**
+Requires `<privilege_1>` AND `<privilege_2>` AND (`<privilege_3>` OR `<privilege_4>`) to access the route.
+```ts
+router.get({
+  path: '/api/path',
+  security: {
+    authz: {
+      requiredPrivileges: [{ allRequired: ['<privilege_1>', '<privilege_2>'], anyRequired: ['<privilege_3>', '<privilege_4>'] }],
+    },
+  },
+  ...
+}, handler);
+```
+
+### Versioned router security configuration examples
+Different security configurations can be applied to each version when using the Versioned Router. This allows your authorization needs to evolve in lockstep with your API.
+
+**Example 1: Default and custom version security.**
+
+1. **Default configuration**: Applies to versions without specific authorization, requires `<privilege_1>`.
+
+2. **Version 1**: Requires **both** `<privilege_1>` and `<privilege_2>` privileges.
+
+3. **Version 2**: Inherits the default authorization configuration, requiring `<privilege_1>`.
+
+```ts
+router.versioned
+  .get({
+    path: '/internal/path',
+    access: 'internal',
+    // default security configuration, will be used for version unless overridden
+    security: {
+      authz: {
+        requiredPrivileges: ['<privilege_1>'],
+      },
+    },
+  })
+  .addVersion({
+    version: '1',
+    validate: false,
+    security: {
+      authz: {
+        requiredPrivileges: ['<privilege_1>', '<privilege_2>'],
+      },
+    },
+  }, handlerV1)
+  .addVersion({
+    version: '2',
+    validate: false,
+  }, handlerV2);
+```
+
+**Example 2: Multiple versions with different security requirements.**
+1. **Default Configuration**: Applies to versions without specific authorization, requires `<privilege_1>`.
+
+2. **Version 1**: Requires **both** `<privilege_1>` and `<privilege_2>` privileges.
+
+3. **Version 2**: Requires `<privilege_3>` AND (`<privilege_1>` OR `<privilege_2>`).
+
+4. **Version 3**: Requires only `<privilege_3>`.
+
+```ts
+router.versioned
+  .get({
+    path: '/internal/path',
+    access: 'internal',
+    // default security configuration, will be used for version unless overridden
+    security: {
+      authz: {
+        requiredPrivileges: ['<privilege_1>'],
+      },
+    },
+  })
+  .addVersion({
+    version: '1',
+    validate: false,
+    security: {
+      authz: {
+        requiredPrivileges: ['<privilege_1>', '<privilege_2>'],
+      },
+    },
+  }, handlerV1)
+  .addVersion({
+    version: '2',
+    validate: false,
+    security: {
+      authz: {
+        requiredPrivileges: ['<privilege_3>', anyRequired: ['<privilege_1>', '<privilege_2>']],
+      },
+    },
+  }, handlerV2)
+  .addVersion({
+    version: '3',
+    validate: false,
+    security: {
+      authz: {
+        requiredPrivileges: ['<privilege_3>'],
+      },
+    },
+  }, handlerV3);
+```
+
+## Authorization response available in route handlers
+The `AuthzResult` object is available in route handlers, which provides information about the privileges granted to the caller.
+For example, you have a route that requires `<privilege_3>` and ANY of the privileges `<privilege_1>` OR `<privilege_2>`:
+```ts
+router.get({
+  path: '/api/path',
+  security: {
+    authz: {
+      requiredPrivileges: ['<privilege_3>', { anyRequired: ['<privilege_1>', '<privilege_2>'] }],
+    },
+  },
+  ...
+}, (context, request, response) => {
+  // The authorization response is available in `request.authzResult`
+  // {
+  //   "<privilege_3>": true,
+  //   "<privilege_1>": true,
+  //   "<privilege_2>": false
+  // }
+});
+```
+
+## OpenAPI specification (OAS) documentation
+Based on the security configuration defined in routes, OAS documentation will automatically generate and include description about the required privileges. 
+This makes it easy to view the security requirements of each endpoint in a standardized format, facilitating better understanding and usage by developers or teams consuming the API.
+
+To check the OAS documentation for a specific API route and see its security details, you can use the following command:
+```sh
+GET /api/oas?pathStartsWith=/your/api/path
+```
+
+## Migrating from `access` tags to `security` configuration
+We aim to use the same privileges that are currently defined with tags `access:<privilege_name>`.
+To assist with this migration, we have created eslint rule `no_deprecated_authz_config`, that will automatically convert your `access` tags to the new `security` configuration.
+It scans route definitions and converts `access` tags to the new `requiredPriviliges` configuration.
+
+Note: The rule is disabled by default to avoid automatic migrations without an oversight. You can perform migrations by running:
+
+**Migrate routes with defined authorization**
+```sh
+MIGRATE_DISABLED_AUTHZ=false MIGRATE_ENABLED_AUTHZ=true npx eslint --ext .ts --fix path/to/your/folder
+```
+
+**Migrate routes opted out from authorization**
+```sh
+MIGRATE_DISABLED_AUTHZ=true MIGRATE_ENABLED_AUTHZ=false npx eslint --ext .ts --fix path/to/your/folder
+```
+We encourage you to migrate routes that are opted out from authorization to new config and provide legitimate reason for disabled authorization.
+It is better to migrate routes opted out from authorization iteratively and elaborate on the reasoning. 
+Routes without a compelling reason to opt-out of authorization should plan to introduce them as soon as possible.
+
+**Migrate all routes**
+```sh
+MIGRATE_DISABLED_AUTHZ=true MIGRATE_ENABLED_AUTHZ=true npx eslint --ext .ts --fix path/to/your/folder
+```
+
+## Questions?
+If you have any questions or need help with API authorization, please reach out to the `@elastic/kibana-security` team.
+
+