Merge branch 'main' into fix-range-alignment

.buildkite/ftr_oblt_serverless_configs.yml

@@ -6,6 +6,10 @@ disabled:
   - x-pack/test_serverless/functional/test_suites/observability/cypress/config_headless.ts
   - x-pack/test_serverless/functional/test_suites/observability/cypress/config_runner.ts
 
+    # serverless config files that run deployment-agnostic tests
+    # Failing https://github.com/elastic/kibana/issues/195811
+  - x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts
+
 defaultQueue: 'n2-4-spot'
 enabled:
   - x-pack/test_serverless/api_integration/test_suites/observability/config.ts
@@ -25,5 +29,3 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group5.ts
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group6.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.screenshots.ts
-  # serverless config files that run deployment-agnostic tests
-  - x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts

.buildkite/ftr_search_serverless_configs.yml

@@ -1,6 +1,10 @@
 disabled:
   # Base config files, only necessary to inform config finding script
 
+  # serverless config files that run deployment-agnostic tests
+  # Failing https://github.com/elastic/kibana/issues/195811
+  - x-pack/test/api_integration/deployment_agnostic/configs/serverless/search.serverless.config.ts
+
 defaultQueue: 'n2-4-spot'
 enabled:
   - x-pack/test_serverless/api_integration/test_suites/search/config.ts
@@ -18,5 +22,3 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group4.ts
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group5.ts
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group6.ts
-  # serverless config files that run deployment-agnostic tests
-  - x-pack/test/api_integration/deployment_agnostic/configs/serverless/search.serverless.config.ts

.buildkite/ftr_security_serverless_configs.yml

@@ -20,6 +20,10 @@ disabled:
   - x-pack/test_serverless/functional/config.base.ts
   - x-pack/test_serverless/shared/config.base.ts
 
+  # serverless config files that run deployment-agnostic tests
+  # Failing https://github.com/elastic/kibana/issues/195811
+  - x-pack/test/api_integration/deployment_agnostic/configs/serverless/security.serverless.config.ts
+
 defaultQueue: 'n2-4-spot'
 enabled:
   - x-pack/test_serverless/api_integration/test_suites/security/config.ts
@@ -100,5 +104,3 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts
   - x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts
-  # serverless config files that run deployment-agnostic tests
-  - x-pack/test/api_integration/deployment_agnostic/configs/serverless/security.serverless.config.ts

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -51,7 +51,7 @@ fi
 if is_pr_with_label "ci:cloud-redeploy"; then
   echo "--- Shutdown Previous Deployment"
   CLOUD_DEPLOYMENT_ID=$(ecctl deployment list --output json | jq -r '.deployments[] | select(.name == "'$CLOUD_DEPLOYMENT_NAME'") | .id')
-  if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
+  if [ -z "${CLOUD_DEPLOYMENT_ID}" ] || [ "${CLOUD_DEPLOYMENT_ID}" == "null" ]; then
     echo "No deployment to remove"
   else
     echo "Shutting down previous deployment..."

.buildkite/scripts/steps/serverless/deploy.sh

@@ -56,7 +56,7 @@ deploy() {
 
   PROJECT_ID=$(jq -r '[.items[] | select(.name == "'$PROJECT_NAME'")] | .[0].id' $PROJECT_EXISTS_LOGS)
   if is_pr_with_label "ci:project-redeploy"; then
-    if [ -z "${PROJECT_ID}" ]; then
+    if [ -z "${PROJECT_ID}" ] || [ "${PROJECT_ID}" == "null" ]; then
       echo "No project to remove"
     else
       echo "Shutting down previous project..."

.github/updatecli/values.d/ironbank.yml

@@ -0,0 +1,2 @@
+config:
+  - path: src/dev/build/tasks/os_packages/docker_generator/templates/ironbank

.github/updatecli/values.d/scm.yml

@@ -0,0 +1,11 @@
+scm:
+  enabled: true
+  owner: elastic
+  repository: kibana
+  branch: main
+  commitusingapi: true
+  # begin updatecli-compose policy values
+  user: kibanamachine
+  email: [email protected]
+  # end updatecli-compose policy values
+

.github/updatecli/values.d/updatecli-compose.yml

@@ -0,0 +1,3 @@
+spec:
+  files:
+    - "updatecli-compose.yaml"

.github/workflows/updatecli-compose.yml

@@ -0,0 +1,38 @@
+---
+name: updatecli-compose
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 6 * * *'
+
+permissions:
+  contents: read
+
+jobs:
+  compose:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      packages: read
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: elastic/oblt-actions/updatecli/run@v1
+        with:
+          command: --experimental compose diff
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: elastic/oblt-actions/updatecli/run@v1
+        with:
+          command: --experimental compose apply
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

dev_docs/nav-kibana-dev.docnav.json

@@ -278,6 +278,10 @@
         {
           "id": "kibDevReactKibanaContext",
           "label": "Kibana React Contexts"
+        },
+        {
+          "id": "kibDevDocsChromeRecentlyAccessed",
+          "label": "Recently Viewed"
         }
       ]
     },

dev_docs/shared_ux/chrome_recently_accessed/chrome_recently_accessed.mdx

@@ -0,0 +1,66 @@
+---
+id: kibDevDocsChromeRecentlyAccessed
+slug: /kibana-dev-docs/chrome/recently-accessed
+title: Chrome Recently Viewed
+description: How to use chrome's recently accessed service to add your links to the recently viewed list in the side navigation.
+date: 2024-10-04
+tags: ['kibana', 'dev', 'contributor', 'chrome', 'navigation', 'shared-ux']
+---
+
+## Introduction
+
+The <DocLink id="kibKbnCoreChromeBrowserPluginApi" section="def-public.ChromeRecentlyAccessed" text="ChromeRecentlyAccessed" /> service allows applications to register recently visited objects. These items are displayed in the "Recently Viewed" section of a side navigation menu, providing users with quick access to their previously visited resources. This service includes methods for adding, retrieving, and subscribing to the recently accessed history.
+
+![Recently viewed section in the sidenav](./chrome_recently_accessed.png)
+
+## Guidelines
+
+The <DocLink id="kibKbnCoreChromeBrowserPluginApi" section="def-public.ChromeRecentlyAccessed" text="ChromeRecentlyAccessed" /> service should be used thoughtfully to provide users with easy access to key resources they've interacted with. Unlike browser history, this feature is for important items that users may want to revisit.
+
+### DOs
+
+- Register important resources that users may want to revisit. Like a dashboard, a saved search, or another specific object.
+- Update the link when the state of the current resource changes. For example, if a user changes the time range while on a dashboard, update the recently viewed link to reflect the latest viewed state where possible. See below for instructions on how to update the link when state changes.
+
+### DON'Ts
+
+- Don't register every page view.
+- Don't register temporary or transient states as individual items.
+- Prevent overloading. Keep the list focused on high-value resources.
+- Don't add a recently viewed object without first speaking to relevant Product Managers.
+
+## Usage
+
+To register an item with the `ChromeRecentlyAccessed` service, provide a unique `id`, a `label`, and a `link`. The `id` is used to identify and deduplicate the item, the `label` is displayed in the "Recently Viewed" list and the `link` is used to navigate to the item when selected.
+
+```ts
+const link = '/app/map/1234';
+const label = 'Map 1234';
+const id = 'map-1234';
+
+coreStart.chrome.recentlyAccessed.add(link, label, id);
+```
+
+To update the link when state changes, add another item with the same `id`. This will replace the existing item in the "Recently Viewed" list.
+
+```ts
+const link = '/app/map/1234';
+const label = 'Map 1234';
+
+coreStart.chrome.recentlyAccessed.add(`/app/map/1234`, label, id);
+
+// User changes the time range and we want to update the link in the "Recently Viewed" list
+coreStart.chrome.recentlyAccessed.add(
+  `/app/map/1234?timeRangeFrom=now-30m&timeRangeTo=now`,
+  label,
+  id
+);
+```
+
+## Implementation details
+
+The <DocLink id="kibKbnCoreChromeBrowserPluginApi" section="def-public.ChromeRecentlyAccessed" text="ChromeRecentlyAccessed" /> services is based on <DocLink id="kibKbnRecentlyAccessedPluginApi" text="@kbn/recently-accessed"/> package. This package provides a `RecentlyAccessedService` that uses browser local storage to manage records of recently accessed objects. Internally it implements the queue with a maximum length of 20 items. When the queue is full, the oldest item is removed.
+Applications can create their own instance of `RecentlyAccessedService` to manage their own list of recently accessed items scoped to their application.
+
+- <DocLink id="kibKbnCoreChromeBrowserPluginApi" section="def-public.ChromeRecentlyAccessed" text="ChromeRecentlyAccessed" /> is a service available via `coreStart.chrome.recentlyAccessed` and should be used to add items to chrome's sidenav.
+- <DocLink id="kibKbnRecentlyAccessedPluginApi" text="@kbn/recently-accessed"/> is package that `ChromeRecentlyAccessed` is using internally and the package can be used to create your own instance and manage your own list of recently accessed items that is independent for chrome's sidenav.

dev_docs/shared_ux/shared_ux_landing.mdx

@@ -66,5 +66,10 @@ layout: landing
       title: 'Kibana React Contexts',
       description: 'Learn how to use common React contexts in Kibana',
     },
+    {
+      pageId: 'kibDevDocsChromeRecentlyAccessed',
+      title: 'Chrome Recently Viewed',
+      description: 'Learn how to add recently viewed items to the side navigation',
+    },
   ]}
 />

docs/maps/connect-to-ems.asciidoc

@@ -1,6 +1,6 @@
 :ems: Elastic Maps Service
 :ems-docker-repo: docker.elastic.co/elastic-maps-service/elastic-maps-server
-:ems-docker-image: {ems-docker-repo}:{version}-amd64
+:ems-docker-image: {ems-docker-repo}:{version}
 :ems-headers-url: https://deployment-host
 
 [[maps-connect-to-ems]]
@@ -81,34 +81,53 @@ If you cannot connect to {ems} from the {kib} server or browser clients, and you
 
 {hosted-ems} is a self-managed version of {ems} offered as a Docker image that provides both the EMS basemaps and EMS boundaries. The image is bundled with basemaps up to zoom level 8. After connecting it to your {es} cluster for license validation, you have the option to download and configure a more detailed basemaps database.
 
-You can use +docker pull+ to download the {hosted-ems} image from the Elastic Docker registry.
-
+. Pull the {hosted-ems} Docker image.
++
 ifeval::["{release-state}"=="unreleased"]
-Version {version} of {hosted-ems} has not yet been released, so no Docker image is currently available for this version.
+WARNING: Version {version} of {hosted-ems} has not yet been released. 
+No Docker image is currently available for this version.
 endif::[]
-
-ifeval::["{release-state}"!="unreleased"]
-
++
 ["source","bash",subs="attributes"]
 ----------------------------------
 docker pull {ems-docker-image}
 ----------------------------------
 
-Start {hosted-ems} and expose the default port `8080`:
+. Optional: Install
+https://docs.sigstore.dev/system_config/installation/[Cosign] for your
+environment. Then use Cosign to verify the {es} image's signature.
++
+[source,sh,subs="attributes"]
+----
+wget https://artifacts.elastic.co/cosign.pub
+cosign verify --key cosign.pub {ems-docker-image}
+----
++
+The `cosign` command prints the check results and the signature payload in JSON format:
++
+[source,sh,subs="attributes"]
+--------------------------------------------
+Verification for {ems-docker-image} --
+The following checks were performed on each of these signatures:
+  - The cosign claims were validated
+  - Existence of the claims in the transparency log was verified offline
+  - The signatures were verified against the specified public key
+--------------------------------------------
+
 
+. Start {hosted-ems} and expose the default port `8080`:
++
 ["source","bash",subs="attributes"]
 ----------------------------------
 docker run --rm --init --publish 8080:8080 \
   {ems-docker-image}
 ----------------------------------
-
++
 Once {hosted-ems} is running, follow instructions from the webpage at `localhost:8080` to define a configuration file and optionally download a more detailed basemaps database.
-
++
 [role="screenshot"]
 image::images/elastic-maps-server-instructions.png[Set-up instructions]
 
-endif::[]
-
 [float]
 [[elastic-maps-server-configuration]]
 ==== Configuration
@@ -193,7 +212,6 @@ One way to configure {hosted-ems} is to provide `elastic-maps-server.yml` via bi
 
 ["source","yaml",subs="attributes"]
 --------------------------------------------
-version: '2'
 services:
   ems-server:
     image: {ems-docker-image}
@@ -212,7 +230,6 @@ These variables can be set with +docker-compose+ like this:
 
 ["source","yaml",subs="attributes"]
 ----------------------------------------------------------
-version: '2'
 services:
   ems-server:
     image: {ems-docker-image}

packages/kbn-apm-synthtrace/src/lib/logs/custom_logsdb_index_templates.ts

@@ -25,6 +25,7 @@ export const indexTemplates: {
     template: {
       settings: {
         mode: 'logsdb',
+        default_pipeline: 'logs@default-pipeline',
       },
     },
     priority: 500,

packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts

@@ -7,16 +7,20 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { Client } from '@elastic/elasticsearch';
+import { Client, estypes } from '@elastic/elasticsearch';
 import { pipeline, Readable } from 'stream';
 import { LogDocument } from '@kbn/apm-synthtrace-client/src/lib/logs';
-import { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
+import { IngestProcessorContainer, MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
+import { ValuesType } from 'utility-types';
 import { SynthtraceEsClient, SynthtraceEsClientOptions } from '../shared/base_client';
 import { getSerializeTransform } from '../shared/get_serialize_transform';
 import { Logger } from '../utils/create_logger';
 import { indexTemplates, IndexTemplateName } from './custom_logsdb_index_templates';
 import { getRoutingTransform } from '../shared/data_stream_get_routing_transform';
 
+export const LogsIndex = 'logs';
+export const LogsCustom = 'logs@custom';
+
 export type LogsSynthtraceEsClientOptions = Omit<SynthtraceEsClientOptions, 'pipeline'>;
 
 export class LogsSynthtraceEsClient extends SynthtraceEsClient<LogDocument> {
@@ -60,6 +64,47 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient<LogDocument> {
       this.logger.error(`Index creation failed: ${index} - ${err.message}`);
     }
   }
+
+  async updateIndexTemplate(
+    indexName: string,
+    modify: (
+      template: ValuesType<
+        estypes.IndicesGetIndexTemplateResponse['index_templates']
+      >['index_template']
+    ) => estypes.IndicesPutIndexTemplateRequest
+  ) {
+    try {
+      const response = await this.client.indices.getIndexTemplate({
+        name: indexName,
+      });
+
+      await Promise.all(
+        response.index_templates.map((template) => {
+          return this.client.indices.putIndexTemplate({
+            ...modify(template.index_template),
+            name: template.name,
+          });
+        })
+      );
+
+      this.logger.info(`Updated ${indexName} index template`);
+    } catch (err) {
+      this.logger.error(`Update index template failed: ${indexName} - ${err.message}`);
+    }
+  }
+
+  async createCustomPipeline(processors: IngestProcessorContainer[]) {
+    try {
+      this.client.ingest.putPipeline({
+        id: LogsCustom,
+        processors,
+        version: 1,
+      });
+      this.logger.info(`Custom pipeline created: ${LogsCustom}`);
+    } catch (err) {
+      this.logger.error(`Custom pipeline creation failed: ${LogsCustom} - ${err.message}`);
+    }
+  }
 }
 
 function logsPipeline() {