Merge branch 'main' into introduce-ilm-policies

lahsivjar · Nov 12, 2024 · 1859780 · 1859780
2 parents 852e5af + 778ab8f
commit 1859780
Show file tree

Hide file tree

Showing 77 changed files with 1,650 additions and 402 deletions.
diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
+            BWC_VERSION: ["8.15.5", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
@@ -272,8 +272,8 @@ steps:
         env:
           BWC_VERSION: 8.14.3
 
-      - label: "{{matrix.image}} / 8.15.4 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.4
+      - label: "{{matrix.image}} / 8.15.5 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.5
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -286,7 +286,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.15.4
+          BWC_VERSION: 8.15.5
 
       - label: "{{matrix.image}} / 8.16.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.0

diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
@@ -287,8 +287,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.15.4 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.4#bwcTest
+      - label: 8.15.5 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.5#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -297,7 +297,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.15.4
+          BWC_VERSION: 8.15.5
         retry:
           automatic:
             - exit_status: "-1"
@@ -429,7 +429,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
+            BWC_VERSION: ["8.15.5", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -471,7 +471,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
+            BWC_VERSION: ["8.15.5", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -14,7 +14,7 @@ BWC_VERSION:
   - "8.12.2"
   - "8.13.4"
   - "8.14.3"
-  - "8.15.4"
+  - "8.15.5"
   - "8.16.0"
   - "8.17.0"
   - "9.0.0"
diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
@@ -1,5 +1,5 @@
 BWC_VERSION:
-  - "8.15.4"
+  - "8.15.5"
   - "8.16.0"
   - "8.17.0"
   - "9.0.0"
diff --git a/docs/changelog/104683.yaml b/docs/changelog/104683.yaml
@@ -0,0 +1,5 @@
+pr: 104683
+summary: "Feature: re-structure document ID generation favoring _id inverted index compression"
+area: Logs
+type: enhancement
+issues: []
diff --git a/docs/changelog/112250.yaml b/docs/changelog/112250.yaml
diff --git a/docs/changelog/113723.yaml b/docs/changelog/113723.yaml
diff --git a/docs/changelog/114407.yaml b/docs/changelog/114407.yaml
diff --git a/docs/changelog/114533.yaml b/docs/changelog/114533.yaml
diff --git a/docs/changelog/114601.yaml b/docs/changelog/114601.yaml
diff --git a/docs/changelog/115181.yaml b/docs/changelog/115181.yaml
diff --git a/docs/changelog/115308.yaml b/docs/changelog/115308.yaml
diff --git a/docs/changelog/115430.yaml b/docs/changelog/115430.yaml
diff --git a/docs/changelog/115459.yaml b/docs/changelog/115459.yaml
diff --git a/docs/changelog/115510.yaml b/docs/changelog/115510.yaml
diff --git a/docs/changelog/115834.yaml b/docs/changelog/115834.yaml
diff --git a/docs/changelog/116031.yaml b/docs/changelog/116031.yaml
diff --git a/docs/changelog/116219.yaml b/docs/changelog/116219.yaml
diff --git a/docs/changelog/116431.yaml b/docs/changelog/116431.yaml
@@ -0,0 +1,5 @@
+pr: 116431
+summary: Adds support for `input_type` field to Vertex inference service
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/116650.yaml b/docs/changelog/116650.yaml
@@ -0,0 +1,5 @@
+pr: 116650
+summary: Fix bug in ML autoscaling when some node info is unavailable
+area: Machine Learning
+type: bug
+issues: []
diff --git a/docs/reference/connector/docs/_connectors-overview-table.asciidoc b/docs/reference/connector/docs/_connectors-overview-table.asciidoc
@@ -44,7 +44,7 @@ NOTE: All connectors are available as self-managed <<es-build-connector,self-man
 |<<es-connectors-salesforce,Salesforce>>|*GA*|8.12+|8.12+|8.11+|8.13+|8.13+|https://github.com/elastic/connectors/tree/main/connectors/sources/salesforce.py[View code]
 |<<es-connectors-servicenow,ServiceNow>>|*GA*|8.10+|8.10+|8.11+|8.13+|8.13+|https://github.com/elastic/connectors/tree/main/connectors/sources/servicenow.py[View code]
 |<<es-connectors-sharepoint-online,Sharepoint Online>>|*GA*|8.9+|8.9+|8.9+|8.9+|8.9+|https://github.com/elastic/connectors/tree/main/connectors/sources/sharepoint_online.py[View code]
-|<<es-connectors-sharepoint,Sharepoint Server>>|*Beta*|8.15+|-|8.11+|8.13+|8.14+|https://github.com/elastic/connectors/tree/main/connectors/sources/sharepoint_server.py[View code]
+|<<es-connectors-sharepoint,Sharepoint Server>>|*Beta*|8.15+|-|8.11+|8.13+|8.15+|https://github.com/elastic/connectors/tree/main/connectors/sources/sharepoint_server.py[View code]
 |<<es-connectors-slack,Slack>>|*Preview*|8.14+|-|-|-|-|https://github.com/elastic/connectors/tree/main/connectors/sources/slack.py[View code]
 |<<es-connectors-teams,Teams>>|*Preview*|8.14+|-|-|8.13+|-|https://github.com/elastic/connectors/tree/main/connectors/sources/teams.py[View code]
 |<<es-connectors-zoom,Zoom>>|*Preview*|8.14+|-|8.11+|8.13+|-|https://github.com/elastic/connectors/tree/main/connectors/sources/zoom.py[View code]

diff --git a/docs/reference/connector/docs/connectors-release-notes.asciidoc b/docs/reference/connector/docs/connectors-release-notes.asciidoc
@@ -4,7 +4,13 @@
 <titleabbrev>Release notes</titleabbrev>
 ++++
 
-[INFO]
+[NOTE]
 ====
-Prior to version 8.16.0, the connector release notes were published as part of the https://www.elastic.co/guide/en/enterprise-search/current/changelog.html[Enterprise Search documentation].
+Prior to version *8.16.0*, the connector release notes were published as part of the {enterprise-search-ref}/changelog.html[Enterprise Search documentation].
 ====
+
+*Release notes*:
+
+* <<es-connectors-release-notes-8-16-0>>
+
+include::release-notes/connectors-release-notes-8.16.0.asciidoc[]
diff --git a/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc b/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc
@@ -87,14 +87,16 @@ Select an expiration date. (At this expiration date, you will need to generate a
 +
 ```
 Graph API
-- Sites.Read.All
+- Sites.Selected
 - Files.Read.All
 - Group.Read.All
 - User.Read.All
 
 Sharepoint
-- Sites.Read.All
+- Sites.Selected
 ```
+NOTE: If the `Comma-separated list of sites` configuration is set to `*` or if a user enables the toggle button `Enumerate all sites`, the connector requires `Sites.Read.All` permission.
+
 * **Grant admin consent**, using the `Grant Admin Consent` link from the permissions screen.
 * Save the tenant name (i.e. Domain name) of Azure platform.
 
@@ -138,7 +140,7 @@ Refer to https://learn.microsoft.com/en-us/sharepoint/dev/general-development/ho
 
 Here's a summary of why we use these Graph API permissions:
 
-* *Sites.Read.All* is used to fetch the sites and their metadata
+* *Sites.Selected* is used to fetch the sites and their metadata
 * *Files.Read.All* is used to fetch Site Drives and files in these drives
 * *Groups.Read.All* is used to fetch groups for document-level permissions
 * *User.Read.All* is used to fetch user information for document-level permissions
@@ -546,14 +548,16 @@ Select an expiration date. (At this expiration date, you will need to generate a
 +
 ```
 Graph API
-- Sites.Read.All
+- Sites.Selected
 - Files.Read.All
 - Group.Read.All
 - User.Read.All
 
 Sharepoint
-- Sites.Read.All
+- Sites.Selected
 ```
+NOTE: If the `Comma-separated list of sites` configuration is set to `*` or if a user enables the toggle button `Enumerate all sites`, the connector requires `Sites.Read.All` permission.
+
 * **Grant admin consent**, using the `Grant Admin Consent` link from the permissions screen.
 * Save the tenant name (i.e. Domain name) of Azure platform.
 
@@ -597,7 +601,7 @@ Refer to https://learn.microsoft.com/en-us/sharepoint/dev/general-development/ho
 
 Here's a summary of why we use these Graph API permissions:
 
-* *Sites.Read.All* is used to fetch the sites and their metadata
+* *Sites.Selected* is used to fetch the sites and their metadata
 * *Files.Read.All* is used to fetch Site Drives and files in these drives
 * *Groups.Read.All* is used to fetch groups for document-level permissions
 * *User.Read.All* is used to fetch user information for document-level permissions

diff --git a/docs/reference/connector/docs/connectors-sharepoint.asciidoc b/docs/reference/connector/docs/connectors-sharepoint.asciidoc
@@ -67,6 +67,9 @@ The following SharePoint Server versions are compatible:
 
 The following configuration fields are required to set up the connector:
 
+`authentication`::
+Authentication mode, either *Basic* or *NTLM*.
+
 `username`::
 The username of the account for the SharePoint Server instance.
 
@@ -133,7 +136,7 @@ The connector syncs the following SharePoint object types:
 [NOTE]
 ====
 * Content from files bigger than 10 MB won't be extracted by default. Use the <<es-connectors-content-extraction-local, self-managed local extraction service>> to handle larger binary files.
-* Permissions are not synced. **All documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elasticsearch Index.
+* Permissions are not synced by default. Enable <<es-dls, document-level security (DLS)>> to sync permissions.
 ====
 
 [discrete#es-connectors-sharepoint-sync-types]
@@ -191,7 +194,7 @@ This connector is written in Python using the {connectors-python}[Elastic connec
 
 View the {connectors-python}/connectors/sources/sharepoint_server.py[source code for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_).
 
-// Closing the collapsible section 
+// Closing the collapsible section
 ===============
 
 
@@ -254,6 +257,9 @@ Once connected, you'll be able to update these values in Kibana.
 
 The following configuration fields are required to set up the connector:
 
+`authentication`::
+Authentication mode, either *Basic* or *NTLM*.
+
 `username`::
 The username of the account for the SharePoint Server instance.
 
@@ -408,5 +414,5 @@ This connector is written in Python using the {connectors-python}[Elastic connec
 
 View the {connectors-python}/connectors/sources/sharepoint_server.py[source code for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_).
 
-// Closing the collapsible section 
+// Closing the collapsible section
 ===============
diff --git a/...reference/connector/docs/release-notes/connectors-release-notes-8.16.0.asciidoc b/...reference/connector/docs/release-notes/connectors-release-notes-8.16.0.asciidoc
@@ -0,0 +1,53 @@
+[[es-connectors-release-notes-8-16-0]]
+=== 8.16.0 connectors release notes
+
+[discrete]
+[[es-connectors-release-notes-deprecation-notice]]
+==== Deprecation notices
+
+* *Direct index access for connectors and sync jobs*
++
+IMPORTANT: Directly accessing connector and sync job state through `.elastic-connectors*` indices is deprecated, and will be disallowed entirely in a future release.
+
+* Instead, the Elasticsearch Connector APIs should be used. Connectors framework code now uses the <<connector-apis,Connector APIs>> by default.
+See https://github.com/elastic/connectors/pull/2884[*PR 2902*].
+
+* *Docker `enterprise-search` namespace deprecation*
++
+IMPORTANT: The `enterprise-search` Docker namespace is deprecated and will be discontinued in a future release. 
++
+Starting in `8.16.0`, Docker images are being transitioned to the new `integrations` namespace, which will become the sole location for future releases. This affects the https://github.com/elastic/connectors[Elastic Connectors] and https://github.com/elastic/data-extraction-service[Elastic Data Extraction Service].
++
+During this transition period, images are published to both namespaces:
++
+** *Example*:
++
+Deprecated namespace::
+`docker.elastic.co/enterprise-search/elastic-connectors:v8.16.0`
++
+New namespace::
+`docker.elastic.co/integrations/elastic-connectors:v8.16.0`
++
+Users should migrate to the new `integrations` namespace as soon as possible to ensure continued access to future releases.
+
+[discrete]
+[[es-connectors-release-notes-8-16-0-enhancements]]
+==== Enhancements
+
+* Docker images now use Chainguard's Wolfi base image (`docker.elastic.co/wolfi/jdk:openjdk-11-dev`), replacing the previous `ubuntu:focal` base.
+
+* The Sharepoint Online connector now works with the `Sites.Selected` permission instead of the broader permission `Sites.Read.All`.
+See https://github.com/elastic/connectors/pull/2762[*PR 2762*].
+
+* Starting in 8.16.0, connectors will start using proper SEMVER, with `MAJOR.MINOR.PATCH`, which aligns with Elasticsearch/Kibana versions. This drops the previous `.BUILD` suffix, which we used to release connectors between Elastic stack releases. Going forward, these inter-stack-release releases will be suffixed instead with `+<timestamp>`, aligning with Elastic Agent and conforming to SEMVER. 
+See https://github.com/elastic/connectors/pull/2749[*PR 2749*].
+
+* Connector logs now use UTC timestamps, instead of machine-local timestamps. This only impacts logging output.
+See https://github.com/elastic/connectors/pull/2695[*PR 2695*].
+
+[discrete]
+[[es-connectors-release-notes-8-16-0-bug-fixes]]
+==== Bug fixes
+
+* The Dropbox connector now fetches the files from team shared folders.
+See https://github.com/elastic/connectors/pull/2718[*PR 2718*].
diff --git a/docs/reference/rest-api/security/bulk-create-roles.asciidoc b/docs/reference/rest-api/security/bulk-create-roles.asciidoc
@@ -1,7 +1,6 @@
 [role="xpack"]
 [[security-api-bulk-put-role]]
 === Bulk create or update roles API
-preview::[]
 ++++
 <titleabbrev>Bulk create or update roles API</titleabbrev>
 ++++
@@ -103,7 +102,9 @@ They have no effect for remote clusters configured with the <<remote-clusters-ce
 `clusters` (required)::: (list) A list of cluster aliases to which the permissions
 in this entry apply.
 `privileges`(required)::: (list) The cluster level privileges that the owners of the role
-have in the specified clusters.
+have in the specified clusters. Note - only a subset of the cluster privileges are supported for remote clusters.
+The <<security-api-get-builtin-privileges-request, builtin privileges API>> can be used to determine
+which privileges are allowed per version.
 
 
 For more information, see <<defining-roles>>.

diff --git a/docs/reference/rest-api/security/bulk-delete-roles.asciidoc b/docs/reference/rest-api/security/bulk-delete-roles.asciidoc
@@ -1,7 +1,6 @@
 [role="xpack"]
 [[security-api-bulk-delete-role]]
 === Bulk delete roles API
-preview::[]
 ++++
 <titleabbrev>Bulk delete roles API</titleabbrev>
 ++++