Merge remote-tracking branch 'upstream/main' into patch_source_mapping

.buildkite/scripts/lucene-snapshot/update-branch.sh

@@ -7,12 +7,21 @@ if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot"* ]]; then
   exit 1
 fi
 
-echo --- Updating "$BUILDKITE_BRANCH" branch with main
+if [[ "$BUILDKITE_BRANCH" == "lucene_snapshot_10" ]]; then
+  UPSTREAM="main"
+elif [[ "$BUILDKITE_BRANCH" == "lucene_snapshot" ]]; then
+  UPSTREAM="8.x"
+else
+  echo "Error: unknown branch: $BUILDKITE_BRANCH"
+  exit 1
+fi
+
+echo --- Updating "$BUILDKITE_BRANCH" branch with "$UPSTREAM"
 
 git config --global user.name elasticsearchmachine
 git config --global user.email '[email protected]'
 
 git checkout "$BUILDKITE_BRANCH"
-git fetch origin main
-git merge --no-edit origin/main
+git fetch origin "$UPSTREAM"
+git merge --no-edit "origin/$UPSTREAM"
 git push origin "$BUILDKITE_BRANCH"

benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/SortBench.java

@@ -22,6 +22,9 @@
 package org.elasticsearch.benchmark.tdigest;
 
 import org.elasticsearch.tdigest.Sort;
+import org.elasticsearch.tdigest.arrays.TDigestDoubleArray;
+import org.elasticsearch.tdigest.arrays.TDigestIntArray;
+import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -35,7 +38,6 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 
-import java.util.Arrays;
 import java.util.Random;
 import java.util.concurrent.TimeUnit;
 
@@ -49,7 +51,7 @@
 @State(Scope.Thread)
 public class SortBench {
     private final int size = 100000;
-    private final double[] values = new double[size];
+    private final TDigestDoubleArray values = WrapperTDigestArrays.INSTANCE.newDoubleArray(size);
 
     @Param({ "0", "1", "-1" })
     public int sortDirection;
@@ -58,22 +60,22 @@ public class SortBench {
     public void setup() {
         Random prng = new Random(999983);
         for (int i = 0; i < size; i++) {
-            values[i] = prng.nextDouble();
+            values.set(i, prng.nextDouble());
         }
         if (sortDirection > 0) {
-            Arrays.sort(values);
+            values.sort();
         } else if (sortDirection < 0) {
-            Arrays.sort(values);
-            Sort.reverse(values, 0, values.length);
+            values.sort();
+            Sort.reverse(values, 0, values.size());
         }
     }
 
     @Benchmark
-    public void quicksort() {
-        int[] order = new int[size];
+    public void stableSort() {
+        TDigestIntArray order = WrapperTDigestArrays.INSTANCE.newIntArray(size);
         for (int i = 0; i < size; i++) {
-            order[i] = i;
+            order.set(i, i);
         }
-        Sort.sort(order, values, null, values.length);
+        Sort.stableSort(order, values, values.size());
     }
 }

benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java

@@ -21,9 +21,9 @@
 
 package org.elasticsearch.benchmark.tdigest;
 
-import org.elasticsearch.tdigest.AVLTreeDigest;
 import org.elasticsearch.tdigest.MergingDigest;
 import org.elasticsearch.tdigest.TDigest;
+import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -61,13 +61,19 @@ public enum TDigestFactory {
         MERGE {
             @Override
             TDigest create(double compression) {
-                return new MergingDigest(compression, (int) (10 * compression));
+                return new MergingDigest(WrapperTDigestArrays.INSTANCE, compression, (int) (10 * compression));
             }
         },
         AVL_TREE {
             @Override
             TDigest create(double compression) {
-                return new AVLTreeDigest(compression);
+                return TDigest.createAvlTreeDigest(WrapperTDigestArrays.INSTANCE, compression);
+            }
+        },
+        HYBRID {
+            @Override
+            TDigest create(double compression) {
+                return TDigest.createHybridDigest(WrapperTDigestArrays.INSTANCE, compression);
             }
         };
 
@@ -77,7 +83,7 @@ TDigest create(double compression) {
     @Param({ "100", "300" })
     double compression;
 
-    @Param({ "MERGE", "AVL_TREE" })
+    @Param({ "MERGE", "AVL_TREE", "HYBRID" })
     TDigestFactory tdigestFactory;
 
     @Param({ "NORMAL", "GAUSSIAN" })

distribution/tools/geoip-cli/src/test/java/org/elasticsearch/geoip/GeoIpCliTests.java

@@ -20,7 +20,6 @@
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.XContentType;
 
-import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -106,9 +105,7 @@ private void verifyOverview() throws Exception {
     private void verifyTarball(Map<String, byte[]> data) throws Exception {
         for (String tgz : List.of("a.tgz", "b.tgz")) {
             try (
-                TarArchiveInputStream tis = new TarArchiveInputStream(
-                    new GZIPInputStream(new BufferedInputStream(Files.newInputStream(target.resolve(tgz))))
-                )
+                TarArchiveInputStream tis = new TarArchiveInputStream(new GZIPInputStream(Files.newInputStream(target.resolve(tgz)), 8192))
             ) {
                 TarArchiveEntry entry = tis.getNextTarEntry();
                 assertNotNull(entry);

distribution/tools/java-version-checker/build.gradle

@@ -1,30 +1,10 @@
 apply plugin: 'elasticsearch.build'
 
-sourceSets {
-  unsupportedJdkVersionEntrypoint
-}
-
-tasks.named(sourceSets.unsupportedJdkVersionEntrypoint.compileJavaTaskName).configure {
-  targetCompatibility = JavaVersion.VERSION_1_8
-}
-
-
-tasks.named("jar") {
-  manifest {
-    attributes("Multi-Release": "true")
-  }
-
-  FileCollection mainOutput = sourceSets.main.output;
-  from(sourceSets.unsupportedJdkVersionEntrypoint.output)
-  eachFile { details ->
-    if (details.path.equals("org/elasticsearch/tools/java_version_checker/JavaVersionChecker.class") &&
-      mainOutput.asFileTree.contains(details.file)) {
-      details.relativePath = details.relativePath.prepend("META-INF/versions/17")
-    }
-  }
+compileJava {
+  options.release = 8
 }
 
 // TODO revisit forbiddenApis issues
-["javadoc", "forbiddenApisMain", "forbiddenApisUnsupportedJdkVersionEntrypoint"].each {
+["javadoc", "forbiddenApisMain"].each {
   tasks.named(it).configure { enabled = false }
 }

distribution/tools/java-version-checker/src/main/java/org/elasticsearch/tools/java_version_checker/JavaVersionChecker.java

@@ -10,9 +10,10 @@
 package org.elasticsearch.tools.java_version_checker;
 
 import java.util.Arrays;
+import java.util.Locale;
 
 /**
- * Java 17 compatible main which just exits without error.
+ * Java 8 compatible main to check the runtime version
  */
 final class JavaVersionChecker {
 
@@ -23,5 +24,27 @@ public static void main(final String[] args) {
         if (args.length != 0) {
             throw new IllegalArgumentException("expected zero arguments but was " + Arrays.toString(args));
         }
+
+        final int MIN_VERSION = 21;
+        final int version;
+        String versionString = System.getProperty("java.specification.version");
+        if (versionString.equals("1.8")) {
+            version = 8;
+        } else {
+            version = Integer.parseInt(versionString);
+        }
+        if (version >= MIN_VERSION) {
+            return;
+        }
+
+        final String message = String.format(
+            Locale.ROOT,
+            "The minimum required Java version is %d; your Java version %d from [%s] does not meet that requirement.",
+            MIN_VERSION,
+            version,
+            System.getProperty("java.home")
+        );
+        System.err.println(message);
+        System.exit(1);
     }
 }

docs/changelog/112677.yaml

@@ -0,0 +1,5 @@
+pr: 112677
+summary: Stream OpenAI Completion
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/112678.yaml

@@ -0,0 +1,6 @@
+pr: 112678
+summary: Make "too many clauses" throw IllegalArgumentException to avoid 500s
+area: Search
+type: bug
+issues:
+ - 112177

docs/changelog/112888.yaml

@@ -0,0 +1,5 @@
+pr: 112888
+summary: Fix `getDatabaseType` for unusual MMDBs
+area: Ingest Node
+type: bug
+issues: []

docs/changelog/112916.yaml

@@ -0,0 +1,5 @@
+pr: 112916
+summary: Allow out of range term queries for numeric types
+area: Search
+type: bug
+issues: []

docs/changelog/112973.yaml

@@ -0,0 +1,5 @@
+pr: 112973
+summary: Fix verbose get data stream API not requiring extra privileges
+area: Data streams
+type: bug
+issues: []

docs/reference/release-notes/8.15.0.asciidoc

@@ -30,6 +30,19 @@ signed integer) may encounter errors (issue: {es-issue}111854[#111854])
 `xpack.security.authc.realms.*.files.role_mapping` configuration option. As a workaround, custom role mappings
 can be configured using the https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-put-role-mapping.html[REST API] (issue: {es-issue}112503[#112503])
 
+* ES|QL queries can lead to node crashes due to Out Of Memory errors when:
+** Multiple indices match the query pattern
+** These indices have many conflicting field mappings
+** Many of those fields are included in the request
+These issues deplete heap memory, increasing the likelihood of OOM errors. (issue: {es-issue}111964[#111964], {es-issue}111358[#111358]).
++
+To work around this issue, you have a number of options:
+** Downgrade to an earlier version
+** Upgrade to 8.15.2 upon release
+** Follow the instructions to
+<<esql-kibana-enable,disable ES|QL queries in {kib}>>
+** Change the default data view in Discover to a smaller set of indices and/or one with fewer mapping conflicts.
+
 [[breaking-8.15.0]]
 [float]
 === Breaking changes

docs/reference/release-notes/8.15.1.asciidoc

@@ -10,6 +10,19 @@ Also see <<breaking-changes-8.15,Breaking changes in 8.15>>.
 `xpack.security.authc.realms.*.files.role_mapping` configuration option. As a workaround, custom role mappings
 can be configured using the https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-put-role-mapping.html[REST API] (issue: {es-issue}112503[#112503])
 
+* ES|QL queries can lead to node crashes due to Out Of Memory errors when:
+** Multiple indices match the query pattern
+** These indices have many conflicting field mappings
+** Many of those fields are included in the request
+These issues deplete heap memory, increasing the likelihood of OOM errors. (issue: {es-issue}111964[#111964], {es-issue}111358[#111358]).
++
+To work around this issue, you have a number of options:
+** Downgrade to an earlier version
+** Upgrade to 8.15.2 upon release
+** Follow the instructions to
+<<esql-kibana-enable,disable ES|QL queries in {kib}>>
+** Change the default data view in Discover to a smaller set of indices and/or one with fewer mapping conflicts.
+
 [[bug-8.15.1]]
 [float]
 === Bug fixes

docs/reference/rest-api/security/get-service-accounts.asciidoc

@@ -66,7 +66,8 @@ GET /_security/service/elastic/fleet-server
       "cluster": [
         "monitor",
         "manage_own_api_key",
-        "read_fleet_secrets"
+        "read_fleet_secrets",
+        "cluster:admin/xpack/connector/*"
       ],
       "indices": [
         {
@@ -238,6 +239,35 @@ GET /_security/service/elastic/fleet-server
             "auto_configure"
           ],
           "allow_restricted_indices": false
+        },
+        {
+          "names": [
+            ".elastic-connectors*"
+          ],
+          "privileges": [
+            "read",
+            "write",
+            "monitor",
+            "create_index",
+            "auto_configure",
+            "maintenance"
+          ],
+          "allow_restricted_indices": false
+        },
+        {
+          "names": [
+            "content-*",
+            ".search-acl-filter-*"
+          ],
+          "privileges": [
+            "read",
+            "write",
+            "monitor",
+            "create_index",
+            "auto_configure",
+            "maintenance"
+          ],
+          "allow_restricted_indices": false
         }
       ],
       "applications": [

docs/reference/snapshot-restore/repository-source-only.asciidoc

@@ -18,7 +18,7 @@ stream or index.
 ==================================================
 
 Source-only snapshots are only supported if the `_source` field is enabled and no source-filtering is applied.
-When you restore a source-only snapshot:
+As a result, indices adopting synthetic source cannot be restored. When you restore a source-only snapshot:
 
  * The restored index is read-only and can only serve `match_all` search or scroll requests to enable reindexing.
 

libs/tdigest/src/main/java/module-info.java

@@ -19,4 +19,5 @@
 
 module org.elasticsearch.tdigest {
     exports org.elasticsearch.tdigest;
+    exports org.elasticsearch.tdigest.arrays;
 }