Merge branch 'elastic:main' into enable_chunking_settings

.buildkite/hooks/pre-command

@@ -78,11 +78,15 @@ if [[ "${USE_SNYK_CREDENTIALS:-}" == "true" ]]; then
 fi
 
 if [[ "${USE_PROD_DOCKER_CREDENTIALS:-}" == "true" ]]; then
-  DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
-  export DOCKER_REGISTRY_USERNAME
+  if which docker > /dev/null 2>&1; then
+    DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
+    export DOCKER_REGISTRY_USERNAME
 
-  DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
-  export DOCKER_REGISTRY_PASSWORD
+    DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
+    export DOCKER_REGISTRY_PASSWORD
+
+    docker login --username "$DOCKER_REGISTRY_USERNAME" --password "$DOCKER_REGISTRY_PASSWORD" docker.elastic.co
+  fi
 fi
 
 if [[ "$BUILDKITE_AGENT_META_DATA_PROVIDER" != *"k8s"* ]]; then

.buildkite/pipelines/periodic-packaging.yml

@@ -30,7 +30,8 @@ steps:
           image: family/elasticsearch-{{matrix.image}}
           diskSizeGb: 350
           machineType: n1-standard-8
-        env: {}
+        env:
+          USE_PROD_DOCKER_CREDENTIALS: "true"
   - group: packaging-tests-upgrade
     steps:
       - label: "{{matrix.image}} / 8.0.1 / packaging-tests-upgrade"

.ci/scripts/packaging-test.sh

@@ -77,5 +77,6 @@ sudo -E env \
   --unset=ES_JAVA_HOME \
   --unset=JAVA_HOME \
   SYSTEM_JAVA_HOME=`readlink -f -n $BUILD_JAVA_HOME` \
+  DOCKER_CONFIG="${HOME}/.docker" \
   ./gradlew -g $HOME/.gradle --scan --parallel --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ --continue $@
 

benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/SortBench.java

@@ -21,10 +21,12 @@
 
 package org.elasticsearch.benchmark.tdigest;
 
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.search.aggregations.metrics.MemoryTrackingTDigestArrays;
 import org.elasticsearch.tdigest.Sort;
+import org.elasticsearch.tdigest.arrays.TDigestArrays;
 import org.elasticsearch.tdigest.arrays.TDigestDoubleArray;
 import org.elasticsearch.tdigest.arrays.TDigestIntArray;
-import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -51,7 +53,8 @@
 @State(Scope.Thread)
 public class SortBench {
     private final int size = 100000;
-    private final TDigestDoubleArray values = WrapperTDigestArrays.INSTANCE.newDoubleArray(size);
+    private final TDigestArrays arrays = new MemoryTrackingTDigestArrays(new NoopCircuitBreaker("default-wrapper-tdigest-arrays"));
+    private final TDigestDoubleArray values = arrays.newDoubleArray(size);
 
     @Param({ "0", "1", "-1" })
     public int sortDirection;
@@ -72,7 +75,7 @@ public void setup() {
 
     @Benchmark
     public void stableSort() {
-        TDigestIntArray order = WrapperTDigestArrays.INSTANCE.newIntArray(size);
+        TDigestIntArray order = arrays.newIntArray(size);
         for (int i = 0; i < size; i++) {
             order.set(i, i);
         }

benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java

@@ -21,9 +21,11 @@
 
 package org.elasticsearch.benchmark.tdigest;
 
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.search.aggregations.metrics.MemoryTrackingTDigestArrays;
 import org.elasticsearch.tdigest.MergingDigest;
 import org.elasticsearch.tdigest.TDigest;
-import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays;
+import org.elasticsearch.tdigest.arrays.TDigestArrays;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -56,24 +58,25 @@
 @Threads(1)
 @State(Scope.Thread)
 public class TDigestBench {
+    private static final TDigestArrays arrays = new MemoryTrackingTDigestArrays(new NoopCircuitBreaker("default-wrapper-tdigest-arrays"));
 
     public enum TDigestFactory {
         MERGE {
             @Override
             TDigest create(double compression) {
-                return new MergingDigest(WrapperTDigestArrays.INSTANCE, compression, (int) (10 * compression));
+                return new MergingDigest(arrays, compression, (int) (10 * compression));
             }
         },
         AVL_TREE {
             @Override
             TDigest create(double compression) {
-                return TDigest.createAvlTreeDigest(WrapperTDigestArrays.INSTANCE, compression);
+                return TDigest.createAvlTreeDigest(arrays, compression);
             }
         },
         HYBRID {
             @Override
             TDigest create(double compression) {
-                return TDigest.createHybridDigest(WrapperTDigestArrays.INSTANCE, compression);
+                return TDigest.createHybridDigest(arrays, compression);
             }
         };
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java

@@ -30,6 +30,7 @@
 import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.TaskAction;
 import org.gradle.process.ExecOperations;
+import org.gradle.process.ExecSpec;
 import org.gradle.workers.WorkAction;
 import org.gradle.workers.WorkParameters;
 import org.gradle.workers.WorkerExecutor;
@@ -166,6 +167,7 @@ private void pullBaseImage(String baseImage) {
             for (int attempt = 1; attempt <= maxAttempts; attempt++) {
                 try {
                     LoggedExec.exec(execOperations, spec -> {
+                        maybeConfigureDockerConfig(spec);
                         spec.executable("docker");
                         spec.args("pull");
                         spec.args(baseImage);
@@ -181,6 +183,13 @@ private void pullBaseImage(String baseImage) {
             throw new GradleException("Failed to pull Docker base image [" + baseImage + "], all attempts failed");
         }
 
+        private void maybeConfigureDockerConfig(ExecSpec spec) {
+            String dockerConfig = System.getenv("DOCKER_CONFIG");
+            if (dockerConfig != null) {
+                spec.environment("DOCKER_CONFIG", dockerConfig);
+            }
+        }
+
         @Override
         public void execute() {
             final Parameters parameters = getParameters();
@@ -193,6 +202,8 @@ public void execute() {
             final boolean isCrossPlatform = isCrossPlatform();
 
             LoggedExec.exec(execOperations, spec -> {
+                maybeConfigureDockerConfig(spec);
+
                 spec.executable("docker");
 
                 if (isCrossPlatform) {

distribution/docker/src/docker/Dockerfile

@@ -163,9 +163,16 @@ RUN <%= retry.loop(package_manager,
           "      ${package_manager} update && \n" +
           "      ${package_manager} upgrade && \n" +
           "      ${package_manager} add --no-cache \n" +
-          "        bash ca-certificates curl libsystemd netcat-openbsd p11-kit p11-kit-trust shadow tini unzip zip zstd && \n" +
+          "        bash java-cacerts curl libstdc++ libsystemd netcat-openbsd p11-kit p11-kit-trust posix-libc-utils shadow tini unzip zip zstd && \n" +
           "      rm -rf /var/cache/apk/* "
      ) %>
+
+# Set Bash as the default shell for future commands
+SHELL ["/bin/bash", "-c"]
+
+# Optionally set Bash as the default shell in the container at runtime
+CMD ["/bin/bash"]
+
 <% } else if (docker_base == "default" || docker_base == "cloud") { %>
 
 # Change default shell to bash, then install required packages with retries.
@@ -224,7 +231,7 @@ COPY --from=builder --chown=0:0 /opt /opt
 <% } %>
 
 ENV PATH /usr/share/elasticsearch/bin:\$PATH
-
+ENV SHELL /bin/bash
 COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 
 # 1. Sync the user and group permissions of /etc/passwd
@@ -249,6 +256,8 @@ RUN chmod g=u /etc/passwd && \\
 # stays up-to-date with changes to Ubuntu's store)
 COPY bin/docker-openjdk /etc/ca-certificates/update.d/docker-openjdk
 RUN /etc/ca-certificates/update.d/docker-openjdk
+<% } else if (docker_base == 'wolfi') { %>
+RUN ln -sf /etc/ssl/certs/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts
 <% } else { %>
 RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts
 <% } %>

docs/changelog/112092.yaml

@@ -0,0 +1,5 @@
+pr: 112092
+summary: "Apply auto-flattening to `subobjects: auto`"
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/112723.yaml

@@ -0,0 +1,6 @@
+pr: 112723
+summary: Improve DateTime error handling and add some bad date tests
+area: Search
+type: bug
+issues:
+ - 112190

docs/changelog/112761.yaml

@@ -0,0 +1,6 @@
+pr: 112761
+summary: Fix collapse interaction with stored fields
+area: Search
+type: bug
+issues:
+ - 112646

docs/changelog/113276.yaml

@@ -0,0 +1,5 @@
+pr: 113276
+summary: Adding component template substitutions to the simulate ingest API
+area: Ingest Node
+type: enhancement
+issues: []

docs/changelog/113314.yaml

@@ -0,0 +1,6 @@
+pr: 113314
+summary: "[ES|QL] Check expression resolved before checking its data type in `ImplicitCasting`"
+area: ES|QL
+type: bug
+issues:
+ - 113242

docs/changelog/113499.yaml

@@ -0,0 +1,6 @@
+pr: 113499
+summary: Fix synthetic source for flattened field when used with `ignore_above`
+area: Logs
+type: bug
+issues:
+ - 112044

docs/reference/indices/put-component-template.asciidoc

@@ -129,14 +129,16 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout]
 [[put-component-template-api-request-body]]
 ==== {api-request-body-title}
 
+// tag::template[]
+
 `template`::
 (Required, object)
 This is the template to be applied, may optionally include a `mappings`,
 `settings`, or `aliases` configuration.
 +
 .Properties of `template`
 [%collapsible%open]
-====
+=====
 `aliases`::
 (Optional, object of objects) Aliases to add.
 +
@@ -147,7 +149,7 @@ include::{es-ref-dir}/indices/create-index.asciidoc[tag=aliases-props]
 include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=mappings]
 
 include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=settings]
-====
+=====
 
 `version`::
 (Optional, integer)
@@ -174,6 +176,7 @@ This map is not automatically generated by {es}.
 Marks this component template as deprecated.
 When a deprecated component template is referenced when creating or updating a non-deprecated index template,
 {es} will emit a deprecation warning.
+end::template[]
 
 [[put-component-template-api-example]]
 ==== {api-examples-title}

docs/reference/inference/delete-inference.asciidoc

@@ -49,13 +49,12 @@ The type of {infer} task that the model performs.
 
 `dry_run`::
 (Optional, Boolean)
-When `true`, checks the {infer} processors that reference the endpoint and
-returns them in a list, but does not delete the endpoint. Defaults to `false`.
+When `true`, checks the `semantic_text` fields and  {infer} processors that reference the endpoint and returns them in a list, but does not delete the endpoint.
+Defaults to `false`.
 
 `force`::
 (Optional, Boolean)
-Deletes the endpoint regardless if it's used in an {infer} pipeline or in a
-`semantic_text` field.
+Deletes the endpoint regardless if it's used in a `semantic_text` field or in an {infer} pipeline.
 
 
 [discrete]