Merge branch 'main' into disable_ml_on_macos_x86_64

droberts195 · Nov 28, 2024 · acf1976 · acf1976
2 parents a2e969d + f096c31
commit acf1976
Show file tree

Hide file tree

Showing 372 changed files with 7,955 additions and 2,775 deletions.
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -47,6 +47,8 @@ export GRADLE_BUILD_CACHE_PASSWORD
 BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/buildkite-api-token)
 export BUILDKITE_API_TOKEN
 
+export GH_TOKEN="$VAULT_GITHUB_TOKEN"
+
 if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then
   data=$(.buildkite/scripts/get-legacy-secret.sh aws-elastic/creds/lucene-snapshots)
 
@@ -117,3 +119,5 @@ if [[ -f /etc/os-release ]] && grep -q '"Amazon Linux 2"' /etc/os-release; then
   echo "$(hostname -i | cut -d' ' -f 2)  $(hostname -f)." | sudo tee /etc/dnsmasq.hosts
   sudo systemctl restart dnsmasq.service
 fi
+
+.buildkite/scripts/get-latest-test-mutes.sh
diff --git a/.buildkite/hooks/pre-command.bat b/.buildkite/hooks/pre-command.bat
@@ -15,9 +15,12 @@ set BUILD_NUMBER=%BUILDKITE_BUILD_NUMBER%
 set COMPOSE_HTTP_TIMEOUT=120
 set JOB_BRANCH=%BUILDKITE_BRANCH%
 
+set GH_TOKEN=%VAULT_GITHUB_TOKEN%
+
 set GRADLE_BUILD_CACHE_USERNAME=vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
 set GRADLE_BUILD_CACHE_PASSWORD=vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
 
 bash.exe -c "nohup bash .buildkite/scripts/setup-monitoring.sh </dev/null >/dev/null 2>&1 &"
+bash.exe -c "bash .buildkite/scripts/get-latest-test-mutes.sh"
 
 exit /b 0
diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
@@ -273,8 +273,8 @@ steps:
         env:
           BWC_VERSION: 8.14.3
 
-      - label: "{{matrix.image}} / 8.15.4 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.4
+      - label: "{{matrix.image}} / 8.15.6 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.6
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -287,7 +287,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.15.4
+          BWC_VERSION: 8.15.6
 
       - label: "{{matrix.image}} / 8.16.2 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.2

diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
@@ -287,8 +287,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.15.4 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.4#bwcTest
+      - label: 8.15.6 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.6#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -297,7 +297,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.15.4
+          BWC_VERSION: 8.15.6
         retry:
           automatic:
             - exit_status: "-1"

diff --git a/.buildkite/scripts/dra-workflow.sh b/.buildkite/scripts/dra-workflow.sh
@@ -75,7 +75,6 @@ find "$WORKSPACE" -type d -path "*/build/distributions" -exec chmod a+w {} \;
 
 echo --- Running release-manager
 
-set +e
 # Artifacts should be generated
 docker run --rm \
   --name release-manager \
@@ -92,16 +91,4 @@ docker run --rm \
   --version "$ES_VERSION" \
   --artifact-set main \
   --dependency "beats:https://artifacts-${WORKFLOW}.elastic.co/beats/${BEATS_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json" \
-  --dependency "ml-cpp:https://artifacts-${WORKFLOW}.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json" \
-2>&1 | tee release-manager.log
-EXIT_CODE=$?
-set -e
-
-# This failure is just generating a ton of noise right now, so let's just ignore it
-# This should be removed once this issue has been fixed
-if grep "elasticsearch-ubi-9.0.0-SNAPSHOT-docker-image.tar.gz" release-manager.log; then
-  echo "Ignoring error about missing ubi artifact"
-  exit 0
-fi
-
-exit "$EXIT_CODE"
+  --dependency "ml-cpp:https://artifacts-${WORKFLOW}.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json"
diff --git a/.buildkite/scripts/get-latest-test-mutes.sh b/.buildkite/scripts/get-latest-test-mutes.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+if [[ ! "${BUILDKITE_PULL_REQUEST:-}" || "${BUILDKITE_AGENT_META_DATA_PROVIDER:-}" == "k8s" ]]; then
+  exit 0
+fi
+
+testMuteBranch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
+testMuteFile="$(mktemp)"
+
+# If this PR contains changes to muted-tests.yml, we disable this functionality
+# Otherwise, we wouldn't be able to test unmutes
+if [[ ! $(gh pr diff "$BUILDKITE_PULL_REQUEST" --name-only | grep 'muted-tests.yml') ]]; then
+  gh api -H 'Accept: application/vnd.github.v3.raw' "repos/elastic/elasticsearch/contents/muted-tests.yml?ref=$testMuteBranch" > "$testMuteFile"
+
+  if [[ -s "$testMuteFile" ]]; then
+    mkdir -p ~/.gradle
+    # This is using gradle.properties instead of an env var so that it's easily compatible with the Windows pre-command hook
+    echo "org.gradle.project.org.elasticsearch.additional.muted.tests=$testMuteFile" >> ~/.gradle/gradle.properties
+  fi
+fi
diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -14,7 +14,7 @@ BWC_VERSION:
   - "8.12.2"
   - "8.13.4"
   - "8.14.3"
-  - "8.15.4"
+  - "8.15.6"
   - "8.16.2"
   - "8.17.0"
   - "8.18.0"

diff --git a/.editorconfig b/.editorconfig
@@ -209,7 +209,7 @@ indent_size = 4
 max_line_length = 140
 ij_java_class_count_to_use_import_on_demand = 999
 ij_java_names_count_to_use_import_on_demand = 999
-ij_java_imports_layout = *,|,com.**,|,org.**,|,java.**,javax.**,|,$*
+ij_java_imports_layout = *,|,com.**,|,org.**,|,java.**,|,javax.**,|,$*
 
 [*.json]
 indent_size = 2

diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java
@@ -77,7 +77,7 @@ public class ScriptScoreBenchmark {
     private final PluginsService pluginsService = new PluginsService(
         Settings.EMPTY,
         null,
-        new PluginsLoader(null, Path.of(System.getProperty("plugins.dir")))
+        PluginsLoader.createPluginsLoader(null, Path.of(System.getProperty("plugins.dir")))
     );
     private final ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, pluginsService.filterPlugins(ScriptPlugin.class).toList());
 

diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle
@@ -142,13 +142,18 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     description = 'Enables preview features on native library module'
     dependsOn tasks.named("enableExternalConfiguration")
 
-    doLast {
-      ['main', 'test'].each { sourceSet ->
-        modifyXml(".idea/modules/libs/native/elasticsearch.libs.native.${sourceSet}.iml") { xml ->
-          xml.component.find { it.'@name' == 'NewModuleRootManager' }?.'@LANGUAGE_LEVEL' = 'JDK_21_PREVIEW'
+    ext {
+      enablePreview = { moduleFile, languageLevel ->
+        modifyXml(moduleFile) { xml ->
+          xml.component.find { it.'@name' == 'NewModuleRootManager' }?.'@LANGUAGE_LEVEL' = languageLevel
         }
       }
     }
+
+    doLast {
+      enablePreview('.idea/modules/libs/native/elasticsearch.libs.native.main.iml', 'JDK_21_PREVIEW')
+      enablePreview('.idea/modules/libs/native/elasticsearch.libs.native.test.iml', 'JDK_21_PREVIEW')
+    }
   }
 
   tasks.register('buildDependencyArtifacts') {

diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java
@@ -22,7 +22,7 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:55b297da5151d2a2997e8ab9729fe1304e4869389d7090ab7031cc29530f69f8",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:32f06b169bb4b0f257fbb10e8c8379f06d3ee1355c89b3327cb623781a29590e",
         "-wolfi",
         "apk"
     ),

diff --git a/...internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestsBuildService.java b/...internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestsBuildService.java
@@ -28,10 +28,12 @@
 import java.io.UncheckedIOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Set;
 
 public abstract class MutedTestsBuildService implements BuildService<MutedTestsBuildService.Params> {
-    private final List<String> excludePatterns = new ArrayList<>();
+    private final Set<String> excludePatterns = new LinkedHashSet<>();
     private final ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory());
 
     public MutedTestsBuildService() {
@@ -43,23 +45,23 @@ public MutedTestsBuildService() {
         }
     }
 
-    public List<String> getExcludePatterns() {
+    public Set<String> getExcludePatterns() {
         return excludePatterns;
     }
 
-    private List<String> buildExcludePatterns(File file) {
+    private Set<String> buildExcludePatterns(File file) {
         List<MutedTest> mutedTests;
 
         try (InputStream is = new BufferedInputStream(new FileInputStream(file))) {
             mutedTests = objectMapper.readValue(is, MutedTests.class).getTests();
             if (mutedTests == null) {
-                return Collections.emptyList();
+                return Collections.emptySet();
             }
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
 
-        List<String> excludes = new ArrayList<>();
+        Set<String> excludes = new LinkedHashSet<>();
         if (mutedTests.isEmpty() == false) {
             for (MutedTestsBuildService.MutedTest mutedTest : mutedTests) {
                 if (mutedTest.getClassName() != null && mutedTest.getMethods().isEmpty() == false) {

diff --git a/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt b/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt
@@ -167,5 +167,3 @@ org.elasticsearch.cluster.SnapshotDeletionsInProgress$Entry#<init>(java.lang.Str
 @defaultMessage Use a Thread constructor with a name, anonymous threads are more difficult to debug
 java.lang.Thread#<init>(java.lang.Runnable)
 java.lang.Thread#<init>(java.lang.ThreadGroup, java.lang.Runnable)
-
-org.elasticsearch.common.bytes.BytesReference#copyBytes(org.elasticsearch.common.bytes.BytesReference) @ This method is a subject for removal. Copying bytes is prone to performance regressions and unnecessary allocations.
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
@@ -98,7 +98,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private static final int ES_DESTROY_TIMEOUT = 20;
     private static final TimeUnit ES_DESTROY_TIMEOUT_UNIT = TimeUnit.SECONDS;
 
-    private static final int NODE_UP_TIMEOUT = 2;
+    private static final int NODE_UP_TIMEOUT = 3;
     private static final TimeUnit NODE_UP_TIMEOUT_UNIT = TimeUnit.MINUTES;
     private static final int ADDITIONAL_CONFIG_TIMEOUT = 15;
     private static final TimeUnit ADDITIONAL_CONFIG_TIMEOUT_UNIT = TimeUnit.SECONDS;

diff --git a/docs/changelog/113120.yaml b/docs/changelog/113120.yaml
@@ -0,0 +1,5 @@
+pr: 113120
+summary: ESQL - enabling scoring with METADATA `_score`
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/114193.yaml b/docs/changelog/114193.yaml
diff --git a/docs/changelog/114227.yaml b/docs/changelog/114227.yaml
diff --git a/docs/changelog/114268.yaml b/docs/changelog/114268.yaml
diff --git a/docs/changelog/114317.yaml b/docs/changelog/114317.yaml
@@ -0,0 +1,5 @@
+pr: 114317
+summary: "ESQL: CATEGORIZE as a `BlockHash`"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/114521.yaml b/docs/changelog/114521.yaml
diff --git a/docs/changelog/114548.yaml b/docs/changelog/114548.yaml
diff --git a/docs/changelog/116277.yaml b/docs/changelog/116277.yaml
diff --git a/docs/changelog/116292.yaml b/docs/changelog/116292.yaml
diff --git a/docs/changelog/116357.yaml b/docs/changelog/116357.yaml
diff --git a/docs/changelog/116382.yaml b/docs/changelog/116382.yaml
diff --git a/docs/changelog/116408.yaml b/docs/changelog/116408.yaml
diff --git a/docs/changelog/116478.yaml b/docs/changelog/116478.yaml
diff --git a/docs/changelog/116650.yaml b/docs/changelog/116650.yaml
diff --git a/docs/changelog/116676.yaml b/docs/changelog/116676.yaml
diff --git a/docs/changelog/116739.yaml b/docs/changelog/116739.yaml
@@ -0,0 +1,5 @@
+pr: 116739
+summary: Change default Docker image to be based on UBI minimal instead of Ubuntu
+area: Infra/Core
+type: enhancement
+issues: []
diff --git a/docs/changelog/116915.yaml b/docs/changelog/116915.yaml
diff --git a/docs/changelog/116918.yaml b/docs/changelog/116918.yaml
diff --git a/docs/changelog/116942.yaml b/docs/changelog/116942.yaml
diff --git a/docs/changelog/116995.yaml b/docs/changelog/116995.yaml
diff --git a/docs/changelog/117182.yaml b/docs/changelog/117182.yaml
diff --git a/docs/changelog/117265.yaml b/docs/changelog/117265.yaml
@@ -0,0 +1,5 @@
+pr: 117265
+summary: Async search responses have CCS metadata while searches are running
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/117303.yaml b/docs/changelog/117303.yaml
@@ -0,0 +1,5 @@
+pr: 117303
+summary: Remove HTTP content copies
+area: Network
+type: enhancement
+issues: []
diff --git a/docs/changelog/117503.yaml b/docs/changelog/117503.yaml
@@ -0,0 +1,6 @@
+pr: 117503
+summary: Fix COUNT filter pushdown
+area: ES|QL
+type: bug
+issues:
+ - 115522
diff --git a/docs/changelog/117551.yaml b/docs/changelog/117551.yaml
@@ -0,0 +1,5 @@
+pr: 117551
+summary: Fix stats by constant expresson with alias
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/117595.yaml b/docs/changelog/117595.yaml
@@ -0,0 +1,5 @@
+pr: 117595
+summary: Fix for Deberta tokenizer when input sequence exceeds 512 tokens
+area: Machine Learning
+type: bug
+issues: []
diff --git a/docs/changelog/117618.yaml b/docs/changelog/117618.yaml
@@ -0,0 +1,5 @@
+pr: 117618
+summary: SearchStatesIt failures reported by CI
+area: Search
+type: bug
+issues: [116617, 116618]