Merge branch 'main' into import-scores

davidkyle · Nov 8, 2023 · e0bdf54 · e0bdf54
2 parents 3f32c5d + e786cfa
commit e0bdf54
Show file tree

Hide file tree

Showing 426 changed files with 4,831 additions and 3,083 deletions.
diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
@@ -40,7 +40,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.15", "8.10.5", "8.11.0", "8.12.0"]
+            BWC_VERSION: ["7.17.15", "8.11.1", "8.12.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
@@ -1681,8 +1681,8 @@ steps:
         env:
           BWC_VERSION: 8.10.4
 
-      - label: "{{matrix.image}} / 8.10.5 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.10.5
+      - label: "{{matrix.image}} / 8.11.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.0
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -1695,10 +1695,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.10.5
+          BWC_VERSION: 8.11.0
 
-      - label: "{{matrix.image}} / 8.11.0 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.0
+      - label: "{{matrix.image}} / 8.11.1 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.1
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -1711,7 +1711,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.11.0
+          BWC_VERSION: 8.11.1
 
       - label: "{{matrix.image}} / 8.12.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.0

diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
@@ -1032,26 +1032,26 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.10.4
-      - label: 8.10.5 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.5#bwcTest
+      - label: 8.11.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
           machineType: custom-32-98304
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.10.5
-      - label: 8.11.0 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest
+          BWC_VERSION: 8.11.0
+      - label: 8.11.1 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.1#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
           machineType: custom-32-98304
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.11.0
+          BWC_VERSION: 8.11.1
       - label: 8.12.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.0#bwcTest
         timeout_in_minutes: 300

diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -102,6 +102,6 @@ BWC_VERSION:
   - "8.10.2"
   - "8.10.3"
   - "8.10.4"
-  - "8.10.5"
   - "8.11.0"
+  - "8.11.1"
   - "8.12.0"
diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
@@ -1,5 +1,4 @@
 BWC_VERSION:
   - "7.17.15"
-  - "8.10.5"
-  - "8.11.0"
+  - "8.11.1"
   - "8.12.0"
diff --git a/...a/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java b/...a/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java
@@ -224,11 +224,6 @@ public Set<String> getMatchingFieldNames(String pattern) {
             throw new UnsupportedOperationException();
         }
 
-        @Override
-        public boolean isFieldMapped(String field) {
-            return field.startsWith("int");
-        }
-
         @Override
         public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryType> context) {
             throw new UnsupportedOperationException();

diff --git a/branches.json b/branches.json
@@ -7,9 +7,6 @@
     {
       "branch": "8.11"
     },
-    {
-      "branch": "8.10"
-    },
     {
       "branch": "7.17"
     }

diff --git a/...ools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPlugin.groovy b/...ools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPlugin.groovy
@@ -12,6 +12,7 @@ import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.internal.test.rest.CopyRestApiTask
 import org.elasticsearch.gradle.internal.test.rest.CopyRestTestsTask
+import org.gradle.api.Action
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.file.Directory
@@ -61,16 +62,24 @@ class DocsTestPlugin implements Plugin<Project> {
             group 'Docs'
             description 'List each snippet'
             defaultSubstitutions = commonDefaultSubstitutions
-            perSnippet { println(it.toString()) }
+            perSnippet = new Action<SnippetsTask.Snippet>() {
+                @Override
+                void execute(SnippetsTask.Snippet snippet) {
+                    println(snippet.toString())
+                }
+            }
         }
         project.tasks.register('listConsoleCandidates', SnippetsTask) {
             group 'Docs'
             description
             'List snippets that probably should be marked // CONSOLE'
             defaultSubstitutions = commonDefaultSubstitutions
-            perSnippet {
-                if (RestTestsFromSnippetsTask.isConsoleCandidate(it)) {
-                    println(it.toString())
+            perSnippet = new Action<SnippetsTask.Snippet>() {
+                @Override
+                void execute(SnippetsTask.Snippet snippet) {
+                    if (RestTestsFromSnippetsTask.isConsoleCandidate(it)) {
+                        println(it.toString())
+                    }
                 }
             }
         }
@@ -80,7 +89,7 @@ class DocsTestPlugin implements Plugin<Project> {
             defaultSubstitutions = commonDefaultSubstitutions
             testRoot.convention(restRootDir)
             doFirst {
-                fileOperations.delete(restRootDir)
+                getFileOperations().delete(testRoot.get())
             }
         }
 

diff --git a/...al/src/main/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromSnippetsTask.groovy b/...al/src/main/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromSnippetsTask.groovy
@@ -10,8 +10,10 @@ package org.elasticsearch.gradle.internal.doc
 
 import groovy.transform.PackageScope
 import org.elasticsearch.gradle.internal.doc.SnippetsTask.Snippet
+import org.gradle.api.Action
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.file.DirectoryProperty
+import org.gradle.api.internal.file.FileOperations
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.Internal
 import org.gradle.api.tasks.OutputDirectory
@@ -24,7 +26,7 @@ import java.nio.file.Path
 /**
  * Generates REST tests for each snippet marked // TEST.
  */
-class RestTestsFromSnippetsTask extends SnippetsTask {
+abstract class RestTestsFromSnippetsTask extends SnippetsTask {
     /**
      * These languages aren't supported by the syntax highlighter so we
      * shouldn't use them.
@@ -64,13 +66,23 @@ class RestTestsFromSnippetsTask extends SnippetsTask {
     @Internal
     Set<String> names = new HashSet<>()
 
+    @Inject
+    abstract FileOperations getFileOperations();
+
     @Inject
     RestTestsFromSnippetsTask(ObjectFactory objectFactory) {
         testRoot = objectFactory.directoryProperty()
         TestBuilder builder = new TestBuilder()
-        perSnippet builder.&handleSnippet
-        doLast builder.&checkUnconverted
-        doLast builder.&finishLastTest
+        perSnippet = new Action<Snippet>() {
+            @Override
+            void execute(Snippet snippet) {
+                builder.handleSnippet(snippet)
+            }
+        }
+        doLast {
+            builder.checkUnconverted()
+            builder.finishLastTest()
+        }
     }
 
     /**
@@ -190,6 +202,7 @@ class RestTestsFromSnippetsTask extends SnippetsTask {
          * Called each time a snippet is encountered. Tracks the snippets and
          * calls buildTest to actually build the test.
          */
+
         void handleSnippet(Snippet snippet) {
             if (RestTestsFromSnippetsTask.isConsoleCandidate(snippet)) {
                 unconvertedCandidates.add(snippet.path.toString()

diff --git a/...-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/SnippetsTask.groovy b/...-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/SnippetsTask.groovy
@@ -11,8 +11,9 @@ package org.elasticsearch.gradle.internal.doc
 import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.core.JsonToken
 
+import org.gradle.api.Action;
 import org.gradle.api.DefaultTask
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.file.ConfigurableFileTree
@@ -44,7 +45,7 @@ class SnippetsTask extends DefaultTask {
      * instance of Snippet.
      */
     @Internal
-    Closure perSnippet
+    Action<Snippet> perSnippet
 
     /**
      * The docs to scan. Defaults to every file in the directory exception the
@@ -134,7 +135,7 @@ class SnippetsTask extends DefaultTask {
                         + "After substitutions and munging, the json looks like:\n" + quoted, e);
                     }
                 }
-                perSnippet(snippet)
+                perSnippet.execute(snippet)
                 snippet = null
             }
             file.eachLine('UTF-8') { String line, int lineNumber ->

diff --git a/...-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java b/...-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java
@@ -72,20 +72,19 @@ public void apply(Project project) {
             createClone.commandLine("git", "clone", buildLayout.getRootDirectory(), gitExtension.getCheckoutDir().get());
         });
 
-        ExtraPropertiesExtension extraProperties = project.getExtensions().getExtraProperties();
         TaskProvider<LoggedExec> findRemoteTaskProvider = tasks.register("findRemote", LoggedExec.class, findRemote -> {
             findRemote.dependsOn(createCloneTaskProvider);
             findRemote.getWorkingDir().set(gitExtension.getCheckoutDir());
             findRemote.commandLine("git", "remote", "-v");
             findRemote.getCaptureOutput().set(true);
-            findRemote.doLast(t -> { extraProperties.set("remoteExists", isRemoteAvailable(remote, findRemote.getOutput())); });
+            findRemote.doLast(t -> System.setProperty("remoteExists", String.valueOf(isRemoteAvailable(remote, findRemote.getOutput()))));
         });
 
         TaskProvider<Task> addRemoteTaskProvider = tasks.register("addRemote", addRemote -> {
             String rootProjectName = project.getRootProject().getName();
 
             addRemote.dependsOn(findRemoteTaskProvider);
-            addRemote.onlyIf("remote exists", task -> ((boolean) extraProperties.get("remoteExists")) == false);
+            addRemote.onlyIf("remote exists", task -> (Boolean.valueOf(providerFactory.systemProperty("remoteExists").get()) == false));
             addRemote.doLast(new Action<Task>() {
                 @Override
                 public void execute(Task task) {

diff --git a/...nal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckForbiddenApisTask.java b/...nal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckForbiddenApisTask.java
@@ -377,6 +377,7 @@ public void checkForbidden() {
             parameters.getTargetCompatibility().set(getTargetCompatibility());
             parameters.getIgnoreFailures().set(getIgnoreFailures());
             parameters.getSuccessMarker().set(getSuccessMarker());
+            parameters.getSignaturesFiles().from(getSignaturesFiles());
         });
     }
 

diff --git a/...internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestTestsTask.java b/...internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestTestsTask.java
@@ -15,6 +15,7 @@
 import org.gradle.api.file.FileSystemOperations;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.file.ProjectLayout;
+import org.gradle.api.internal.file.FileOperations;
 import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.ListProperty;
 import org.gradle.api.tasks.IgnoreEmptyDirectories;
@@ -43,7 +44,7 @@
  *
  * @see RestResourcesPlugin
  */
-public class CopyRestTestsTask extends DefaultTask {
+public abstract class CopyRestTestsTask extends DefaultTask {
     private static final String REST_TEST_PREFIX = "rest-api-spec/test";
     private final ListProperty<String> includeCore;
     private final ListProperty<String> includeXpack;
@@ -62,6 +63,9 @@ public class CopyRestTestsTask extends DefaultTask {
     private final ProjectLayout projectLayout;
     private final FileSystemOperations fileSystemOperations;
 
+    @Inject
+    public abstract FileOperations getFileOperations();
+
     @Inject
     public CopyRestTestsTask(
         ProjectLayout projectLayout,

diff --git a/...rg/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java b/...rg/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java
@@ -457,15 +457,17 @@ public void transform() throws IOException {
                     Collections.singletonList(new Skip(skippedFilesWithReason.get(file)))
                 );
             } else {
+                List<RestTestTransform<?>> transformations = new ArrayList<>(getTransformations().get());
+
                 if (skippedFilesWithTestAndReason.containsKey(file)) {
                     // skip the named tests for this file
                     skippedFilesWithTestAndReason.get(file).forEach(fullTestNameAndReasonPair -> {
                         String prefix = file.getName().replace(".yml", "/");
                         String singleTestName = fullTestNameAndReasonPair.getLeft().replaceAll(".*" + prefix, "");
-                        getTransformations().add(new Skip(singleTestName, fullTestNameAndReasonPair.getRight()));
+                        transformations.add(new Skip(singleTestName, fullTestNameAndReasonPair.getRight()));
                     });
                 }
-                transformRestTests = transformer.transformRestTests(new LinkedList<>(tests), getTransformations().get());
+                transformRestTests = transformer.transformRestTests(new LinkedList<>(tests), transformations);
             }
 
             // convert to url to ensure forward slashes

diff --git a/...level/src/main/java/org/elasticsearch/client/analytics/StringStatsAggregationBuilder.java b/...level/src/main/java/org/elasticsearch/client/analytics/StringStatsAggregationBuilder.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
-import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.xcontent.ParseField;
@@ -71,12 +70,6 @@ public String getType() {
         return NAME;
     }
 
-    @Override
-    protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
-        // This would be called from the same thing that calls innerBuild, which also throws. So it's "safe" to throw here.
-        throw new UnsupportedOperationException();
-    }
-
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         return builder.field(StringStatsAggregationBuilder.SHOW_DISTRIBUTION_FIELD.getPreferredName(), showDistribution);

diff --git a/...ibution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java b/...ibution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java
@@ -137,7 +137,7 @@ private List<String> jvmOptions(
         );
         substitutedJvmOptions.addAll(machineDependentHeap.determineHeapSettings(config, substitutedJvmOptions));
         final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
-        final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();
+        final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions(args.nodeSettings());
 
         final List<String> apmOptions = APMJvmOptions.apmJvmOptions(args.nodeSettings(), args.secrets(), args.logsDir(), tmpDir);
 

diff --git a/...ibution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/...ibution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java
@@ -8,13 +8,16 @@
 
 package org.elasticsearch.server.cli;
 
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+
 import java.util.List;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 final class SystemJvmOptions {
 
-    static List<String> systemJvmOptions() {
+    static List<String> systemJvmOptions(Settings nodeSettings) {
         return Stream.of(
             /*
              * Cache ttl in seconds for positive DNS lookups noting that this overrides the JDK security property networkaddress.cache.ttl;
@@ -61,7 +64,8 @@ static List<String> systemJvmOptions() {
              * explore alternatives. See org.elasticsearch.xpack.searchablesnapshots.preallocate.Preallocate.
              */
             "--add-opens=java.base/java.io=org.elasticsearch.preallocate",
-            maybeOverrideDockerCgroup()
+            maybeOverrideDockerCgroup(),
+            maybeSetActiveProcessorCount(nodeSettings)
         ).filter(e -> e.isEmpty() == false).collect(Collectors.toList());
     }
 
@@ -85,4 +89,16 @@ private static String maybeOverrideDockerCgroup() {
         }
         return "";
     }
+
+    /*
+     * node.processors determines thread pool sizes for Elasticsearch. When it
+     * is set, we need to also tell the JVM to respect a different value
+     */
+    private static String maybeSetActiveProcessorCount(Settings nodeSettings) {
+        if (EsExecutors.NODE_PROCESSORS_SETTING.exists(nodeSettings)) {
+            int allocated = EsExecutors.allocatedProcessors(nodeSettings);
+            return "-XX:ActiveProcessorCount=" + allocated;
+        }
+        return "";
+    }
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -7,9 +7,6 @@ @@
         {
           "branch": "8.11"
         },
-        {
-          "branch": "8.10"
-        },
         {
           "branch": "7.17"
         }
@@ Expand Down @@