Merge remote-tracking branch 'origin/main' into rework_rrf_to_work_th…

…rough_rewrite_phase
pmpailis · Sep 4, 2024 · b33fa45 · b33fa45
2 parents 3e0574a + d5bae2c
commit b33fa45
Show file tree

Hide file tree

Showing 537 changed files with 18,119 additions and 4,583 deletions.
diff --git a/.github/workflows/sync-main-to-jdk-branch.yml b/.github/workflows/sync-main-to-jdk-branch.yml
@@ -0,0 +1,20 @@
+# Daily update of JDK update branch with changes from main
+name: "Merge main to openjdk23-bundle branch"
+on:
+  schedule:
+    - cron: '30 17 * * *'
+  workflow_dispatch: {}
+
+jobs:
+  merge-branch:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@master
+
+      - name: merge
+        uses: devmasx/[email protected]
+        with:
+          type: 'now'
+          target_branch: openjdk23-bundle
+          github_token: ${{ secrets.ELASTICSEARCHMACHINE_TOKEN }}
diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle
@@ -1,4 +1,5 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
+import org.elasticsearch.gradle.internal.test.TestUtil
 
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
@@ -29,6 +30,7 @@ tasks.named("javadoc").configure { enabled = false }
 configurations {
   expression
   painless
+  nativeLib
 }
 
 dependencies {
@@ -45,6 +47,7 @@ dependencies {
   implementation project(path: ':libs:elasticsearch-simdvec')
   expression(project(path: ':modules:lang-expression', configuration: 'zip'))
   painless(project(path: ':modules:lang-painless', configuration: 'zip'))
+  nativeLib(project(':libs:elasticsearch-native'))
   api "org.openjdk.jmh:jmh-core:$versions.jmh"
   annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh"
   // Dependencies of JMH
@@ -76,17 +79,8 @@ tasks.register("copyPainless", Copy) {
 tasks.named("run").configure {
   executable = "${BuildParams.runtimeJavaHome}/bin/java"
   args << "-Dplugins.dir=${buildDir}/plugins" << "-Dtests.index=${buildDir}/index"
-  dependsOn "copyExpression", "copyPainless"
-  systemProperty 'es.nativelibs.path', file("../libs/native/libraries/build/platform/${platformName()}-${os.arch}")
-}
-
-String platformName() {
-  String name = System.getProperty("os.name");
-  if (name.startsWith("Mac")) {
-    return "darwin";
-  } else {
-    return name.toLowerCase(Locale.ROOT);
-  }
+  dependsOn "copyExpression", "copyPainless", configurations.nativeLib
+  systemProperty 'es.nativelibs.path', TestUtil.getTestLibraryPath(file("../libs/native/libraries/build/platform/").toString())
 }
 
 spotless {

diff --git a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle
@@ -19,11 +19,14 @@ import java.time.LocalDateTime
 develocity {
 
   buildScan {
-    URL jenkinsUrl = System.getenv('JENKINS_URL') ? new URL(System.getenv('JENKINS_URL')) : null
-    String buildKiteUrl = System.getenv('BUILDKITE_BUILD_URL') ? System.getenv('BUILDKITE_BUILD_URL') : null
+
+    def onCI = System.getenv('CI') ? Boolean.parseBoolean(System.getenv('CI')) : false
+
+    // Disable async upload in CI to ensure scan upload completes before CI agent is terminated
+    uploadInBackground = onCI == false
 
     // Automatically publish scans from Elasticsearch CI
-    if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev') || System.getenv('BUILDKITE') == 'true') {
+    if (onCI) {
       publishing.onlyIf { true }
       server = 'https://gradle-enterprise.elastic.co'
     } else if( server.isPresent() == false) {
@@ -38,73 +41,9 @@ develocity {
       if (BuildParams.inFipsJvm) {
         tag 'FIPS'
       }
-
-      // Jenkins-specific build scan metadata
-      if (jenkinsUrl) {
-        // Disable async upload in CI to ensure scan upload completes before CI agent is terminated
-        uploadInBackground = false
-
-        String buildNumber = System.getenv('BUILD_NUMBER')
-        String buildUrl = System.getenv('BUILD_URL')
-        String jobName = System.getenv('JOB_NAME')
-        String nodeName = System.getenv('NODE_NAME')
-        String jobBranch = System.getenv('ghprbTargetBranch') ?: System.getenv('JOB_BRANCH')
-
-        // Link to Jenkins worker logs and system metrics
-        if (nodeName) {
-          link 'System logs', "https://ci-stats.elastic.co/app/infra#/logs?&logFilter=(expression:'host.name:${nodeName}',kind:kuery)"
-          buildFinished {
-            link 'System metrics', "https://ci-stats.elastic.co/app/metrics/detail/host/${nodeName}"
-          }
-        }
-
-        // Parse job name in the case of matrix builds
-        // Matrix job names come in the form of "base-job-name/matrix_param1=value1,matrix_param2=value2"
-        def splitJobName = jobName.split('/')
-        if (splitJobName.length > 1 && splitJobName.last() ==~ /^([a-zA-Z0-9_\-]+=[a-zA-Z0-9_\-&\.]+,?)+$/) {
-          def baseJobName = splitJobName.dropRight(1).join('/')
-          tag baseJobName
-          tag splitJobName.last()
-          value 'Job Name', baseJobName
-          def matrixParams = splitJobName.last().split(',')
-          matrixParams.collect { it.split('=') }.each { param ->
-            value "MATRIX_${param[0].toUpperCase()}", param[1]
-          }
-        } else {
-          tag jobName
-          value 'Job Name', jobName
-        }
-
-        tag 'CI'
-        link 'CI Build', buildUrl
-        link 'GCP Upload',
-          "https://console.cloud.google.com/storage/browser/_details/elasticsearch-ci-artifacts/jobs/${URLEncoder.encode(jobName, "UTF-8")}/build/${buildNumber}.tar.bz2"
-        value 'Job Number', buildNumber
-        if (jobBranch) {
-          tag jobBranch
-          value 'Git Branch', jobBranch
-        }
-
-        System.getenv().getOrDefault('NODE_LABELS', '').split(' ').each {
-          value 'Jenkins Worker Label', it
-        }
-
-        // Add SCM information
-        def isPrBuild = System.getenv('ROOT_BUILD_CAUSE_GHPRBCAUSE') != null
-        if (isPrBuild) {
-          value 'Git Commit ID', System.getenv('ghprbActualCommit')
-          tag "pr/${System.getenv('ghprbPullId')}"
-          tag 'pull-request'
-          link 'Source', "https://github.com/elastic/elasticsearch/tree/${System.getenv('ghprbActualCommit')}"
-          link 'Pull Request', System.getenv('ghprbPullLink')
-        } else {
-          value 'Git Commit ID', BuildParams.gitRevision
-          link 'Source', "https://github.com/elastic/elasticsearch/tree/${BuildParams.gitRevision}"
-        }
-      } else if (buildKiteUrl) { //Buildkite-specific build scan metadata
-        // Disable async upload in CI to ensure scan upload completes before CI agent is terminated
-        uploadInBackground = false
-
+      println "onCI = $onCI"
+      if (onCI) { //Buildkite-specific build scan metadata
+        String buildKiteUrl = System.getenv('BUILDKITE_BUILD_URL')
         def branch = System.getenv('BUILDKITE_PULL_REQUEST_BASE_BRANCH') ?: System.getenv('BUILDKITE_BRANCH')
         def repoMatcher = System.getenv('BUILDKITE_REPO') =~ /(https:\/\/github\.com\/|git@github\.com:)(\S+)\.git/
         def repository = repoMatcher.matches() ? repoMatcher.group(2) : "<unknown>"

diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle
@@ -167,8 +167,8 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
             vmParameters = [
               '-ea',
               '-Djava.security.manager=allow',
-              '-Djava.locale.providers=SPI,COMPAT',
-              '-Des.nativelibs.path=' + testLibraryPath,
+              '-Djava.locale.providers=SPI,CLDR',
+              '-Des.nativelibs.path="' + testLibraryPath + '"',
               // TODO: only open these for mockito when it is modularized
               '--add-opens=java.base/java.security.cert=ALL-UNNAMED',
               '--add-opens=java.base/java.nio.channels=ALL-UNNAMED',

diff --git a/...internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java b/...internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java
@@ -92,7 +92,7 @@ public void execute(Task t) {
                     mkdirs(test.getWorkingDir().toPath().resolve("temp").toFile());
 
                     // TODO remove once jvm.options are added to test system properties
-                    test.systemProperty("java.locale.providers", "SPI,COMPAT");
+                    test.systemProperty("java.locale.providers", "SPI,CLDR");
                 }
             });
             test.getJvmArgumentProviders().add(nonInputProperties);

diff --git a/build-tools-internal/src/main/resources/changelog-schema.json b/build-tools-internal/src/main/resources/changelog-schema.json
@@ -28,6 +28,7 @@
             "Autoscaling",
             "CAT APIs",
             "CCR",
+            "CCS",
             "CRUD",
             "Client",
             "Cluster Coordination",

diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile
@@ -22,7 +22,7 @@
 <% if (docker_base == 'iron_bank') { %>
 ARG BASE_REGISTRY=registry1.dso.mil
 ARG BASE_IMAGE=ironbank/redhat/ubi/ubi9
-ARG BASE_TAG=9.3
+ARG BASE_TAG=9.4
 <% } %>
 
 ################################################################################

diff --git a/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml b/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml
@@ -14,7 +14,7 @@ tags:
 # Build args passed to Dockerfile ARGs
 args:
   BASE_IMAGE: "redhat/ubi/ubi9"
-  BASE_TAG: "9.3"
+  BASE_TAG: "9.4"
 
 # Docker image labels
 labels:

diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java
@@ -32,6 +32,7 @@
 import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.Locale;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * The main CLI for running Elasticsearch.
@@ -44,6 +45,8 @@ class ServerCli extends EnvironmentAwareCommand {
     private final OptionSpecBuilder quietOption;
     private final OptionSpec<String> enrollmentTokenOption;
 
+    // flag for indicating shutdown has begun. we use an AtomicBoolean to double as a synchronization object
+    private final AtomicBoolean shuttingDown = new AtomicBoolean(false);
     private volatile ServerProcess server;
 
     // visible for testing
@@ -98,7 +101,14 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce
             syncPlugins(terminal, env, processInfo);
 
             ServerArgs args = createArgs(options, env, secrets, processInfo);
-            this.server = startServer(terminal, processInfo, args);
+            synchronized (shuttingDown) {
+                // if we are shutting down there is no reason to start the server
+                if (shuttingDown.get()) {
+                    terminal.println("CLI is shutting down, skipping starting server process");
+                    return;
+                }
+                this.server = startServer(terminal, processInfo, args);
+            }
         }
 
         if (options.has(daemonizeOption)) {
@@ -233,8 +243,11 @@ private ServerArgs createArgs(OptionSet options, Environment env, SecureSettings
 
     @Override
     public void close() throws IOException {
-        if (server != null) {
-            server.stop();
+        synchronized (shuttingDown) {
+            shuttingDown.set(true);
+            if (server != null) {
+                server.stop();
+            }
         }
     }
 

diff --git a/...ibution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/...ibution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java
@@ -10,6 +10,7 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.UpdateForV9;
 
 import java.util.List;
 import java.util.Map;
@@ -59,11 +60,7 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                 "-Dlog4j.shutdownHookEnabled=false",
                 "-Dlog4j2.disable.jmx=true",
                 "-Dlog4j2.formatMsgNoLookups=true",
-                /*
-                 * Due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise time/date
-                 * parsing will break in an incompatible way for some date patterns and locales.
-                 */
-                "-Djava.locale.providers=SPI,COMPAT",
+                "-Djava.locale.providers=" + getLocaleProviders(),
                 maybeEnableNativeAccess(),
                 maybeOverrideDockerCgroup(distroType),
                 maybeSetActiveProcessorCount(nodeSettings),
@@ -75,6 +72,16 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
         ).filter(e -> e.isEmpty() == false).collect(Collectors.toList());
     }
 
+    @UpdateForV9    // only use CLDR in v9+
+    private static String getLocaleProviders() {
+        /*
+         * Specify SPI to load IsoCalendarDataProvider (see #48209), specifying the first day of week as Monday.
+         * When on pre-23, use COMPAT instead to maintain existing date formats as much as we can.
+         * When on JDK 23+, use the default CLDR locale database, as COMPAT was removed in JDK 23.
+         */
+        return Runtime.version().feature() >= 23 ? "SPI,CLDR" : "SPI,COMPAT";
+    }
+
     /*
      * The virtual file /proc/self/cgroup should list the current cgroup
      * membership. For each hierarchy, you can follow the cgroup path from