Merge branch 'main' into ml_metrics

.buildkite/packer_cache.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+ROOT_DIR=$(cd "$(dirname "$0")/.." && pwd)
+
+branches=($(cat "$ROOT_DIR/branches.json" | jq -r '.branches[].branch'))
+for branch in "${branches[@]}"; do
+  echo "Resolving dependencies for ${branch} branch"
+  rm -rf "checkout/$branch"
+  git clone /opt/git-mirrors/elastic-elasticsearch --branch "$branch" --single-branch "checkout/$branch"
+
+  CHECKOUT_DIR=$(cd "./checkout/${branch}" && pwd)
+  CI_DIR="$CHECKOUT_DIR/.ci"
+
+  if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
+    ## On ARM we use a different properties file for setting java home
+    ## Also, we don't bother attempting to resolve dependencies for the 6.8 branch
+    source "$CI_DIR/java-versions-aarch64.properties"
+    export JAVA16_HOME="$HOME/.java/jdk16"
+  else
+    source "$CI_DIR/java-versions.properties"
+    ## We are caching BWC versions too, need these so we can build those
+    export JAVA8_HOME="$HOME/.java/java8"
+    export JAVA11_HOME="$HOME/.java/java11"
+    export JAVA12_HOME="$HOME/.java/openjdk12"
+    export JAVA13_HOME="$HOME/.java/openjdk13"
+    export JAVA14_HOME="$HOME/.java/openjdk14"
+    export JAVA15_HOME="$HOME/.java/openjdk15"
+    export JAVA16_HOME="$HOME/.java/openjdk16"
+  fi
+
+  export JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA"
+  "checkout/${branch}/gradlew" --project-dir "$CHECKOUT_DIR" --parallel -s resolveAllDependencies -Dorg.gradle.warning.mode=none -DisCI
+  rm -rf "checkout/${branch}"
+done

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.16", "8.11.3", "8.12.0", "8.13.0"]
+            BWC_VERSION: ["7.17.17", "8.11.4", "8.12.0", "8.13.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1089,6 +1089,22 @@ steps:
         env:
           BWC_VERSION: 7.17.16
 
+      - label: "{{matrix.image}} / 7.17.17 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.17
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.17
+
       - label: "{{matrix.image}} / 8.0.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.0
         timeout_in_minutes: 300
@@ -1761,6 +1777,22 @@ steps:
         env:
           BWC_VERSION: 8.11.3
 
+      - label: "{{matrix.image}} / 8.11.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.4
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.11.4
+
       - label: "{{matrix.image}} / 8.12.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic-platform-support.yml

@@ -80,3 +80,19 @@ steps:
           diskName: /dev/sda1
         env:
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+  - group: platform-support-unix-aws
+    steps:
+      - label: "{{matrix.image}} / platform-support-aws"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true functionalTests
+        timeout_in_minutes: 420
+        matrix:
+          setup:
+            image:
+              - amazonlinux-2023
+        agents:
+          provider: aws
+          imagePrefix: elasticsearch-{{matrix.image}}
+          instanceType: m6a.8xlarge
+          diskSizeGb: 350
+          diskType: gp3
+          diskName: /dev/sda1

.buildkite/pipelines/periodic.yml

@@ -662,6 +662,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 7.17.16
+      - label: 7.17.17 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.17#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.17
       - label: 8.0.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.0.0#bwcTest
         timeout_in_minutes: 300
@@ -1082,6 +1092,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.11.3
+      - label: 8.11.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.4#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.11.4
       - label: 8.12.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.0#bwcTest
         timeout_in_minutes: 300

.buildkite/scripts/periodic.trigger.sh

@@ -6,11 +6,26 @@ echo "steps:"
 
 source .buildkite/scripts/branches.sh
 
+IS_FIRST=true
+SKIP_DELAY="${SKIP_DELAY:-false}"
+
 for BRANCH in "${BRANCHES[@]}"; do
   INTAKE_PIPELINE_SLUG="elasticsearch-intake"
   BUILD_JSON=$(curl -sH "Authorization: Bearer ${BUILDKITE_API_TOKEN}" "https://api.buildkite.com/v2/organizations/elastic/pipelines/${INTAKE_PIPELINE_SLUG}/builds?branch=${BRANCH}&state=passed&per_page=1" | jq '.[0] | {commit: .commit, url: .web_url}')
   LAST_GOOD_COMMIT=$(echo "${BUILD_JSON}" | jq -r '.commit')
 
+  # Put a delay between each branch's set of pipelines by prepending each non-first branch with a sleep
+  # This is to smooth out the spike in agent requests
+  if [[ "$IS_FIRST" != "true" && "$SKIP_DELAY" != "true" ]]; then
+      cat <<EOF
+  - command: sleep 540
+    soft_fail: true
+  - wait: ~
+    continue_on_failure: true
+EOF
+  fi
+  IS_FIRST=false
+
   cat <<EOF
   - trigger: elasticsearch-periodic
     label: Trigger periodic pipeline for $BRANCH

.ci/bwcVersions

@@ -65,6 +65,7 @@ BWC_VERSION:
   - "7.17.14"
   - "7.17.15"
   - "7.17.16"
+  - "7.17.17"
   - "8.0.0"
   - "8.0.1"
   - "8.1.0"
@@ -107,5 +108,6 @@ BWC_VERSION:
   - "8.11.1"
   - "8.11.2"
   - "8.11.3"
+  - "8.11.4"
   - "8.12.0"
   - "8.13.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
-  - "7.17.16"
-  - "8.11.3"
+  - "7.17.17"
+  - "8.11.4"
   - "8.12.0"
   - "8.13.0"

.github/CODEOWNERS

@@ -23,5 +23,6 @@ x-pack/plugin/core/src/main/resources/fleet-* @elastic/fleet
 # Kibana Security
 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @elastic/kibana-security
 
-# APM
-x-pack/plugin/apm-data @elastic/apm-server
+# APM Data index templates, etc.
+x-pack/plugin/apm-data/src/main/resources @elastic/apm-server
+x-pack/plugin/apm-data/src/yamlRestTest/resources @elastic/apm-server

build-conventions/build.gradle

@@ -92,3 +92,7 @@ project.getPlugins().withType(JavaBasePlugin.class) {
         }
     }
 }
+
+tasks.withType(JavaCompile).configureEach {
+  options.incremental = System.getenv("JENKINS_URL") == null && System.getenv("BUILDKITE_BUILD_URL") == null && System.getProperty("isCI") == null
+}

build-tools-internal/build.gradle

@@ -229,6 +229,7 @@ sourceSets {
 
 tasks.withType(JavaCompile).configureEach {
   options.encoding = 'UTF-8'
+  options.incremental = System.getenv("JENKINS_URL") == null && System.getenv("BUILDKITE_BUILD_URL") == null && System.getProperty("isCI") == null
 }
 
 tasks.named('licenseHeaders').configure {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java

@@ -109,6 +109,10 @@ private TaskProvider<LoggedExec> createRunBwcGradleTask(
                 loggedExec.args("-Dorg.elasticsearch.build.cache.url=" + buildCacheUrl);
             }
 
+            if (System.getProperty("isCI") != null) {
+                loggedExec.args("-DisCI");
+            }
+
             loggedExec.args("-Dbuild.snapshot=true", "-Dscan.tag.NESTED");
             final LogLevel logLevel = project.getGradle().getStartParameter().getLogLevel();
             List<LogLevel> nonDefaultLogLevels = Arrays.asList(LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java

@@ -132,6 +132,7 @@ public static void configureCompile(Project project) {
             compileTask.getConventionMapping().map("sourceCompatibility", () -> java.getSourceCompatibility().toString());
             compileTask.getConventionMapping().map("targetCompatibility", () -> java.getTargetCompatibility().toString());
             compileOptions.getRelease().set(releaseVersionProviderFromCompileTask(project, compileTask));
+            compileOptions.setIncremental(BuildParams.isCi() == false);
         });
         // also apply release flag to groovy, which is used in build-tools
         project.getTasks().withType(GroovyCompile.class).configureEach(compileTask -> {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java

@@ -124,7 +124,9 @@ public void apply(Project project) {
             params.setGitOrigin(gitInfo.getOrigin());
             params.setBuildDate(ZonedDateTime.now(ZoneOffset.UTC));
             params.setTestSeed(getTestSeed());
-            params.setIsCi(System.getenv("JENKINS_URL") != null || System.getenv("BUILDKITE_BUILD_URL") != null);
+            params.setIsCi(
+                System.getenv("JENKINS_URL") != null || System.getenv("BUILDKITE_BUILD_URL") != null || System.getProperty("isCI") != null
+            );
             params.setDefaultParallel(ParallelDetector.findDefaultParallel(project));
             params.setInFipsJvm(Util.getBooleanProperty("tests.fips.enabled", false));
             params.setIsSnapshotBuild(Util.getBooleanProperty("build.snapshot", true));

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/packer/CacheCacheableTestFixtures.java

@@ -25,7 +25,9 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.reflect.Constructor;
 import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLClassLoader;
@@ -74,9 +76,14 @@ public void execute() {
                 Set<Class<?>> classes = (Set<Class<?>>) reflections.getSubTypesOf(ifClass);
 
                 for (Class<?> cacheableTestFixtureClazz : classes) {
-                    Object o = cacheableTestFixtureClazz.getDeclaredConstructor().newInstance();
-                    Method cacheMethod = cacheableTestFixtureClazz.getMethod("cache");
-                    cacheMethod.invoke(o);
+                    if (Modifier.isAbstract(cacheableTestFixtureClazz.getModifiers()) == false) {
+                        Constructor<?> declaredConstructor = cacheableTestFixtureClazz.getDeclaredConstructor();
+                        declaredConstructor.setAccessible(true);
+                        Object o = declaredConstructor.newInstance();
+                        Method cacheMethod = cacheableTestFixtureClazz.getMethod("cache");
+                        System.out.println("Caching resources from " + cacheableTestFixtureClazz.getName());
+                        cacheMethod.invoke(o);
+                    }
                 }
             } catch (Exception e) {
                 throw new RuntimeException(e);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixtureTask.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.gradle.internal.testfixtures;
+
+import org.gradle.api.DefaultTask;
+import org.gradle.api.file.DirectoryProperty;
+import org.gradle.api.tasks.Internal;
+
+public abstract class TestFixtureTask extends DefaultTask {
+
+    @Internal
+    abstract DirectoryProperty getFixturesDir();
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java

@@ -81,37 +81,36 @@ public void apply(Project project) {
         );
 
         ExtraPropertiesExtension ext = project.getExtensions().getByType(ExtraPropertiesExtension.class);
-        File testfixturesDir = project.file("testfixtures_shared");
-        ext.set("testFixturesDir", testfixturesDir);
+        File testFixturesDir = project.file("testfixtures_shared");
+        ext.set("testFixturesDir", testFixturesDir);
 
         if (project.file(DOCKER_COMPOSE_YML).exists()) {
             project.getPluginManager().apply(BasePlugin.class);
             project.getPluginManager().apply(DockerComposePlugin.class);
-
-            TaskProvider<Task> preProcessFixture = project.getTasks().register("preProcessFixture", t -> {
-                t.doFirst(new Action<Task>() {
-                    @Override
-                    public void execute(Task task) {
-                        try {
-                            Files.createDirectories(testfixturesDir.toPath());
-                        } catch (IOException e) {
-                            throw new UncheckedIOException(e);
-                        }
+            TaskProvider<TestFixtureTask> preProcessFixture = project.getTasks().register("preProcessFixture", TestFixtureTask.class, t -> {
+                t.getFixturesDir().set(testFixturesDir);
+                t.doFirst(task -> {
+                    try {
+                        Files.createDirectories(testFixturesDir.toPath());
+                    } catch (IOException e) {
+                        throw new UncheckedIOException(e);
                     }
                 });
             });
             TaskProvider<Task> buildFixture = project.getTasks()
                 .register("buildFixture", t -> t.dependsOn(preProcessFixture, tasks.named("composeUp")));
 
-            TaskProvider<Task> postProcessFixture = project.getTasks().register("postProcessFixture", task -> {
-                task.dependsOn(buildFixture);
-                configureServiceInfoForTask(
-                    task,
-                    project,
-                    false,
-                    (name, port) -> task.getExtensions().getByType(ExtraPropertiesExtension.class).set(name, port)
-                );
-            });
+            TaskProvider<TestFixtureTask> postProcessFixture = project.getTasks()
+                .register("postProcessFixture", TestFixtureTask.class, task -> {
+                    task.getFixturesDir().set(testFixturesDir);
+                    task.dependsOn(buildFixture);
+                    configureServiceInfoForTask(
+                        task,
+                        project,
+                        false,
+                        (name, port) -> task.getExtensions().getByType(ExtraPropertiesExtension.class).set(name, port)
+                    );
+                });
 
             maybeSkipTask(dockerSupport, preProcessFixture);
             maybeSkipTask(dockerSupport, postProcessFixture);
@@ -138,7 +137,7 @@ public void execute(Task task) {
                 t.mustRunAfter(preProcessFixture);
             });
             tasks.named("composePull").configure(t -> t.mustRunAfter(preProcessFixture));
-            tasks.named("composeDown").configure(t -> t.doLast(t2 -> getFileSystemOperations().delete(d -> d.delete(testfixturesDir))));
+            tasks.named("composeDown").configure(t -> t.doLast(t2 -> getFileSystemOperations().delete(d -> d.delete(testFixturesDir))));
         } else {
             project.afterEvaluate(spec -> {
                 if (extension.fixtures.isEmpty()) {
@@ -179,7 +178,7 @@ private void maybeSkipTasks(TaskContainer tasks, Provider<DockerSupportService>
         tasks.withType(taskClass).configureEach(t -> maybeSkipTask(dockerSupport, t));
     }
 
-    private void maybeSkipTask(Provider<DockerSupportService> dockerSupport, TaskProvider<Task> task) {
+    private void maybeSkipTask(Provider<DockerSupportService> dockerSupport, TaskProvider<? extends Task> task) {
         task.configure(t -> maybeSkipTask(dockerSupport, t));
     }
 

build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt

@@ -163,6 +163,10 @@ org.elasticsearch.cluster.ClusterFeatures#allNodeFeatures()
 @defaultMessage ClusterFeatures#clusterHasFeature is for internal use only. Use FeatureService#clusterHasFeature to determine if a feature is present on the cluster.
 org.elasticsearch.cluster.ClusterFeatures#clusterHasFeature(org.elasticsearch.features.NodeFeature)
 
+@defaultMessage Do not construct this records outside the source files they are declared in
+org.elasticsearch.cluster.SnapshotsInProgress$ShardSnapshotStatus#<init>(java.lang.String, org.elasticsearch.cluster.SnapshotsInProgress$ShardState, org.elasticsearch.repositories.ShardGeneration, java.lang.String, org.elasticsearch.repositories.ShardSnapshotResult)
+org.elasticsearch.cluster.SnapshotDeletionsInProgress$Entry#<init>(java.lang.String, java.util.List, long, long, org.elasticsearch.cluster.SnapshotDeletionsInProgress$State, java.lang.String)
+
 @defaultMessage Use a Thread constructor with a name, anonymous threads are more difficult to debug
 java.lang.Thread#<init>(java.lang.Runnable)
 java.lang.Thread#<init>(java.lang.ThreadGroup, java.lang.Runnable)

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.13.0
-lucene            = 9.9.0
+lucene            = 9.9.0-snapshot-bb4fec631e6
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 21.0.1+12@415e3f918a1f4062a0074a2794853d0d