Merge branch 'main' into disable_ml_on_macos_x86_64

.buildkite/scripts/dra-workflow.sh

@@ -75,6 +75,7 @@ find "$WORKSPACE" -type d -path "*/build/distributions" -exec chmod a+w {} \;
 
 echo --- Running release-manager
 
+set +e
 # Artifacts should be generated
 docker run --rm \
   --name release-manager \
@@ -91,4 +92,16 @@ docker run --rm \
   --version "$ES_VERSION" \
   --artifact-set main \
   --dependency "beats:https://artifacts-${WORKFLOW}.elastic.co/beats/${BEATS_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json" \
-  --dependency "ml-cpp:https://artifacts-${WORKFLOW}.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json"
+  --dependency "ml-cpp:https://artifacts-${WORKFLOW}.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json" \
+2>&1 | tee release-manager.log
+EXIT_CODE=$?
+set -e
+
+# This failure is just generating a ton of noise right now, so let's just ignore it
+# This should be removed once this issue has been fixed
+if grep "elasticsearch-ubi-9.0.0-SNAPSHOT-docker-image.tar.gz" release-manager.log; then
+  echo "Ignoring error about missing ubi artifact"
+  exit 0
+fi
+
+exit "$EXIT_CODE"

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PomValidationTask.java

@@ -16,6 +16,8 @@
 import org.gradle.api.file.RegularFileProperty;
 import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.tasks.InputFile;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.TaskAction;
 
 import java.io.FileReader;
@@ -37,6 +39,7 @@ public PomValidationTask(ObjectFactory objects) {
     }
 
     @InputFile
+    @PathSensitive(PathSensitivity.RELATIVE)
     public RegularFileProperty getPomFile() {
         return pomFile;
     }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java

@@ -115,9 +115,9 @@ private static TaskProvider<LoggedExec> createRunBwcGradleTask(
 
             if (OS.current() == OS.WINDOWS) {
                 loggedExec.getExecutable().set("cmd");
-                loggedExec.args("/C", "call", new File(checkoutDir.get(), "gradlew").toString());
+                loggedExec.args("/C", "call", "gradlew");
             } else {
-                loggedExec.getExecutable().set(new File(checkoutDir.get(), "gradlew").toString());
+                loggedExec.getExecutable().set("./gradlew");
             }
 
             if (useUniqueUserHome) {
@@ -177,7 +177,7 @@ private static String readFromFile(File file) {
         }
     }
 
-    public static abstract class JavaHomeValueSource implements ValueSource<String, JavaHomeValueSource.Params> {
+    public abstract static class JavaHomeValueSource implements ValueSource<String, JavaHomeValueSource.Params> {
 
         private String minimumCompilerVersionPath(Version bwcVersion) {
             return (bwcVersion.onOrAfter(BUILD_TOOL_MINIMUM_VERSION))

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -22,7 +22,7 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:973431347ad45f40e01afbbd010bf9de929c088a63382239b90dd84f39618bc8",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:55b297da5151d2a2997e8ab9729fe1304e4869389d7090ab7031cc29530f69f8",
         "-wolfi",
         "apk"
     ),

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java

@@ -23,6 +23,7 @@
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.Copy;
+import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.jvm.toolchain.JavaToolchainService;
 import org.gradle.language.base.plugins.LifecycleBasePlugin;
@@ -322,7 +323,7 @@ static void createBuildBwcTask(
             File expectedOutputFile = useNativeExpanded
                 ? new File(projectArtifact.expandedDistDir, "elasticsearch-" + bwcVersion.get() + "-SNAPSHOT")
                 : projectArtifact.distFile;
-            c.getInputs().file(new File(project.getBuildDir(), "refspec"));
+            c.getInputs().file(new File(project.getBuildDir(), "refspec")).withPathSensitivity(PathSensitivity.RELATIVE);
             if (useNativeExpanded) {
                 c.getOutputs().dir(expectedOutputFile);
             } else {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java

@@ -19,6 +19,7 @@
 import org.gradle.api.plugins.quality.Checkstyle;
 import org.gradle.api.plugins.quality.CheckstyleExtension;
 import org.gradle.api.provider.Provider;
+import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.SourceSetContainer;
 import org.gradle.api.tasks.TaskProvider;
 
@@ -42,18 +43,23 @@ public TaskProvider<? extends Task> createTask(Project project) {
         File checkstyleSuppressions = new File(checkstyleDir, "checkstyle_suppressions.xml");
         File checkstyleConf = new File(checkstyleDir, "checkstyle.xml");
         TaskProvider<Task> copyCheckstyleConf = project.getTasks().register("copyCheckstyleConf");
-
         // configure inputs and outputs so up to date works properly
         copyCheckstyleConf.configure(t -> t.getOutputs().files(checkstyleSuppressions, checkstyleConf));
         if ("jar".equals(checkstyleConfUrl.getProtocol())) {
             try {
                 JarURLConnection jarURLConnection = (JarURLConnection) checkstyleConfUrl.openConnection();
-                copyCheckstyleConf.configure(t -> t.getInputs().file(jarURLConnection.getJarFileURL()));
+                copyCheckstyleConf.configure(
+                    t -> t.getInputs().file(jarURLConnection.getJarFileURL()).withPathSensitivity(PathSensitivity.RELATIVE)
+                );
             } catch (IOException e) {
                 throw new UncheckedIOException(e);
             }
         } else if ("file".equals(checkstyleConfUrl.getProtocol())) {
-            copyCheckstyleConf.configure(t -> t.getInputs().files(checkstyleConfUrl.getFile(), checkstyleSuppressionsUrl.getFile()));
+            copyCheckstyleConf.configure(
+                t -> t.getInputs()
+                    .files(checkstyleConfUrl.getFile(), checkstyleSuppressionsUrl.getFile())
+                    .withPathSensitivity(PathSensitivity.RELATIVE)
+            );
         }
 
         // Explicitly using an Action interface as java lambdas

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsTask.java

@@ -19,6 +19,8 @@
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.SkipWhenEmpty;
 import org.gradle.api.tasks.StopExecutionException;
 import org.gradle.api.tasks.TaskAction;
@@ -79,6 +81,7 @@ private static boolean isExecutableFile(File file) {
     @InputFiles
     @IgnoreEmptyDirectories
     @SkipWhenEmpty
+    @PathSensitive(PathSensitivity.RELATIVE)
     public FileCollection getFiles() {
         return getSources().get()
             .stream()

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestTestsTask.java

@@ -24,6 +24,8 @@
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.Optional;
 import org.gradle.api.tasks.OutputDirectory;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.SkipWhenEmpty;
 import org.gradle.api.tasks.TaskAction;
 import org.gradle.api.tasks.util.PatternFilterable;
@@ -106,6 +108,7 @@ public Map<String, String> getSubstitutions() {
     @SkipWhenEmpty
     @IgnoreEmptyDirectories
     @InputFiles
+    @PathSensitive(PathSensitivity.RELATIVE)
     public FileTree getInputDir() {
         FileTree coreFileTree = null;
         FileTree xpackFileTree = null;

build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java

@@ -20,6 +20,7 @@
 import org.gradle.api.provider.Property;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
+import org.gradle.api.tasks.CacheableTask;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.Optional;
@@ -53,6 +54,7 @@
  * Exec task implementation.
  */
 @SuppressWarnings("unchecked")
+@CacheableTask
 public abstract class LoggedExec extends DefaultTask implements FileSystemOperationsAware {
 
     private static final Logger LOGGER = Logging.getLogger(LoggedExec.class);
@@ -87,6 +89,14 @@ public abstract class LoggedExec extends DefaultTask implements FileSystemOperat
     abstract public Property<Boolean> getCaptureOutput();
 
     @Input
+    public Provider<String> getWorkingDirPath() {
+        return getWorkingDir().map(file -> {
+            String relativeWorkingDir = projectLayout.getProjectDirectory().getAsFile().toPath().relativize(file.toPath()).toString();
+            return relativeWorkingDir;
+        });
+    }
+
+    @Internal
     abstract public Property<File> getWorkingDir();
 
     @Internal
@@ -117,9 +127,10 @@ public LoggedExec(
      * can be reused across different build invocations.
      * */
     private void setupDefaultEnvironment(ProviderFactory providerFactory) {
-        getEnvironment().putAll(providerFactory.environmentVariablesPrefixedBy("BUILDKITE"));
         getEnvironment().putAll(providerFactory.environmentVariablesPrefixedBy("GRADLE_BUILD_CACHE"));
-        getEnvironment().putAll(providerFactory.environmentVariablesPrefixedBy("VAULT"));
+
+        getNonTrackedEnvironment().putAll(providerFactory.environmentVariablesPrefixedBy("BUILDKITE"));
+        getNonTrackedEnvironment().putAll(providerFactory.environmentVariablesPrefixedBy("VAULT"));
         Provider<String> javaToolchainHome = providerFactory.environmentVariable("JAVA_TOOLCHAIN_HOME");
         if (javaToolchainHome.isPresent()) {
             getEnvironment().put("JAVA_TOOLCHAIN_HOME", javaToolchainHome);

build-tools/src/main/java/org/elasticsearch/gradle/plugin/GeneratePluginPropertiesTask.java

@@ -19,10 +19,13 @@
 import org.gradle.api.file.RegularFileProperty;
 import org.gradle.api.provider.ListProperty;
 import org.gradle.api.provider.Property;
+import org.gradle.api.tasks.CacheableTask;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.Optional;
 import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.TaskAction;
 import org.objectweb.asm.ClassReader;
 import org.objectweb.asm.tree.ClassNode;
@@ -39,6 +42,7 @@
 
 import javax.inject.Inject;
 
+@CacheableTask
 public abstract class GeneratePluginPropertiesTask extends DefaultTask {
 
     public static final String PROPERTIES_FILENAME = "plugin-descriptor.properties";
@@ -82,6 +86,7 @@ public GeneratePluginPropertiesTask(ProjectLayout projectLayout) {
     public abstract Property<Boolean> getIsLicensed();
 
     @InputFiles
+    @PathSensitive(PathSensitivity.RELATIVE)
     public abstract ConfigurableFileCollection getModuleInfoFile();
 
     @OutputFile

docs/changelog/115020.yaml

@@ -0,0 +1,5 @@
+pr: 115020
+summary: Adding endpoint creation validation for all task types to remaining services
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/117246.yaml

@@ -0,0 +1,5 @@
+pr: 117246
+summary: LOOKUP JOIN using field-caps for field mapping
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/117312.yaml

@@ -0,0 +1,5 @@
+pr: 117312
+summary: Add missing `async_search` query parameters to rest-api-spec
+area: Search
+type: bug
+issues: []

docs/changelog/117404.yaml

@@ -0,0 +1,5 @@
+pr: 117404
+summary: Correct bit * byte and bit * float script comparisons
+area: Vector Search
+type: bug
+issues: []

docs/reference/data-streams/tsds-reindex.asciidoc

@@ -202,7 +202,7 @@ POST /_component_template/destination_template
 POST /_index_template/2
 {
   "index_patterns": [
-    "k8s*"
+    "k9s*"
   ],
   "composed_of": [
     "destination_template"

docs/reference/esql/esql-kibana.asciidoc

@@ -106,12 +106,27 @@ detailed warning, expand the query bar, and click *warnings*.
 ==== Query history
 
 You can reuse your recent {esql} queries in the query bar.
-In the query bar click *Show recent queries*.
+In the query bar, click *Show recent queries*.
 
 You can then scroll through your recent queries:
 
 image::images/esql/esql-discover-query-history.png[align="center",size="50%"]
 
+[discrete]
+[[esql-kibana-starred-queries]]
+==== Starred queries
+
+From the query history, you can mark some queries as favorite to find and access them faster later.
+
+In the query bar, click *Show recent queries*.
+
+From the **Recent** tab, you can star any queries you want.
+
+In the **Starred** tab, find all the queries you have previously starred.
+
+image::images/esql/esql-discover-query-starred.png[align="center",size="50%"]
+
+
 [discrete]
 [[esql-kibana-results-table]]
 === The results table