Merge branch 'main' into 13531-range-agg

.github/workflows/gradle-check.yml

@@ -14,7 +14,7 @@ permissions:
 jobs:
   check-files:
     runs-on: ubuntu-latest
-    outputs: 
+    outputs:
       RUN_GRADLE_CHECK: ${{ steps.changed-files-specific.outputs.any_changed }}
     steps:
     - uses: actions/checkout@v4
@@ -26,7 +26,7 @@ jobs:
           release-notes/*.md
           .github/**
           *.md
-    
+
   gradle-check:
     needs: check-files
     if: github.repository == 'opensearch-project/OpenSearch' && needs.check-files.outputs.RUN_GRADLE_CHECK == 'true'
@@ -158,20 +158,11 @@ jobs:
 
             Please examine the workflow log, locate, and copy-paste the failure(s) below, then iterate to green. Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?
 
-      - name: Create Issue On Push Failure
-        if: ${{ github.event_name == 'push' && failure() }}
-        uses: dblock/create-a-github-issue@v3
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          assignees: ${{ github.event.head_commit.author.username }}, ${{ github.triggering_actor }}
-          filename: .github/ISSUE_TEMPLATE/failed_check.md
-
   check-result:
     needs: [check-files, gradle-check]
     if: always()
     runs-on: ubuntu-latest
     steps:
       - name: Fail if gradle-check fails
         if: ${{ needs.check-files.outputs.RUN_GRADLE_CHECK && needs.gradle-check.result == 'failure' }}
-        run: exit 1
+        run: exit 1

CHANGELOG-3.0.md

@@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Dependencies
 
 ### Changed
+- Changed locale provider from COMPAT to CLDR  ([#14345](https://github.com/opensearch-project/OpenSearch/pull/14345))
 - Migrate client transports to Apache HttpClient / Core 5.x ([#4459](https://github.com/opensearch-project/OpenSearch/pull/4459))
 - Change http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773))
 - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792))

CHANGELOG.md

@@ -5,11 +5,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x]
 ### Added
+- Add fingerprint ingest processor ([#13724](https://github.com/opensearch-project/OpenSearch/pull/13724))
+- [Remote Store] Rate limiter for remote store low priority uploads ([#14374](https://github.com/opensearch-project/OpenSearch/pull/14374/))
 - Apply the date histogram rewrite optimization to range aggregation ([#13865](https://github.com/opensearch-project/OpenSearch/pull/13865))
 
 ### Dependencies
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))
 - Update to Apache Lucene 9.11.0 ([#14042](https://github.com/opensearch-project/OpenSearch/pull/14042))
+- Bump `netty` from 4.1.110.Final to 4.1.111.Final ([#14356](https://github.com/opensearch-project/OpenSearch/pull/14356))
+- Bump `org.wiremock:wiremock-standalone` from 3.3.1 to 3.6.0 ([#14361](https://github.com/opensearch-project/OpenSearch/pull/14361))
+- Bump `reactor` from 3.5.17 to 3.5.18 ([#14395](https://github.com/opensearch-project/OpenSearch/pull/14395))
+- Bump `reactor-netty` from 1.1.19 to 1.1.20 ([#14395](https://github.com/opensearch-project/OpenSearch/pull/14395))
+- Bump `commons-net:commons-net` from 3.10.0 to 3.11.1 ([#14396](https://github.com/opensearch-project/OpenSearch/pull/14396))
+- Bump `com.nimbusds:nimbus-jose-jwt` from 9.37.3 to 9.40 ([#14398](https://github.com/opensearch-project/OpenSearch/pull/14398))
+- Bump `org.apache.commons:commons-configuration2` from 2.10.1 to 2.11.0 ([#14399](https://github.com/opensearch-project/OpenSearch/pull/14399))
+- Bump `com.gradle.develocity` from 3.17.4 to 3.17.5 ([#14397](https://github.com/opensearch-project/OpenSearch/pull/14397))
 
 ### Changed
 
@@ -18,6 +28,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Removed
 
 ### Fixed
+- Fix handling of Short and Byte data types in ScriptProcessor ingest pipeline ([#14379](https://github.com/opensearch-project/OpenSearch/issues/14379))
 
 ### Security
 

DEVELOPER_GUIDE.md

@@ -651,16 +651,18 @@ Note that these snapshots do not follow the Maven [naming convention](https://ma
 
 ### Flaky Tests
 
-OpenSearch has a very large test suite with long running, often failing (flaky), integration tests. Such individual tests are labelled as [Flaky Random Test Failure](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aopen+is%3Aissue+label%3A%22flaky-test%22). Your help is wanted fixing these!
+If you encounter a test failure locally or in CI that is seemingly unrelated to the change in your pull request, it may be a known flaky test or a new test failure. OpenSearch has a very large test suite with long running, often failing (flaky), integration tests. Such individual tests are labelled as [Flaky Random Test Failure](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aopen+is%3Aissue+label%3A%22flaky-test%22). Your help is wanted fixing these!
 
-If you encounter a build/test failure in CI that is unrelated to the change in your pull request, it may be a known flaky test, or a new test failure.
+The automation [gradle-check-flaky-test-detector](https://build.ci.opensearch.org/job/gradle-check-flaky-test-detector/), which runs in OpenSearch public Jenkins, identifies failing flaky issues that are part of post-merge actions. Once a flaky test is identified, the automation creates an issue with detailed report that includes links to all relevant commits, the Gradle check build log, the test report, and pull requests that are impacted with the flaky test failures. This automation leverages data from the [OpenSearch Metrics Project](https://github.com/opensearch-project/opensearch-metrics) to establish a baseline for creating the issue and updating the flaky test report. For all flaky test issues created by automation, visit this [link](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A%3Etest-failure+author%3Aopensearch-ci-bot).
+
+If you still see a failing test that is not part of the post merge actions, please do:
+
+* Follow failed CI links, and locate the failing test(s) or use the [Gradle Check Metrics Dashboard](#gradle-check-metrics-dashboard).
+* Copy-paste the failure into a comment of your PR.
+* Search through issues using the name of the failed test for whether this is a known flaky test.
+* If no existing issue is found, open one.
+* Retry CI via the GitHub UX or by pushing an update to your PR.
 
-1. Follow failed CI links, and locate the failing test(s).
-2. Copy-paste the failure into a comment of your PR.
-3. Search through [issues](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aopen+is%3Aissue+label%3A%22flaky-test%22) using the name of the failed test for whether this is a known flaky test.
-4. If an existing issue is found, paste a link to the known issue in a comment to your PR.
-5. If no existing issue is found, open one.
-6. Retry CI via the GitHub UX or by pushing an update to your PR.
 
 ### Gradle Check Metrics Dashboard
 

buildSrc/build.gradle

@@ -128,7 +128,7 @@ dependencies {
   testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
   testFixturesApi gradleApi()
   testFixturesApi gradleTestKit()
-  testImplementation 'org.wiremock:wiremock-standalone:3.3.1'
+  testImplementation 'org.wiremock:wiremock-standalone:3.6.0'
   testImplementation "org.mockito:mockito-core:${props.getProperty('mockito')}"
   integTestImplementation('org.spockframework:spock-core:2.3-groovy-3.0') {
     exclude module: "groovy"

buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java

@@ -110,7 +110,7 @@ public void execute(Task t) {
                     if (BuildParams.getRuntimeJavaVersion() == JavaVersion.VERSION_1_8) {
                         test.systemProperty("java.locale.providers", "SPI,JRE");
                     } else {
-                        test.systemProperty("java.locale.providers", "SPI,COMPAT");
+                        test.systemProperty("java.locale.providers", "SPI,CLDR");
                         if (test.getJavaVersion().compareTo(JavaVersion.VERSION_17) < 0) {
                             test.jvmArgs("--illegal-access=warn");
                         }

buildSrc/src/main/java/org/opensearch/gradle/info/GlobalBuildInfoPlugin.java

@@ -199,7 +199,28 @@ private JavaVersion determineJavaVersion(String description, File javaHome, Java
     }
 
     private JvmInstallationMetadata getJavaInstallation(File javaHome) {
-        final InstallationLocation location = new InstallationLocation(javaHome, "Java home");
+        InstallationLocation location = null;
+
+        try {
+            try {
+                // The InstallationLocation(File, String) is used by Gradle pre-8.8
+                location = (InstallationLocation) MethodHandles.publicLookup()
+                    .findConstructor(InstallationLocation.class, MethodType.methodType(void.class, File.class, String.class))
+                    .invokeExact(javaHome, "Java home");
+            } catch (Throwable ex) {
+                // The InstallationLocation::userDefined is used by Gradle post-8.7
+                location = (InstallationLocation) MethodHandles.publicLookup()
+                    .findStatic(
+                        InstallationLocation.class,
+                        "userDefined",
+                        MethodType.methodType(InstallationLocation.class, File.class, String.class)
+                    )
+                    .invokeExact(javaHome, "Java home");
+
+            }
+        } catch (Throwable ex) {
+            throw new IllegalStateException("Unable to find suitable InstallationLocation constructor / factory method", ex);
+        }
 
         try {
             try {

buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java

@@ -148,8 +148,8 @@ private void configureGeneralTaskDefaults(Project project) {
         project.getTasks().withType(AbstractCopyTask.class).configureEach(t -> {
             t.dependsOn(project.getTasks().withType(EmptyDirTask.class));
             t.setIncludeEmptyDirs(true);
-            t.setDirMode(0755);
-            t.setFileMode(0644);
+            t.dirPermissions(perms -> perms.unix(0755));
+            t.filePermissions(perms -> perms.unix(0644));
         });
 
         // common config across all archives

buildSrc/src/main/java/org/opensearch/gradle/precommit/DependencyLicensesPrecommitPlugin.java

@@ -33,11 +33,14 @@
 package org.opensearch.gradle.precommit;
 
 import org.opensearch.gradle.dependencies.CompileOnlyResolvePlugin;
+import org.opensearch.gradle.util.GradleUtils;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.ProjectDependency;
+import org.gradle.api.file.FileCollection;
 import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.api.provider.Provider;
 import org.gradle.api.tasks.TaskProvider;
 
 public class DependencyLicensesPrecommitPlugin extends PrecommitPlugin {
@@ -48,15 +51,16 @@ public TaskProvider<? extends Task> createTask(Project project) {
         TaskProvider<DependencyLicensesTask> dependencyLicenses = project.getTasks()
             .register("dependencyLicenses", DependencyLicensesTask.class);
 
+        final Configuration runtimeClasspath = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
+        final Configuration compileOnly = project.getConfigurations()
+            .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
+        final Provider<FileCollection> provider = project.provider(
+            () -> GradleUtils.getFiles(project, runtimeClasspath, dependency -> dependency instanceof ProjectDependency == false)
+                .minus(compileOnly)
+        );
+
         // only require dependency licenses for non-opensearch deps
-        dependencyLicenses.configure(t -> {
-            Configuration runtimeClasspath = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
-            Configuration compileOnly = project.getConfigurations()
-                .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
-            t.setDependencies(
-                runtimeClasspath.fileCollection(dependency -> dependency instanceof ProjectDependency == false).minus(compileOnly)
-            );
-        });
+        dependencyLicenses.configure(t -> t.getDependencies().set(provider));
 
         // we also create the updateShas helper task that is associated with dependencyLicenses
         project.getTasks().register("updateShas", UpdateShasTask.class, t -> t.setParentTask(dependencyLicenses));

buildSrc/src/main/java/org/opensearch/gradle/precommit/DependencyLicensesTask.java

@@ -39,6 +39,7 @@
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
+import org.gradle.api.provider.Property;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputDirectory;
 import org.gradle.api.tasks.InputFiles;
@@ -121,7 +122,7 @@ public class DependencyLicensesTask extends DefaultTask {
     /**
      * A collection of jar files that should be checked.
      */
-    private FileCollection dependencies;
+    private Property<FileCollection> dependenciesProvider;
 
     /**
      * The directory to find the license and sha files in.
@@ -158,12 +159,11 @@ public void mapping(Map<String, String> props) {
     }
 
     @InputFiles
-    public FileCollection getDependencies() {
-        return dependencies;
-    }
-
-    public void setDependencies(FileCollection dependencies) {
-        this.dependencies = dependencies;
+    public Property<FileCollection> getDependencies() {
+        if (dependenciesProvider == null) {
+            dependenciesProvider = getProject().getObjects().property(FileCollection.class);
+        }
+        return dependenciesProvider;
     }
 
     @Optional
@@ -190,6 +190,11 @@ public void ignoreSha(String dep) {
 
     @TaskAction
     public void checkDependencies() throws IOException, NoSuchAlgorithmException {
+        if (dependenciesProvider == null) {
+            throw new GradleException("No dependencies variable defined.");
+        }
+
+        final FileCollection dependencies = dependenciesProvider.get();
         if (dependencies == null) {
             throw new GradleException("No dependencies variable defined.");
         }
@@ -226,7 +231,7 @@ public void checkDependencies() throws IOException, NoSuchAlgorithmException {
             }
         }
 
-        checkDependencies(licenses, notices, sources, shaFiles);
+        checkDependencies(dependencies, licenses, notices, sources, shaFiles);
 
         licenses.forEach((item, exists) -> failIfAnyMissing(item, exists, "license"));
 
@@ -255,6 +260,7 @@ private void failIfAnyMissing(String item, Boolean exists, String type) {
     }
 
     private void checkDependencies(
+        FileCollection dependencies,
         Map<String, Boolean> licenses,
         Map<String, Boolean> notices,
         Map<String, Boolean> sources,

buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditTask.java

@@ -37,6 +37,7 @@
 import org.opensearch.gradle.LoggedExec;
 import org.opensearch.gradle.OS;
 import org.opensearch.gradle.dependencies.CompileOnlyResolvePlugin;
+import org.opensearch.gradle.util.GradleUtils;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.JavaVersion;
 import org.gradle.api.artifacts.Configuration;
@@ -203,11 +204,13 @@ public Set<File> getJarsToScan() {
         // or dependencies added as `files(...)`, we can't be sure if those are third party or not.
         // err on the side of scanning these to make sure we don't miss anything
         Spec<Dependency> reallyThirdParty = dep -> dep.getGroup() != null && dep.getGroup().startsWith("org.opensearch") == false;
-        Set<File> jars = getRuntimeConfiguration().getResolvedConfiguration().getFiles(reallyThirdParty);
-        Set<File> compileOnlyConfiguration = getProject().getConfigurations()
-            .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)
-            .getResolvedConfiguration()
-            .getFiles(reallyThirdParty);
+
+        Set<File> jars = GradleUtils.getFiles(getProject(), getRuntimeConfiguration(), reallyThirdParty).getFiles();
+        Set<File> compileOnlyConfiguration = GradleUtils.getFiles(
+            getProject(),
+            getProject().getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME),
+            reallyThirdParty
+        ).getFiles();
         // don't scan provided dependencies that we already scanned, e.x. don't scan cores dependencies for every plugin
         if (compileOnlyConfiguration != null) {
             jars.removeAll(compileOnlyConfiguration);

buildSrc/src/main/java/org/opensearch/gradle/precommit/UpdateShasTask.java

@@ -66,7 +66,7 @@ public UpdateShasTask() {
     public void updateShas() throws NoSuchAlgorithmException, IOException {
         Set<File> shaFiles = parentTask.get().getShaFiles();
 
-        for (File dependency : parentTask.get().getDependencies()) {
+        for (File dependency : parentTask.get().getDependencies().get()) {
             String jarName = dependency.getName();
             File shaFile = parentTask.get().getShaFile(jarName);
 

buildSrc/src/main/java/org/opensearch/gradle/tar/SymbolicLinkPreservingTar.java

@@ -184,7 +184,7 @@ private void visitSymbolicLink(final FileCopyDetailsInternal details) {
                 visitedSymbolicLinks.add(details.getFile());
                 final TarArchiveEntry entry = new TarArchiveEntry(details.getRelativePath().getPathString(), TarConstants.LF_SYMLINK);
                 entry.setModTime(getModTime(details));
-                entry.setMode(UnixStat.LINK_FLAG | details.getMode());
+                entry.setMode(UnixStat.LINK_FLAG | details.getPermissions().toUnixNumeric());
                 try {
                     entry.setLinkName(Files.readSymbolicLink(details.getFile().toPath()).toString());
                     tar.putArchiveEntry(entry);
@@ -197,7 +197,7 @@ private void visitSymbolicLink(final FileCopyDetailsInternal details) {
             private void visitDirectory(final FileCopyDetailsInternal details) {
                 final TarArchiveEntry entry = new TarArchiveEntry(details.getRelativePath().getPathString() + "/");
                 entry.setModTime(getModTime(details));
-                entry.setMode(UnixStat.DIR_FLAG | details.getMode());
+                entry.setMode(UnixStat.DIR_FLAG | details.getPermissions().toUnixNumeric());
                 try {
                     tar.putArchiveEntry(entry);
                     tar.closeArchiveEntry();
@@ -209,7 +209,7 @@ private void visitDirectory(final FileCopyDetailsInternal details) {
             private void visitFile(final FileCopyDetailsInternal details) {
                 final TarArchiveEntry entry = new TarArchiveEntry(details.getRelativePath().getPathString());
                 entry.setModTime(getModTime(details));
-                entry.setMode(UnixStat.FILE_FLAG | details.getMode());
+                entry.setMode(UnixStat.FILE_FLAG | details.getPermissions().toUnixNumeric());
                 entry.setSize(details.getSize());
                 try {
                     tar.putArchiveEntry(entry);