Merge branch 'main' into OS3268-limit-nested-depth

Signed-off-by: Mikhail Khludnev <[email protected]>

.ci/bwcVersions

@@ -28,3 +28,4 @@ BWC_VERSION:
   - "2.11.1"
   - "2.11.2"
   - "2.12.0"
+  - "2.13.0"

.github/ISSUE_TEMPLATE/meta.yml

@@ -0,0 +1,28 @@
+name: ✨ Meta Issue
+description: An issue that collects other issues together to describe a larger project or activity.
+title: '[META] <title>'
+labels: ['Meta, untriaged']
+body:
+  - type: textarea
+    attributes:
+      label: Please describe the end goal of this project
+      description: A clear and concise description of this project/endeavor.  This should be understandable to someone with no context.
+      placeholder: Ex. Views is a way to project indices in OpenSearch, these views act as a focal point for describing the underlying data and how the data is accessed. It allows for restricting the scope and filtering the response consistently.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Supporting References
+      description: Please provide links (and descriptions!) to RFCs, design docs, etc
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Issues
+      description: Please create a list of issues that should be tracked by this meta issue, including a short description.  The purpose is to provide everyone on the project with an "at a glance" update of the state us the work being tracked.  If you use the format "- [ ]" it will put your list into a checklist.
+      placeholder: Ex. - [ ] https://github.com/opensearch-project/security/issues/3888 Add views to the cluster metadata schema
+    validations:
+      required: true
+
+
+

.github/workflows/maintainer-approval.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - id: find-maintainers
-        uses: actions/github-script@v7
+        uses: actions/github-script@v7.0.1
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           result-encoding: string

.github/workflows/triage.yml

@@ -9,7 +9,7 @@ jobs:
     if: github.repository == 'opensearch-project/OpenSearch'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/github-script@v7
+      - uses: actions/github-script@v7.0.1
         with:
           script: |
             const { issue, repository } = context.payload;

.github/workflows/version.yml

@@ -129,7 +129,7 @@ jobs:
 
       - name: Create tracking issue
         id: create-issue
-        uses: actions/github-script@v6.4.0
+        uses: actions/github-script@v7.0.1
         with:
           script: |
             const body = `

benchmarks/src/main/java/org/opensearch/benchmark/index/codec/fuzzy/FilterConstructionBenchmark.java

@@ -0,0 +1,67 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.benchmark.index.codec.fuzzy;
+
+import org.apache.lucene.util.BytesRef;
+import org.opensearch.common.UUIDs;
+import org.opensearch.index.codec.fuzzy.FuzzySet;
+import org.opensearch.index.codec.fuzzy.FuzzySetFactory;
+import org.opensearch.index.codec.fuzzy.FuzzySetParameters;
+import org.opensearch.index.mapper.IdFieldMapper;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+@Fork(3)
+@Warmup(iterations = 2)
+@Measurement(iterations = 5, time = 60, timeUnit = TimeUnit.SECONDS)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+public class FilterConstructionBenchmark {
+
+    private List<BytesRef> items;
+
+    @Param({ "1000000", "10000000", "50000000" })
+    private int numIds;
+
+    @Param({ "0.0511", "0.1023", "0.2047" })
+    private double fpp;
+
+    private FuzzySetFactory fuzzySetFactory;
+    private String fieldName;
+
+    @Setup
+    public void setupIds() {
+        this.fieldName = IdFieldMapper.NAME;
+        this.items = IntStream.range(0, numIds).mapToObj(i -> new BytesRef(UUIDs.base64UUID())).collect(Collectors.toList());
+        FuzzySetParameters parameters = new FuzzySetParameters(() -> fpp);
+        this.fuzzySetFactory = new FuzzySetFactory(Map.of(fieldName, parameters));
+    }
+
+    @Benchmark
+    public FuzzySet buildFilter() throws IOException {
+        return fuzzySetFactory.createFuzzySet(items.size(), fieldName, () -> items.iterator());
+    }
+}

benchmarks/src/main/java/org/opensearch/benchmark/index/codec/fuzzy/FilterLookupBenchmark.java

@@ -0,0 +1,80 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.benchmark.index.codec.fuzzy;
+
+import org.apache.lucene.util.BytesRef;
+import org.opensearch.common.UUIDs;
+import org.opensearch.index.codec.fuzzy.FuzzySet;
+import org.opensearch.index.codec.fuzzy.FuzzySetFactory;
+import org.opensearch.index.codec.fuzzy.FuzzySetParameters;
+import org.opensearch.index.mapper.IdFieldMapper;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+@Fork(3)
+@Warmup(iterations = 2)
+@Measurement(iterations = 5, time = 60, timeUnit = TimeUnit.SECONDS)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+public class FilterLookupBenchmark {
+
+    @Param({ "50000000", "1000000" })
+    private int numItems;
+
+    @Param({ "1000000" })
+    private int searchKeyCount;
+
+    @Param({ "0.0511", "0.1023", "0.2047" })
+    private double fpp;
+
+    private FuzzySet fuzzySet;
+    private List<BytesRef> items;
+    private Random random = new Random();
+
+    @Setup
+    public void setupFilter() throws IOException {
+        String fieldName = IdFieldMapper.NAME;
+        items = IntStream.range(0, numItems).mapToObj(i -> new BytesRef(UUIDs.base64UUID())).collect(Collectors.toList());
+        FuzzySetParameters parameters = new FuzzySetParameters(() -> fpp);
+        fuzzySet = new FuzzySetFactory(Map.of(fieldName, parameters)).createFuzzySet(numItems, fieldName, () -> items.iterator());
+    }
+
+    @Benchmark
+    public void contains_withExistingKeys(Blackhole blackhole) throws IOException {
+        for (int i = 0; i < searchKeyCount; i++) {
+            blackhole.consume(fuzzySet.contains(items.get(random.nextInt(items.size()))) == FuzzySet.Result.MAYBE);
+        }
+    }
+
+    @Benchmark
+    public void contains_withRandomKeys(Blackhole blackhole) throws IOException {
+        for (int i = 0; i < searchKeyCount; i++) {
+            blackhole.consume(fuzzySet.contains(new BytesRef(UUIDs.base64UUID())));
+        }
+    }
+}

build.gradle

@@ -55,7 +55,7 @@ plugins {
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
   id "com.diffplug.spotless" version "6.25.0" apply false
-  id "org.gradle.test-retry" version "1.5.4" apply false
+  id "org.gradle.test-retry" version "1.5.8" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'
 }

buildSrc/src/main/java/org/opensearch/gradle/RepositoriesSetupPlugin.java

@@ -94,7 +94,7 @@ public static void configureRepositories(Project project) {
             String revision = matcher.group(1);
             MavenArtifactRepository luceneRepo = repos.maven(repo -> {
                 repo.setName("lucene-snapshots");
-                repo.setUrl("https://artifacts.opensearch.org/snapshots/lucene/");
+                repo.setUrl("https://ci.opensearch.org/ci/dbc/snapshots/lucene/");
             });
             repos.exclusiveContent(exclusiveRepo -> {
                 exclusiveRepo.filter(

distribution/packages/build.gradle

@@ -63,7 +63,7 @@ import java.util.regex.Pattern
  */
 
 plugins {
-  id "com.netflix.nebula.ospackage-base" version "11.6.0"
+  id "com.netflix.nebula.ospackage-base" version "11.8.0"
 }
 
 void addProcessFilesTask(String type, boolean jdk) {

distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/ListPluginsCommand.java

@@ -78,15 +78,14 @@ private void printPlugin(Environment env, Terminal terminal, Path plugin, String
         PluginInfo info = PluginInfo.readFromProperties(env.pluginsDir().resolve(plugin));
         terminal.println(Terminal.Verbosity.SILENT, prefix + info.getName());
         terminal.println(Terminal.Verbosity.VERBOSE, info.toString(prefix));
-        if (info.getOpenSearchVersion().equals(Version.CURRENT) == false) {
+        if (!PluginsService.isPluginVersionCompatible(info, Version.CURRENT)) {
             terminal.errorPrintln(
                 "WARNING: plugin ["
                     + info.getName()
                     + "] was built for OpenSearch version "
-                    + info.getVersion()
-                    + " but version "
+                    + info.getOpenSearchVersionRangesString()
+                    + " and is not compatible with "
                     + Version.CURRENT
-                    + " is required"
             );
         }
     }

distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/InstallPluginCommandTests.java

@@ -70,8 +70,10 @@
 import org.opensearch.core.util.FileSystemUtils;
 import org.opensearch.env.Environment;
 import org.opensearch.env.TestEnvironment;
+import org.opensearch.semver.SemverRange;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.PosixPermissionsResetter;
+import org.opensearch.test.VersionUtils;
 import org.junit.After;
 import org.junit.Before;
 
@@ -284,6 +286,35 @@ static void writePlugin(String name, Path structure, String... additionalProps)
         writeJar(structure.resolve("plugin.jar"), className);
     }
 
+    static void writePlugin(String name, Path structure, SemverRange opensearchVersionRange, String... additionalProps) throws IOException {
+        String[] properties = Stream.concat(
+            Stream.of(
+                "description",
+                "fake desc",
+                "name",
+                name,
+                "version",
+                "1.0",
+                "dependencies",
+                "{opensearch:\"" + opensearchVersionRange + "\"}",
+                "java.version",
+                System.getProperty("java.specification.version"),
+                "classname",
+                "FakePlugin"
+            ),
+            Arrays.stream(additionalProps)
+        ).toArray(String[]::new);
+        PluginTestUtil.writePluginProperties(structure, properties);
+        String className = name.substring(0, 1).toUpperCase(Locale.ENGLISH) + name.substring(1) + "Plugin";
+        writeJar(structure.resolve("plugin.jar"), className);
+    }
+
+    static Path createPlugin(String name, Path structure, SemverRange opensearchVersionRange, String... additionalProps)
+        throws IOException {
+        writePlugin(name, structure, opensearchVersionRange, additionalProps);
+        return writeZip(structure, null);
+    }
+
     static void writePluginSecurityPolicy(Path pluginDir, String... permissions) throws IOException {
         StringBuilder securityPolicyContent = new StringBuilder("grant {\n  ");
         for (String permission : permissions) {
@@ -867,6 +898,32 @@ public void testInstallMisspelledOfficialPlugins() throws Exception {
         assertThat(e.getMessage(), containsString("Unknown plugin unknown_plugin"));
     }
 
+    public void testInstallPluginWithCompatibleDependencies() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        String pluginZip = createPlugin("fake", pluginDir, SemverRange.fromString("~" + Version.CURRENT.toString())).toUri()
+            .toURL()
+            .toString();
+        skipJarHellCommand.execute(terminal, Collections.singletonList(pluginZip), false, env.v2());
+        assertThat(terminal.getOutput(), containsString("100%"));
+    }
+
+    public void testInstallPluginWithIncompatibleDependencies() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        // Core version is behind plugin version by one w.r.t patch, hence incompatible
+        Version coreVersion = Version.CURRENT;
+        Version pluginVersion = VersionUtils.getVersion(coreVersion.major, coreVersion.minor, (byte) (coreVersion.revision + 1));
+        String pluginZip = createPlugin("fake", pluginDir, SemverRange.fromString("~" + pluginVersion.toString())).toUri()
+            .toURL()
+            .toString();
+        IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> skipJarHellCommand.execute(terminal, Collections.singletonList(pluginZip), false, env.v2())
+        );
+        assertThat(e.getMessage(), containsString("Plugin [fake] was built for OpenSearch version ~" + pluginVersion));
+    }
+
     public void testBatchFlag() throws Exception {
         MockTerminal terminal = new MockTerminal();
         installPlugin(terminal, true);

distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/ListPluginsCommandTests.java

@@ -278,12 +278,49 @@ public void testExistingIncompatiblePlugin() throws Exception {
         buildFakePlugin(env, "fake desc 2", "fake_plugin2", "org.fake2");
 
         MockTerminal terminal = listPlugins(home);
-        String message = "plugin [fake_plugin1] was built for OpenSearch version 1.0 but version " + Version.CURRENT + " is required";
+        String message = "plugin [fake_plugin1] was built for OpenSearch version 5.0.0 and is not compatible with " + Version.CURRENT;
         assertEquals("fake_plugin1\nfake_plugin2\n", terminal.getOutput());
         assertEquals("WARNING: " + message + "\n", terminal.getErrorOutput());
 
         String[] params = { "-s" };
         terminal = listPlugins(home, params);
         assertEquals("fake_plugin1\nfake_plugin2\n", terminal.getOutput());
     }
+
+    public void testPluginWithDependencies() throws Exception {
+        PluginTestUtil.writePluginProperties(
+            env.pluginsDir().resolve("fake_plugin1"),
+            "description",
+            "fake desc 1",
+            "name",
+            "fake_plugin1",
+            "version",
+            "1.0",
+            "dependencies",
+            "{opensearch:\"" + Version.CURRENT + "\"}",
+            "java.version",
+            System.getProperty("java.specification.version"),
+            "classname",
+            "org.fake1"
+        );
+        String[] params = { "-v" };
+        MockTerminal terminal = listPlugins(home, params);
+        assertEquals(
+            buildMultiline(
+                "Plugins directory: " + env.pluginsDir(),
+                "fake_plugin1",
+                "- Plugin information:",
+                "Name: fake_plugin1",
+                "Description: fake desc 1",
+                "Version: 1.0",
+                "OpenSearch Version: " + Version.CURRENT.toString(),
+                "Java Version: " + System.getProperty("java.specification.version"),
+                "Native Controller: false",
+                "Extended Plugins: []",
+                " * Classname: org.fake1",
+                "Folder name: null"
+            ),
+            terminal.getOutput()
+        );
+    }
 }