Merge branch 'main' into grok

Signed-off-by: Sandesh Kumar <[email protected]>

CHANGELOG.md

@@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.apache.commons:commons-configuration2` from 2.10.1 to 2.11.0 ([#14399](https://github.com/opensearch-project/OpenSearch/pull/14399))
 - Bump `com.gradle.develocity` from 3.17.4 to 3.17.5 ([#14397](https://github.com/opensearch-project/OpenSearch/pull/14397))
 - Bump `opentelemetry` from 1.36.0 to 1.39.0 ([#14457](https://github.com/opensearch-project/OpenSearch/pull/14457))
+- Bump `azure-identity` from 1.11.4 to 1.13.0, Bump `msal4j` from 1.14.3 to 1.15.1, Bump `msal4j-persistence-extension` from 1.2.0 to 1.3.0 ([#14506](https://github.com/opensearch-project/OpenSearch/pull/14506))
 
 ### Changed
 - unsignedLongRangeQuery now returns MatchNoDocsQuery if the lower bounds are greater than the upper bounds ([#14416](https://github.com/opensearch-project/OpenSearch/pull/14416))
@@ -33,10 +34,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Removed
 
 ### Fixed
+- Fix bug in SBP cancellation logic ([#13259](https://github.com/opensearch-project/OpenSearch/pull/13474))
 - Fix handling of Short and Byte data types in ScriptProcessor ingest pipeline ([#14379](https://github.com/opensearch-project/OpenSearch/issues/14379))
 - Switch to iterative version of WKT format parser ([#14086](https://github.com/opensearch-project/OpenSearch/pull/14086))
 - Fix the computed max shards of cluster to avoid int overflow ([#14155](https://github.com/opensearch-project/OpenSearch/pull/14155))
 - Fixed rest-high-level client searchTemplate & mtermVectors endpoints to have a leading slash ([#14465](https://github.com/opensearch-project/OpenSearch/pull/14465))
+- Write shard level metadata blob when snapshotting searchable snapshot indexes ([#13190](https://github.com/opensearch-project/OpenSearch/pull/13190))
+- Fix aggs result of NestedAggregator with sub NestedAggregator ([#13324](https://github.com/opensearch-project/OpenSearch/pull/13324))
+- Add ListPitInfo::getKeepAlive() getter ([#14495](https://github.com/opensearch-project/OpenSearch/pull/14495))
 - Refactoring Grok.validatePatternBank by using an iterative approach ([#14206](https://github.com/opensearch-project/OpenSearch/pull/14206))
 
 ### Security

TESTING.md

@@ -91,21 +91,23 @@ This will instruct all JVMs (including any that run cli tools such as creating t
 
 ## Test case filtering
 
--   `tests.class` is a class-filtering shell-like glob pattern
--   `tests.method` is a method-filtering glob pattern.
+To be able to run a single test you need to specify the module where you're running the tests from.
+
+Example: `./gradlew server:test --tests "*.ReplicaShardBatchAllocatorTests.testNoAsyncFetchData"`
 
 Run a single test case (variants)
 
-    ./gradlew test -Dtests.class=org.opensearch.package.ClassName
-    ./gradlew test "-Dtests.class=*.ClassName"
+    ./gradlew module:test --tests org.opensearch.package.ClassName
+    ./gradlew module:test --tests org.opensearch.package.ClassName.testName
+    ./gradlew module:test --tests "*.ClassName"
 
 Run all tests in a package and its sub-packages
 
-    ./gradlew test "-Dtests.class=org.opensearch.package.*"
+    ./gradlew module:test --tests "org.opensearch.package.*"
 
 Run any test methods that contain *esi* (e.g.: .r*esi*ze.)
 
-    ./gradlew test "-Dtests.method=*esi*"
+    ./gradlew module:test --tests "*esi*"
 
 Run all tests that are waiting for a bugfix (disabled by default)
 

plugins/repository-azure/build.gradle

@@ -57,11 +57,11 @@ dependencies {
   api "io.netty:netty-transport-native-unix-common:${versions.netty}"
   implementation project(':modules:transport-netty4')
   api 'com.azure:azure-storage-blob:12.23.0'
-  api 'com.azure:azure-identity:1.11.4'
+  api 'com.azure:azure-identity:1.13.0'
   // Start of transitive dependencies for azure-identity
-  api 'com.microsoft.azure:msal4j-persistence-extension:1.2.0'
+  api 'com.microsoft.azure:msal4j-persistence-extension:1.3.0'
   api "net.java.dev.jna:jna-platform:${versions.jna}"
-  api 'com.microsoft.azure:msal4j:1.14.3'
+  api 'com.microsoft.azure:msal4j:1.15.1'
   api 'com.nimbusds:oauth2-oidc-sdk:11.9.1'
   api 'com.nimbusds:nimbus-jose-jwt:9.40'
   api 'com.nimbusds:content-type:2.3'

plugins/repository-azure/licenses/azure-identity-1.13.0.jar.sha1

@@ -0,0 +1 @@
+54b44a74636322d06e9dc42d611a9f12a0966790

plugins/repository-azure/licenses/msal4j-1.15.1.jar.sha1

@@ -0,0 +1 @@
+cd1daa94b81bd97153536b661c31295f99cbb8e7

plugins/repository-azure/licenses/msal4j-persistence-extension-1.3.0.jar.sha1

@@ -0,0 +1 @@
+8a8ef1517d27a5b4de1512ef94679bdb59f210b6

server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java

@@ -1405,7 +1405,7 @@ public void testPitCreatedOnReplica() throws Exception {
             .setPointInTime(new PointInTimeBuilder(pitResponse.getId()).setKeepAlive(TimeValue.timeValueDays(1)))
             .setRequestCache(false)
             .get();
-        PitTestsUtil.assertUsingGetAllPits(client(replica), pitResponse.getId(), pitResponse.getCreationTime());
+        PitTestsUtil.assertUsingGetAllPits(client(replica), pitResponse.getId(), pitResponse.getCreationTime(), TimeValue.timeValueDays(1));
         assertSegments(false, INDEX_NAME, 1, client(replica), pitResponse.getId());
 
         List<String> currentFiles = List.of(replicaShard.store().directory().listAll());

server/src/internalClusterTest/java/org/opensearch/search/pit/PitMultiNodeIT.java

@@ -104,7 +104,7 @@ public void testPit() throws Exception {
         assertEquals(2, searchResponse.getSuccessfulShards());
         assertEquals(2, searchResponse.getTotalShards());
         validatePitStats("index", 2, 2);
-        PitTestsUtil.assertUsingGetAllPits(client(), pitResponse.getId(), pitResponse.getCreationTime());
+        PitTestsUtil.assertUsingGetAllPits(client(), pitResponse.getId(), pitResponse.getCreationTime(), TimeValue.timeValueDays(1));
         assertSegments(false, client(), pitResponse.getId());
     }
 
@@ -131,7 +131,12 @@ public void testCreatePitWhileNodeDropWithAllowPartialCreationTrue() throws Exce
             public Settings onNodeStopped(String nodeName) throws Exception {
                 ActionFuture<CreatePitResponse> execute = client().execute(CreatePitAction.INSTANCE, request);
                 CreatePitResponse pitResponse = execute.get();
-                PitTestsUtil.assertUsingGetAllPits(client(), pitResponse.getId(), pitResponse.getCreationTime());
+                PitTestsUtil.assertUsingGetAllPits(
+                    client(),
+                    pitResponse.getId(),
+                    pitResponse.getCreationTime(),
+                    TimeValue.timeValueDays(1)
+                );
                 assertSegments(false, "index", 1, client(), pitResponse.getId());
                 assertEquals(1, pitResponse.getSuccessfulShards());
                 assertEquals(2, pitResponse.getTotalShards());
@@ -164,7 +169,12 @@ public Settings onNodeStopped(String nodeName) throws Exception {
                 assertEquals(0, searchResponse.getSkippedShards());
                 assertEquals(2, searchResponse.getTotalShards());
                 validatePitStats("index", 1, 1);
-                PitTestsUtil.assertUsingGetAllPits(client(), pitResponse.getId(), pitResponse.getCreationTime());
+                PitTestsUtil.assertUsingGetAllPits(
+                    client(),
+                    pitResponse.getId(),
+                    pitResponse.getCreationTime(),
+                    TimeValue.timeValueDays(1)
+                );
                 return super.onNodeStopped(nodeName);
             }
         });

server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java

@@ -17,6 +17,7 @@
 import org.opensearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
 import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
 import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
+import org.opensearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
 import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest;
 import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder;
@@ -53,11 +54,13 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 
@@ -132,21 +135,24 @@ public void testCreateSearchableSnapshot() throws Exception {
 
     public void testSnapshottingSearchableSnapshots() throws Exception {
         final String repoName = "test-repo";
+        final String initSnapName = "initial-snapshot";
         final String indexName = "test-idx";
+        final String repeatSnapNamePrefix = "test-repeated-snap-";
+        final String repeatIndexNamePrefix = indexName + "-copy-";
         final Client client = client();
 
         // create an index, add data, snapshot it, then delete it
         internalCluster().ensureAtLeastNumDataNodes(1);
         createIndexWithDocsAndEnsureGreen(0, 100, indexName);
         createRepositoryWithSettings(null, repoName);
-        takeSnapshot(client, "initial-snapshot", repoName, indexName);
+        takeSnapshot(client, initSnapName, repoName, indexName);
         deleteIndicesAndEnsureGreen(client, indexName);
 
         // restore the index as a searchable snapshot
         internalCluster().ensureAtLeastNumSearchNodes(1);
         client.admin()
             .cluster()
-            .prepareRestoreSnapshot(repoName, "initial-snapshot")
+            .prepareRestoreSnapshot(repoName, initSnapName)
             .setRenamePattern("(.+)")
             .setRenameReplacement("$1-copy-0")
             .setStorageType(RestoreSnapshotRequest.StorageType.REMOTE_SNAPSHOT)
@@ -159,7 +165,7 @@ public void testSnapshottingSearchableSnapshots() throws Exception {
 
         // Test that the searchable snapshot index can continue to be snapshotted and restored
         for (int i = 0; i < 4; i++) {
-            final String repeatedSnapshotName = "test-repeated-snap-" + i;
+            final String repeatedSnapshotName = repeatSnapNamePrefix + i;
             takeSnapshot(client, repeatedSnapshotName, repoName);
             deleteIndicesAndEnsureGreen(client, "_all");
             client.admin()
@@ -181,21 +187,34 @@ public void testSnapshottingSearchableSnapshots() throws Exception {
         final Map<String, List<String>> snapshotInfoMap = response.getSnapshots()
             .stream()
             .collect(Collectors.toMap(s -> s.snapshotId().getName(), SnapshotInfo::indices));
-        assertEquals(
-            Map.of(
-                "initial-snapshot",
-                List.of("test-idx"),
-                "test-repeated-snap-0",
-                List.of("test-idx-copy-0"),
-                "test-repeated-snap-1",
-                List.of("test-idx-copy-1"),
-                "test-repeated-snap-2",
-                List.of("test-idx-copy-2"),
-                "test-repeated-snap-3",
-                List.of("test-idx-copy-3")
-            ),
-            snapshotInfoMap
-        );
+        final Map<String, List<String>> expect = new HashMap<>();
+        expect.put(initSnapName, List.of(indexName));
+        IntStream.range(0, 4).forEach(i -> expect.put(repeatSnapNamePrefix + i, List.of(repeatIndexNamePrefix + i)));
+        assertEquals(expect, snapshotInfoMap);
+
+        String[] snapNames = new String[5];
+        IntStream.range(0, 4).forEach(i -> snapNames[i] = repeatSnapNamePrefix + i);
+        snapNames[4] = initSnapName;
+        SnapshotsStatusResponse snapshotsStatusResponse = client.admin()
+            .cluster()
+            .prepareSnapshotStatus(repoName)
+            .addSnapshots(snapNames)
+            .execute()
+            .actionGet();
+        snapshotsStatusResponse.getSnapshots().forEach(s -> {
+            String snapName = s.getSnapshot().getSnapshotId().getName();
+            assertEquals(1, s.getIndices().size());
+            assertEquals(1, s.getShards().size());
+            if (snapName.equals("initial-snapshot")) {
+                assertNotNull(s.getIndices().get("test-idx"));
+                assertTrue(s.getShards().get(0).getStats().getTotalFileCount() > 0);
+            } else {
+                assertTrue(snapName.startsWith(repeatSnapNamePrefix));
+                assertEquals(1, s.getIndices().size());
+                assertNotNull(s.getIndices().get(repeatIndexNamePrefix + snapName.substring(repeatSnapNamePrefix.length())));
+                assertEquals(0L, s.getShards().get(0).getStats().getTotalFileCount());
+            }
+        });
     }
 
     /**

server/src/main/java/org/opensearch/action/search/ListPitInfo.java

@@ -53,6 +53,10 @@ public long getCreationTime() {
         return creationTime;
     }
 
+    public long getKeepAlive() {
+        return keepAlive;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(pitId);

server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java

@@ -2801,9 +2801,12 @@ public void snapshotShard(
             long indexIncrementalSize = 0;
             long indexTotalFileSize = 0;
             final BlockingQueue<BlobStoreIndexShardSnapshot.FileInfo> filesToSnapshot = new LinkedBlockingQueue<>();
-            // If we did not find a set of files that is equal to the current commit we determine the files to upload by comparing files
-            // in the commit with files already in the repository
-            if (filesFromSegmentInfos == null) {
+            if (store.indexSettings().isRemoteSnapshot()) {
+                // If the source of the data is another remote snapshot (i.e. searchable snapshot) then no need to snapshot the shard
+                indexCommitPointFiles = List.of();
+            } else if (filesFromSegmentInfos == null) {
+                // If we did not find a set of files that is equal to the current commit we determine the files to upload by comparing files
+                // in the commit with files already in the repository
                 indexCommitPointFiles = new ArrayList<>();
                 final Collection<String> fileNames;
                 final Store.MetadataSnapshot metadataFromStore;

server/src/main/java/org/opensearch/search/ResourceType.java

@@ -0,0 +1,41 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search;
+
+/**
+ * Enum to hold the resource type
+ */
+public enum ResourceType {
+    CPU("cpu"),
+    JVM("jvm");
+
+    private final String name;
+
+    ResourceType(String name) {
+        this.name = name;
+    }
+
+    /**
+     * The string match here is case-sensitive
+     * @param s name matching the resource type name
+     * @return a {@link ResourceType}
+     */
+    public static ResourceType fromName(String s) {
+        for (ResourceType resourceType : values()) {
+            if (resourceType.getName().equals(s)) {
+                return resourceType;
+            }
+        }
+        throw new IllegalArgumentException("Unknown resource type: [" + s + "]");
+    }
+
+    private String getName() {
+        return name;
+    }
+}