diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a1915a3767f5..d89f509edbda9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,7 +85,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix multi-search with template doesn't return status code ([#16265](https://github.com/opensearch-project/OpenSearch/pull/16265)) - [Streaming Indexing] Fix intermittent 'The bulk request must be terminated by a newline [\n]' failures [#16337](https://github.com/opensearch-project/OpenSearch/pull/16337)) - Fix wrong default value when setting `index.number_of_routing_shards` to null on index creation ([#16331](https://github.com/opensearch-project/OpenSearch/pull/16331)) -- Fix disk usage exceeds threshold cluster can't spin up issue ([#15258](https://github.com/opensearch-project/OpenSearch/pull/15258))) +- [Workload Management] Make query groups persistent across process restarts [#16370](https://github.com/opensearch-project/OpenSearch/pull/16370) - Fix inefficient Stream API call chains ending with count() ([#15386](https://github.com/opensearch-project/OpenSearch/pull/15386)) diff --git a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java index 7aa63a2736a8c..e9219de218aef 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java @@ -31,7 +31,6 @@ package org.opensearch.bootstrap; -import org.opensearch.common.logging.LogConfigurator; import org.opensearch.common.settings.KeyStoreCommandTestCase; import org.opensearch.common.settings.KeyStoreWrapper; import org.opensearch.common.settings.SecureSettings; @@ -39,7 +38,6 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.settings.SecureString; import org.opensearch.env.Environment; -import org.opensearch.node.Node; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; import org.junit.Before; @@ -53,14 +51,8 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; public class BootstrapTests extends OpenSearchTestCase { Environment env; @@ -139,38 +131,4 @@ private void assertPassphraseRead(String source, String expected) { } } - public void testInitExecutionOrder() throws Exception { - AtomicInteger order = new AtomicInteger(0); - CountDownLatch countDownLatch = new CountDownLatch(1); - Thread mockThread = new Thread(() -> { - assertEquals(0, order.getAndIncrement()); - countDownLatch.countDown(); - }); - - Node mockNode = mock(Node.class); - doAnswer(invocation -> { - try { - boolean threadStarted = countDownLatch.await(1000, TimeUnit.MILLISECONDS); - assertTrue( - "Waited for one second but the keepAliveThread isn't started, please check the execution order of" - + "keepAliveThread.start and node.start", - threadStarted - ); - } catch (InterruptedException e) { - fail("Thread interrupted"); - } - assertEquals(1, order.getAndIncrement()); - return null; - }).when(mockNode).start(); - - LogConfigurator.registerErrorListener(); - Bootstrap testBootstrap = new Bootstrap(mockThread, mockNode); - Bootstrap.setInstance(testBootstrap); - - Bootstrap.startInstance(testBootstrap); - - verify(mockNode).start(); - assertEquals(2, order.get()); - } - } diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/Writeable.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/Writeable.java index 960f4bec5eeb5..a46f2bebe3ad2 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/Writeable.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/Writeable.java @@ -83,7 +83,7 @@ public static > void registerReader(final byte ordinal, fina public static void registerClassAlias(final Class classInstance, final Class classGeneric) { if (WRITER_CUSTOM_CLASS_MAP.putIfAbsent(classInstance, classGeneric) != null) { - throw new IllegalArgumentException("Streamable custom class already registered [" + classInstance.getClass() + "]"); + throw new IllegalArgumentException("Streamable custom class already registered [" + classInstance.getName() + "]"); } } @@ -96,7 +96,7 @@ public static > W getWriter(final Class clazz) { } /** - * Returns the ristered reader keyed by the unique ordinal + * Returns the registered reader keyed by the unique ordinal */ @SuppressWarnings("unchecked") public static > R getReader(final byte b) { diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java index bbb55204712d1..82cc3951be753 100644 --- a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java +++ b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java @@ -57,6 +57,7 @@ public final class MediaTypeRegistry { private static Map formatToMediaType = Map.of(); private static Map typeWithSubtypeToMediaType = Map.of(); + private static Map knownStringsToMediaType = Map.of(); // Default mediaType singleton private static MediaType DEFAULT_MEDIA_TYPE; @@ -84,6 +85,8 @@ private static void register(MediaType[] acceptedMediaTypes, Map typeMap = new HashMap<>(typeWithSubtypeToMediaType); Map formatMap = new HashMap<>(formatToMediaType); + Map knownStringMap = new HashMap<>(knownStringsToMediaType); + for (MediaType mediaType : acceptedMediaTypes) { if (formatMap.containsKey(mediaType.format())) { throw new IllegalArgumentException("unable to register mediaType: [" + mediaType.format() + "]. Type already exists."); @@ -107,13 +110,24 @@ private static void register(MediaType[] acceptedMediaTypes, Map getNamedXWriteables() { DecommissionAttributeMetadata::fromXContent ) ); + entries.add( + new NamedXContentRegistry.Entry( + Metadata.Custom.class, + new ParseField(QueryGroupMetadata.TYPE), + QueryGroupMetadata::fromXContent + ) + ); return entries; } diff --git a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java index 7da52147661dc..9aec81536dbd0 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java @@ -81,6 +81,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -729,7 +730,7 @@ public static void validateRepositoryMetadataSettings( + " in the name as this delimiter is used to create pinning entity" ); } - if (repositoryWithShallowV2Exists(repositories)) { + if (repositoryWithShallowV2Exists(repositories, repositoryName)) { throw new RepositoryException( repositoryName, "setting " @@ -763,8 +764,13 @@ public static void validateRepositoryMetadataSettings( } } - private static boolean repositoryWithShallowV2Exists(Map repositories) { - return repositories.values().stream().anyMatch(repo -> SHALLOW_SNAPSHOT_V2.get(repo.getMetadata().settings())); + private static boolean repositoryWithShallowV2Exists(Map repositories, String repositoryName) { + return repositories.values() + .stream() + .anyMatch( + repository -> SHALLOW_SNAPSHOT_V2.get(repository.getMetadata().settings()) + && !Objects.equals(repository.getMetadata().name(), repositoryName) + ); } private static boolean pinnedTimestampExistsWithDifferentRepository( diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java index 40afdbfbdaa9e..c2f8b17fcf166 100644 --- a/server/src/main/java/org/opensearch/search/SearchService.java +++ b/server/src/main/java/org/opensearch/search/SearchService.java @@ -721,14 +721,15 @@ private SearchPhaseResult executeQueryPhase(ShardSearchRequest request, SearchSh } } catch (Exception e) { // execution exception can happen while loading the cache, strip it - if (e instanceof ExecutionException) { - e = (e.getCause() == null || e.getCause() instanceof Exception) - ? (Exception) e.getCause() - : new OpenSearchException(e.getCause()); + Exception exception = e; + if (exception instanceof ExecutionException) { + exception = (exception.getCause() == null || exception.getCause() instanceof Exception) + ? (Exception) exception.getCause() + : new OpenSearchException(exception.getCause()); } - logger.trace("Query phase failed", e); - processFailure(readerContext, e); - throw e; + logger.trace("Query phase failed", exception); + processFailure(readerContext, exception); + throw exception; } finally { taskResourceTrackingService.writeTaskResourceUsage(task, clusterService.localNode().getId()); } diff --git a/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java b/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java index f8240e775cfa5..8d2f2b743af9f 100644 --- a/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java +++ b/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java @@ -71,6 +71,7 @@ import org.opensearch.common.settings.SettingsModule; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.GatewayAllocator; import org.opensearch.plugins.ClusterPlugin; import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry; @@ -331,10 +332,18 @@ public void testRejectsDuplicateExistingShardsAllocatorName() { public void testQueryGroupMetadataRegister() { List customEntries = ClusterModule.getNamedWriteables(); + List customXEntries = ClusterModule.getNamedXWriteables(); assertTrue( customEntries.stream() .anyMatch(entry -> entry.categoryClass == Metadata.Custom.class && entry.name.equals(QueryGroupMetadata.TYPE)) ); + + assertTrue( + customXEntries.stream() + .anyMatch( + entry -> entry.categoryClass == Metadata.Custom.class && entry.name.getPreferredName().equals(QueryGroupMetadata.TYPE) + ) + ); } public void testRerouteServiceSetForBalancedShardsAllocator() { diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java index e280ab8c7a73c..f2b06b0926b81 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java @@ -431,6 +431,13 @@ public void testRepositoryCreationShallowV2() throws Exception { ); } + // Modify repo-1 settings. This should go through + updateRepository( + client, + "test-repo-1", + Settings.builder().put(snapshotRepoSettings1).put("max_snapshot_bytes_per_sec", "10k").build() + ); + // Disable shallow snapshot V2 setting on test-repo-1 updateRepository( client,