From 8b358525793dc27cf1113bfbfd49227467161d0e Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 14 Nov 2023 14:53:18 -0500 Subject: [PATCH 1/3] Add system param to _cat/indices to only list system indices from SystemIndexPlugin.getSystemIndexDescriptors Signed-off-by: Craig Perkins --- .../org/opensearch/action/ActionModule.java | 4 +- .../rest/action/cat/RestIndicesAction.java | 16 ++++++ .../RenamedTimeoutRequestParameterTests.java | 3 +- .../action/cat/RestIndicesActionTests.java | 52 ++++++++++++++++--- 4 files changed, 65 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index 46775466aa615..45087d72044a2 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -512,6 +512,7 @@ public class ActionModule extends AbstractModule { private final RequestValidators indicesAliasesRequestRequestValidators; private final ThreadPool threadPool; private final ExtensionsManager extensionsManager; + private final SystemIndices systemIndices; public ActionModule( Settings settings, @@ -536,6 +537,7 @@ public ActionModule( this.actionPlugins = actionPlugins; this.threadPool = threadPool; this.extensionsManager = extensionsManager; + this.systemIndices = systemIndices; actions = setupActions(actionPlugins); actionFilters = setupActionFilters(actionPlugins); dynamicActionRegistry = new DynamicActionRegistry(); @@ -922,7 +924,7 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestClusterManagerAction()); registerHandler.accept(new RestNodesAction()); registerHandler.accept(new RestTasksAction(nodesInCluster)); - registerHandler.accept(new RestIndicesAction()); + registerHandler.accept(new RestIndicesAction(systemIndices)); registerHandler.accept(new RestSegmentsAction()); // Fully qualified to prevent interference with rest.action.count.RestCountAction registerHandler.accept(new org.opensearch.rest.action.cat.RestCountAction()); diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index 23cc1cb507072..217e07bc64b73 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -59,6 +59,7 @@ import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.Strings; import org.opensearch.index.IndexSettings; +import org.opensearch.indices.SystemIndices; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; import org.opensearch.rest.action.RestResponseListener; @@ -97,6 +98,12 @@ public class RestIndicesAction extends AbstractCatAction { private static final String DUPLICATE_PARAMETER_ERROR_MESSAGE = "Please only use one of the request parameters [master_timeout, cluster_manager_timeout]."; + private final SystemIndices systemIndices; + + public RestIndicesAction(SystemIndices systemIndices) { + this.systemIndices = systemIndices; + } + @Override public List routes() { return unmodifiableList(asList(new Route(GET, "/_cat/indices"), new Route(GET, "/_cat/indices/{index}"))); @@ -123,6 +130,7 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); final IndicesOptions indicesOptions = IndicesOptions.fromRequest(request, IndicesOptions.strictExpand()); final boolean local = request.paramAsBoolean("local", false); + final boolean system = request.paramAsBoolean("system", false); TimeValue clusterManagerTimeout = request.paramAsTime("cluster_manager_timeout", DEFAULT_CLUSTER_MANAGER_NODE_TIMEOUT); // Remove the if condition and statements inside after removing MASTER_ROLE. if (request.hasParam("master_timeout")) { @@ -716,6 +724,7 @@ Table buildTable( ) { final String healthParam = request.param("health"); + final boolean systemParam = request.paramAsBoolean("system", false); final Table table = getTableWithHeader(request); indicesSettings.forEach((indexName, settings) -> { @@ -755,6 +764,13 @@ Table buildTable( } } + if (systemParam) { + if (!systemIndices.isSystemIndex(indexName)) { + // System filter is enabled but index is not a system index + return; + } + } + final CommonStats primaryStats; final CommonStats totalStats; diff --git a/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java b/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java index c7e1039686cc9..43843aef0d483 100644 --- a/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java +++ b/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java @@ -17,6 +17,7 @@ import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.indices.SystemIndices; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.action.admin.cluster.RestCleanupRepositoryAction; import org.opensearch.rest.action.admin.cluster.RestCloneSnapshotAction; @@ -155,7 +156,7 @@ public void testCatAllocation() { } public void testCatIndices() { - RestIndicesAction action = new RestIndicesAction(); + RestIndicesAction action = new RestIndicesAction(new SystemIndices(Collections.emptyMap())); Exception e = assertThrows(OpenSearchParseException.class, () -> action.doCatRequest(getRestRequestWithBothParams(), client)); assertThat(e.getMessage(), containsString(DUPLICATE_PARAMETER_ERROR_MESSAGE)); assertWarnings(MASTER_TIMEOUT_DEPRECATED_MESSAGE); diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java index 96b1c75371697..adb4073c4fff4 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java @@ -46,10 +46,14 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.index.IndexSettings; +import org.opensearch.indices.SystemIndexDescriptor; +import org.opensearch.indices.SystemIndices; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestRequest; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; @@ -70,9 +74,15 @@ public void testBuildTable() { final Map indicesHealths = new LinkedHashMap<>(); final Map indicesStats = new LinkedHashMap<>(); + String systemIndexName = ".system-index"; + List indexNames = new ArrayList<>(); + indexNames.add(systemIndexName); for (int i = 0; i < numIndices; i++) { String indexName = "index-" + i; + indexNames.add(indexName); + } + for (String indexName : indexNames) { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) @@ -137,17 +147,13 @@ public void testBuildTable() { } } - final RestIndicesAction action = new RestIndicesAction(); + final RestIndicesAction action = new RestIndicesAction( + new SystemIndices(Map.of("testplugin", List.of(new SystemIndexDescriptor(systemIndexName, "Example of a system index")))) + ); final Table table = action.buildTable(new FakeRestRequest(), indicesSettings, indicesHealths, indicesStats, indicesMetadatas); // now, verify the table is correct - List headers = table.getHeaders(); - assertThat(headers.get(0).value, equalTo("health")); - assertThat(headers.get(1).value, equalTo("status")); - assertThat(headers.get(2).value, equalTo("index")); - assertThat(headers.get(3).value, equalTo("uuid")); - assertThat(headers.get(4).value, equalTo("pri")); - assertThat(headers.get(5).value, equalTo("rep")); + verifyTableHeaders(table); final List> rows = table.getRows(); assertThat(rows.size(), equalTo(indicesMetadatas.size())); @@ -178,5 +184,35 @@ public void testBuildTable() { assertThat(row.get(5).value, nullValue()); } } + + final Table tableWithSystemIndexOnly = action.buildTable( + new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).withParams(Map.of("system", "true")).build(), + indicesSettings, + indicesHealths, + indicesStats, + indicesMetadatas + ); + + // now, verify the table is correct + verifyTableHeaders(tableWithSystemIndexOnly); + + final List> systemIndexRows = tableWithSystemIndexOnly.getRows(); + assertThat(systemIndexRows.size(), equalTo(1)); + + for (final List row : systemIndexRows) { + final String indexName = (String) row.get(2).value; + + assertThat(indexName, equalTo(".system-index")); + } + } + + private void verifyTableHeaders(Table table) { + List headers = table.getHeaders(); + assertThat(headers.get(0).value, equalTo("health")); + assertThat(headers.get(1).value, equalTo("status")); + assertThat(headers.get(2).value, equalTo("index")); + assertThat(headers.get(3).value, equalTo("uuid")); + assertThat(headers.get(4).value, equalTo("pri")); + assertThat(headers.get(5).value, equalTo("rep")); } } From 4af2dd558f3195b7a153a77efd2e5247733ff7f1 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 14 Nov 2023 14:59:57 -0500 Subject: [PATCH 2/3] Add entry to CHANGELOG Signed-off-by: Craig Perkins --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index daf5568fa7a77..a38e4a112354e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add cluster state stats ([#10670](https://github.com/opensearch-project/OpenSearch/pull/10670)) - Adding slf4j license header to LoggerMessageFormat.java ([#11069](https://github.com/opensearch-project/OpenSearch/pull/11069)) - [Streaming Indexing] Introduce new experimental server HTTP transport based on Netty 4 and Project Reactor (Reactor Netty) ([#9672](https://github.com/opensearch-project/OpenSearch/pull/9672)) +- Add system param to /_cat/indices to only list system indices from SystemIndexPlugin.getSystemIndexDescriptors ([11201](https://github.com/opensearch-project/OpenSearch/pull/11201)) ### Dependencies - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298)) From 7ad8bc7d54eed5e01174ee857c297be79e6fdcff Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 15 Nov 2023 12:03:10 -0500 Subject: [PATCH 3/3] Add column for description as well Signed-off-by: Craig Perkins --- .../opensearch/rest/action/cat/RestIndicesAction.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index 217e07bc64b73..57f1957b78b0e 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -59,6 +59,7 @@ import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.Strings; import org.opensearch.index.IndexSettings; +import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.indices.SystemIndices; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; @@ -366,6 +367,11 @@ protected Table getTableWithHeader(final RestRequest request) { table.addCell("store.size", "sibling:pri;alias:ss,storeSize;text-align:right;desc:store size of primaries & replicas"); table.addCell("pri.store.size", "text-align:right;desc:store size of primaries"); + final boolean systemParam = request.paramAsBoolean("system", false); + if (systemParam) { + table.addCell("desc", "alias:dsc;desc:description of system index"); + } + table.addCell("completion.size", "sibling:pri;alias:cs,completionSize;default:false;text-align:right;desc:size of completion"); table.addCell("pri.completion.size", "default:false;text-align:right;desc:size of completion"); @@ -800,6 +806,11 @@ Table buildTable( table.addCell(totalStats.getStore() == null ? null : totalStats.getStore().size()); table.addCell(primaryStats.getStore() == null ? null : primaryStats.getStore().size()); + if (systemParam) { + SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(indexName); + table.addCell(descriptor == null ? null : descriptor.getDescription()); + } + table.addCell(totalStats.getCompletion() == null ? null : totalStats.getCompletion().getSize()); table.addCell(primaryStats.getCompletion() == null ? null : primaryStats.getCompletion().getSize());