From 04682dc1cd646b368b8f31470dcf71db11eae75d Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Tue, 29 Oct 2024 12:19:53 +0100 Subject: [PATCH 01/18] Try to simplify geometries that fail with TopologyException (#115834) This geometries are valid and they can actually be simplified so lets make the clipping algorithm a best effort and return the original geometry in those cases so the simplification can handle it. --- docs/changelog/115834.yaml | 5 +++++ .../xpack/vectortile/feature/FeatureFactory.java | 7 +++++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/115834.yaml diff --git a/docs/changelog/115834.yaml b/docs/changelog/115834.yaml new file mode 100644 index 0000000000000..91f9e9a4e2e41 --- /dev/null +++ b/docs/changelog/115834.yaml @@ -0,0 +1,5 @@ +pr: 115834 +summary: Try to simplify geometries that fail with `TopologyException` +area: Geo +type: bug +issues: [] diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java index 0c4ff1780ae1e..b5f9088edc4be 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java @@ -307,8 +307,11 @@ private static org.locationtech.jts.geom.Geometry clipGeometry( return null; } } catch (TopologyException ex) { - // we should never get here but just to be super safe because a TopologyException will kill the node - throw new IllegalArgumentException(ex); + // Note we should never throw a TopologyException as it kill the node + // unfortunately the intersection method is not perfect and it will throw this error for complex + // geometries even when valid. We can still simplify such geometry so we just return the original and + // let the simplification process handle it. + return geometry; } } } From b868677c5b156233257612ab2121f4a01ca69aed Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 29 Oct 2024 12:26:27 +0100 Subject: [PATCH 02/18] Fix file settings service tests (#115770) This PR addresses some of the failure causes tracked under https://github.com/elastic/elasticsearch/issues/115280 and https://github.com/elastic/elasticsearch/issues/115725: the latch-await setup was rather convoluted and the move command not always correctly invoked in the right order. This PR cleans up latching by separating awaiting the first processing call (on start) from waiting on the subsequent call. Also, it makes writing the file more robust w.r.t. OS'es where `atomic_move` may not be available. This should address failures around the timeout await, and the assertion failures around invoked methods tracked here: https://github.com/elastic/elasticsearch/issues/115725#issuecomment-2441989470 But will likely require another round of changes to address the failures to delete files. Relates: https://github.com/elastic/elasticsearch/issues/115280 Relates: https://github.com/elastic/elasticsearch/issues/115725 --- muted-tests.yml | 3 - .../service/FileSettingsServiceTests.java | 85 ++++++++++++------- 2 files changed, 52 insertions(+), 36 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 4315f1283a347..419e8fbb68566 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -236,9 +236,6 @@ tests: - class: org.elasticsearch.xpack.inference.DefaultEndPointsIT method: testInferDeploysDefaultE5 issue: https://github.com/elastic/elasticsearch/issues/115361 -- class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests - method: testProcessFileChanges - issue: https://github.com/elastic/elasticsearch/issues/115280 - class: org.elasticsearch.xpack.security.FileSettingsRoleMappingsRestartIT method: testFileSettingsReprocessedOnRestartWithoutVersionChange issue: https://github.com/elastic/elasticsearch/issues/115450 diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java index 8af36e2f9677e..f67d7ddcc7550 100644 --- a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java @@ -24,6 +24,8 @@ import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.BuildVersion; import org.elasticsearch.env.Environment; @@ -39,9 +41,10 @@ import org.mockito.stubbing.Answer; import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.util.List; import java.util.Map; import java.util.Set; @@ -50,6 +53,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; +import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import static org.elasticsearch.node.Node.NODE_NAME_SETTING; import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.hasEntry; @@ -190,9 +195,7 @@ public void testInitialFileWorks() throws Exception { return null; }).when(controller).process(any(), any(XContentParser.class), any(), any()); - CountDownLatch latch = new CountDownLatch(1); - - fileSettingsService.addFileChangedListener(latch::countDown); + CountDownLatch fileProcessingLatch = new CountDownLatch(1); Files.createDirectories(fileSettingsService.watchedFileDir()); // contents of the JSON don't matter, we just need a file to exist @@ -202,15 +205,14 @@ public void testInitialFileWorks() throws Exception { try { return invocation.callRealMethod(); } finally { - latch.countDown(); + fileProcessingLatch.countDown(); } }).when(fileSettingsService).processFileOnServiceStart(); fileSettingsService.start(); fileSettingsService.clusterChanged(new ClusterChangedEvent("test", clusterService.state(), ClusterState.EMPTY_STATE)); - // wait for listener to be called - assertTrue(latch.await(20, TimeUnit.SECONDS)); + longAwait(fileProcessingLatch); verify(fileSettingsService, times(1)).processFileOnServiceStart(); verify(controller, times(1)).process(any(), any(XContentParser.class), eq(ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION), any()); @@ -223,40 +225,40 @@ public void testProcessFileChanges() throws Exception { return null; }).when(controller).process(any(), any(XContentParser.class), any(), any()); - // we get three events: initial clusterChanged event, first write, second write - CountDownLatch latch = new CountDownLatch(3); - - fileSettingsService.addFileChangedListener(latch::countDown); - - Files.createDirectories(fileSettingsService.watchedFileDir()); - // contents of the JSON don't matter, we just need a file to exist - writeTestFile(fileSettingsService.watchedFile(), "{}"); - + CountDownLatch changesOnStartLatch = new CountDownLatch(1); doAnswer((Answer) invocation -> { try { return invocation.callRealMethod(); } finally { - latch.countDown(); + changesOnStartLatch.countDown(); } }).when(fileSettingsService).processFileOnServiceStart(); + + CountDownLatch changesLatch = new CountDownLatch(1); doAnswer((Answer) invocation -> { try { return invocation.callRealMethod(); } finally { - latch.countDown(); + changesLatch.countDown(); } }).when(fileSettingsService).processFileChanges(); + Files.createDirectories(fileSettingsService.watchedFileDir()); + // contents of the JSON don't matter, we just need a file to exist + writeTestFile(fileSettingsService.watchedFile(), "{}"); + fileSettingsService.start(); fileSettingsService.clusterChanged(new ClusterChangedEvent("test", clusterService.state(), ClusterState.EMPTY_STATE)); - // second file change; contents still don't matter - overwriteTestFile(fileSettingsService.watchedFile(), "{}"); - // wait for listener to be called (once for initial processing, once for subsequent update) - assertTrue(latch.await(20, TimeUnit.SECONDS)); + longAwait(changesOnStartLatch); verify(fileSettingsService, times(1)).processFileOnServiceStart(); verify(controller, times(1)).process(any(), any(XContentParser.class), eq(ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION), any()); + + // second file change; contents still don't matter + writeTestFile(fileSettingsService.watchedFile(), "[]"); + longAwait(changesLatch); + verify(fileSettingsService, times(1)).processFileChanges(); verify(controller, times(1)).process(any(), any(XContentParser.class), eq(ReservedStateVersionCheck.HIGHER_VERSION_ONLY), any()); } @@ -295,9 +297,7 @@ public void testStopWorksInMiddleOfProcessing() throws Exception { // Make some fake settings file to cause the file settings service to process it writeTestFile(fileSettingsService.watchedFile(), "{}"); - // we need to wait a bit, on MacOS it may take up to 10 seconds for the Java watcher service to notice the file, - // on Linux is instantaneous. Windows is instantaneous too. - assertTrue(processFileLatch.await(30, TimeUnit.SECONDS)); + longAwait(processFileLatch); // Stopping the service should interrupt the watcher thread, we should be able to stop fileSettingsService.stop(); @@ -352,15 +352,34 @@ public void testHandleSnapshotRestoreResetsMetadata() throws Exception { } // helpers - private void writeTestFile(Path path, String contents) throws IOException { - Path tempFilePath = createTempFile(); - Files.writeString(tempFilePath, contents); - Files.move(tempFilePath, path, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + private static void writeTestFile(Path path, String contents) { + Path tempFile = null; + try { + tempFile = Files.createTempFile(path.getParent(), path.getFileName().toString(), "tmp"); + Files.writeString(tempFile, contents); + + try { + Files.move(tempFile, path, REPLACE_EXISTING, ATOMIC_MOVE); + } catch (AtomicMoveNotSupportedException e) { + Files.move(tempFile, path, REPLACE_EXISTING); + } + } catch (final IOException e) { + throw new UncheckedIOException(Strings.format("could not write file [%s]", path.toAbsolutePath()), e); + } finally { + // we are ignoring exceptions here, so we do not need handle whether or not tempFile was initialized nor if the file exists + IOUtils.deleteFilesIgnoringExceptions(tempFile); + } } - private void overwriteTestFile(Path path, String contents) throws IOException { - Path tempFilePath = createTempFile(); - Files.writeString(tempFilePath, contents); - Files.move(tempFilePath, path, StandardCopyOption.REPLACE_EXISTING); + // this waits for up to 20 seconds to account for watcher service differences between OSes + // on MacOS it may take up to 10 seconds for the Java watcher service to notice the file, + // on Linux is instantaneous. Windows is instantaneous too. + private static void longAwait(CountDownLatch latch) { + try { + assertTrue("longAwait: CountDownLatch did not reach zero within the timeout", latch.await(20, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + fail(e, "longAwait: interrupted waiting for CountDownLatch to reach zero"); + } } } From a7031d871654c4ab73fc747b43de2bcbf863cf45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Fred=C3=A9n?= <109296772+jfreden@users.noreply.github.com> Date: Tue, 29 Oct 2024 12:31:02 +0100 Subject: [PATCH 03/18] Add ECK Role Mapping Cleanup (#115823) * Add security migration for cleaning up ECK role mappings --- docs/changelog/115823.yaml | 5 + .../FileSettingsRoleMappingUpgradeIT.java | 40 +- .../metadata/ReservedStateMetadata.java | 15 + .../elasticsearch/index/IndexVersions.java | 2 +- .../support/mapper/ExpressionRoleMapping.java | 15 +- .../security/authz/RoleMappingMetadata.java | 8 + .../RoleMappingFileSettingsIT.java | 23 + ...eanupRoleMappingDuplicatesMigrationIT.java | 417 ++++++++++++++++++ .../xpack/security/SecurityFeatures.java | 3 +- .../support/SecurityIndexManager.java | 83 +++- .../security/support/SecurityMigrations.java | 235 +++++++--- .../support/SecuritySystemIndices.java | 1 + .../authc/AuthenticationServiceTests.java | 1 + .../authc/esnative/NativeRealmTests.java | 1 + .../mapper/NativeRoleMappingStoreTests.java | 1 + .../authz/store/CompositeRolesStoreTests.java | 1 + .../store/NativePrivilegeStoreTests.java | 1 + .../CacheInvalidatorRegistryTests.java | 1 + .../support/SecurityIndexManagerTests.java | 138 ++++++ .../support/SecurityMigrationsTests.java | 174 ++++++++ x-pack/qa/rolling-upgrade/build.gradle | 2 + .../upgrades/AbstractUpgradeTestCase.java | 21 + .../SecurityIndexRoleMappingCleanupIT.java | 146 ++++++ ...SecurityIndexRolesMetadataMigrationIT.java | 19 +- .../operator_defined_role_mappings.json | 38 ++ 25 files changed, 1298 insertions(+), 93 deletions(-) create mode 100644 docs/changelog/115823.yaml create mode 100644 x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/CleanupRoleMappingDuplicatesMigrationIT.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityMigrationsTests.java create mode 100644 x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRoleMappingCleanupIT.java create mode 100644 x-pack/qa/rolling-upgrade/src/test/resources/operator_defined_role_mappings.json diff --git a/docs/changelog/115823.yaml b/docs/changelog/115823.yaml new file mode 100644 index 0000000000000..a6119e0fa56e4 --- /dev/null +++ b/docs/changelog/115823.yaml @@ -0,0 +1,5 @@ +pr: 115823 +summary: Add ECK Role Mapping Cleanup +area: Security +type: bug +issues: [] diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java index 834d97f755dfb..4caf33feeeebb 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java @@ -23,19 +23,20 @@ import org.junit.rules.TemporaryFolder; import org.junit.rules.TestRule; -import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.function.Supplier; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; public class FileSettingsRoleMappingUpgradeIT extends ParameterizedRollingUpgradeTestCase { - private static final String settingsJSON = """ + private static final int ROLE_MAPPINGS_CLEANUP_MIGRATION_VERSION = 2; + private static final String SETTING_JSON = """ { "metadata": { "version": "1", @@ -53,7 +54,6 @@ public class FileSettingsRoleMappingUpgradeIT extends ParameterizedRollingUpgrad }"""; private static final TemporaryFolder repoDirectory = new TemporaryFolder(); - private static final ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) .version(getOldClusterTestVersion()) @@ -68,7 +68,7 @@ public String get() { .setting("xpack.security.enabled", "true") // workaround to avoid having to set up clients and authorization headers .setting("xpack.security.authc.anonymous.roles", "superuser") - .configFile("operator/settings.json", Resource.fromString(settingsJSON)) + .configFile("operator/settings.json", Resource.fromString(SETTING_JSON)) .build(); @ClassRule @@ -91,7 +91,30 @@ public void checkVersions() { ); } - public void testRoleMappingsAppliedOnUpgrade() throws IOException { + private static void waitForSecurityMigrationCompletionIfIndexExists() throws Exception { + final Request request = new Request("GET", "_cluster/state/metadata/.security-7"); + assertBusy(() -> { + Map indices = new XContentTestUtils.JsonMapView(entityAsMap(client().performRequest(request))).get( + "metadata.indices" + ); + assertNotNull(indices); + // If the security index exists, migration needs to happen. There is a bug in pre cluster state role mappings code that tries + // to write file based role mappings before security index manager state is recovered, this makes it look like the security + // index is outdated (isIndexUpToDate == false). Because we can't rely on the index being there for old versions, this check + // is needed. + if (indices.containsKey(".security-7")) { + // JsonMapView doesn't support . prefixed indices (splits on .) + @SuppressWarnings("unchecked") + String responseVersion = new XContentTestUtils.JsonMapView((Map) indices.get(".security-7")).get( + "migration_version.version" + ); + assertNotNull(responseVersion); + assertTrue(Integer.parseInt(responseVersion) >= ROLE_MAPPINGS_CLEANUP_MIGRATION_VERSION); + } + }); + } + + public void testRoleMappingsAppliedOnUpgrade() throws Exception { if (isOldCluster()) { Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata"); List roleMappings = new XContentTestUtils.JsonMapView(entityAsMap(client().performRequest(clusterStateRequest))).get( @@ -107,11 +130,10 @@ public void testRoleMappingsAppliedOnUpgrade() throws IOException { ).get("metadata.role_mappings.role_mappings"); assertThat(clusterStateRoleMappings, is(not(nullValue()))); assertThat(clusterStateRoleMappings.size(), equalTo(1)); - + waitForSecurityMigrationCompletionIfIndexExists(); assertThat( entityAsMap(client().performRequest(new Request("GET", "/_security/role_mapping"))).keySet(), - // TODO change this to `contains` once the clean-up migration work is merged - hasItem("everyone_kibana-read-only-operator-mapping") + contains("everyone_kibana-read-only-operator-mapping") ); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateMetadata.java index 2390c96664057..a0b35f7cfc3eb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateMetadata.java @@ -91,6 +91,21 @@ public Set conflicts(String handlerName, Set modified) { return Collections.unmodifiableSet(intersect); } + /** + * Get the reserved keys for the handler name + * + * @param handlerName handler name to get keys for + * @return set of keys for that handler + */ + public Set keys(String handlerName) { + ReservedStateHandlerMetadata handlerMetadata = handlers.get(handlerName); + if (handlerMetadata == null || handlerMetadata.keys().isEmpty()) { + return Collections.emptySet(); + } + + return Collections.unmodifiableSet(handlerMetadata.keys()); + } + /** * Reads an {@link ReservedStateMetadata} from a {@link StreamInput} * diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index efb1facc79b3a..2919f98ee200e 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -128,7 +128,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion MERGE_ON_RECOVERY_VERSION = def(8_515_00_0, Version.LUCENE_9_11_1); public static final IndexVersion UPGRADE_TO_LUCENE_9_12 = def(8_516_00_0, Version.LUCENE_9_12_0); public static final IndexVersion ENABLE_IGNORE_ABOVE_LOGSDB = def(8_517_00_0, Version.LUCENE_9_12_0); - + public static final IndexVersion ADD_ROLE_MAPPING_CLEANUP_MIGRATION = def(8_518_00_0, Version.LUCENE_9_12_0); public static final IndexVersion UPGRADE_TO_LUCENE_10_0_0 = def(9_000_00_0, Version.LUCENE_10_0_0); /* diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java index c504ebe56ed45..41fd3c6938dfc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java @@ -206,7 +206,7 @@ public RoleMapperExpression getExpression() { * that match the {@link #getExpression() expression} in this mapping. */ public List getRoles() { - return Collections.unmodifiableList(roles); + return roles != null ? Collections.unmodifiableList(roles) : Collections.emptyList(); } /** @@ -214,7 +214,7 @@ public List getRoles() { * that should be assigned to users that match the {@link #getExpression() expression} in this mapping. */ public List getRoleTemplates() { - return Collections.unmodifiableList(roleTemplates); + return roleTemplates != null ? Collections.unmodifiableList(roleTemplates) : Collections.emptyList(); } /** @@ -223,7 +223,7 @@ public List getRoleTemplates() { * This is not used within the mapping process, and does not affect whether the expression matches, nor which roles are assigned. */ public Map getMetadata() { - return Collections.unmodifiableMap(metadata); + return metadata != null ? Collections.unmodifiableMap(metadata) : Collections.emptyMap(); } /** @@ -233,6 +233,15 @@ public boolean isEnabled() { return enabled; } + /** + * Whether this mapping is an operator defined/read only role mapping + */ + public boolean isReadOnly() { + return metadata != null && metadata.get(ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_METADATA_FLAG) instanceof Boolean readOnly + ? readOnly + : false; + } + @Override public String toString() { return getClass().getSimpleName() + "<" + name + " ; " + roles + "/" + roleTemplates + " = " + Strings.toString(expression) + ">"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java index 74c6223b1ebdd..31fe86ca77edd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java @@ -191,6 +191,14 @@ public static boolean hasFallbackName(ExpressionRoleMapping expressionRoleMappin return expressionRoleMapping.getName().equals(FALLBACK_NAME); } + /** + * Check if any of the role mappings have a fallback name + * @return true if any role mappings have the fallback name + */ + public boolean hasAnyMappingWithFallbackName() { + return roleMappings.stream().anyMatch(RoleMappingMetadata::hasFallbackName); + } + /** * Parse a role mapping from XContent, restoring the name from a reserved metadata field. * Used to parse a role mapping annotated with its name in metadata via @see {@link #copyWithNameInMetadata(ExpressionRoleMapping)}. diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java index fdd854e7a9673..9e36055e917a6 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java @@ -204,6 +204,29 @@ public void clusterChanged(ClusterChangedEvent event) { return new Tuple<>(savedClusterState, metadataVersion); } + // Wait for any file metadata + public static Tuple setupClusterStateListener(String node) { + ClusterService clusterService = internalCluster().clusterService(node); + CountDownLatch savedClusterState = new CountDownLatch(1); + AtomicLong metadataVersion = new AtomicLong(-1); + clusterService.addListener(new ClusterStateListener() { + @Override + public void clusterChanged(ClusterChangedEvent event) { + ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE); + if (reservedState != null) { + ReservedStateHandlerMetadata handlerMetadata = reservedState.handlers().get(ReservedRoleMappingAction.NAME); + if (handlerMetadata != null) { + clusterService.removeListener(this); + metadataVersion.set(event.state().metadata().version()); + savedClusterState.countDown(); + } + } + } + }); + + return new Tuple<>(savedClusterState, metadataVersion); + } + public static Tuple setupClusterStateListenerForCleanup(String node) { ClusterService clusterService = internalCluster().clusterService(node); CountDownLatch savedClusterState = new CountDownLatch(1); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/CleanupRoleMappingDuplicatesMigrationIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/CleanupRoleMappingDuplicatesMigrationIT.java new file mode 100644 index 0000000000000..63c510062bdad --- /dev/null +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/CleanupRoleMappingDuplicatesMigrationIT.java @@ -0,0 +1,417 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security.support; + +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.reservedstate.service.FileSettingsService; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.SecurityIntegTestCase; +import org.elasticsearch.xpack.core.security.action.UpdateIndexMigrationVersionAction; +import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingAction; +import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingRequest; +import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingResponse; +import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsAction; +import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequest; +import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsResponse; +import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingAction; +import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequest; +import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingResponse; +import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; +import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression; +import org.junit.Before; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import static org.elasticsearch.integration.RoleMappingFileSettingsIT.setupClusterStateListener; +import static org.elasticsearch.integration.RoleMappingFileSettingsIT.writeJSONFile; +import static org.elasticsearch.xpack.core.security.action.UpdateIndexMigrationVersionAction.MIGRATION_VERSION_CUSTOM_DATA_KEY; +import static org.elasticsearch.xpack.core.security.action.UpdateIndexMigrationVersionAction.MIGRATION_VERSION_CUSTOM_KEY; +import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false) +public class CleanupRoleMappingDuplicatesMigrationIT extends SecurityIntegTestCase { + + private final AtomicLong versionCounter = new AtomicLong(1); + + @Before + public void resetVersion() { + versionCounter.set(1); + } + + private static final String TEST_JSON_WITH_ROLE_MAPPINGS = """ + { + "metadata": { + "version": "%s", + "compatibility": "8.4.0" + }, + "state": { + "role_mappings": { + "everyone_kibana_alone": { + "enabled": true, + "roles": [ "kibana_user" ], + "rules": { "field": { "username": "*" } }, + "metadata": { + "uuid" : "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7", + "_foo": "something" + } + }, + "everyone_fleet_alone": { + "enabled": false, + "roles": [ "fleet_user" ], + "rules": { "field": { "username": "*" } }, + "metadata": { + "uuid" : "b9a59ba9-6b92-4be3-bb8d-02bb270cb3a7", + "_foo": "something_else" + } + } + } + } + }"""; + + private static final String TEST_JSON_WITH_FALLBACK_NAME = """ + { + "metadata": { + "version": "%s", + "compatibility": "8.4.0" + }, + "state": { + "role_mappings": { + "name_not_available_after_deserialization": { + "enabled": true, + "roles": [ "kibana_user", "kibana_admin" ], + "rules": { "field": { "username": "*" } }, + "metadata": { + "uuid" : "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7", + "_foo": "something" + } + } + } + } + }"""; + + private static final String TEST_JSON_WITH_EMPTY_ROLE_MAPPINGS = """ + { + "metadata": { + "version": "%s", + "compatibility": "8.4.0" + }, + "state": { + "role_mappings": {} + } + }"""; + + public void testMigrationSuccessful() throws Exception { + internalCluster().setBootstrapMasterNodeIndex(0); + ensureGreen(); + final String masterNode = internalCluster().getMasterName(); + + // Create a native role mapping to create security index and trigger migration (skipped initially) + createNativeRoleMapping("everyone_kibana_alone"); + createNativeRoleMapping("everyone_fleet_alone"); + createNativeRoleMapping("dont_clean_this_up"); + assertAllRoleMappings("everyone_kibana_alone", "everyone_fleet_alone", "dont_clean_this_up"); + + // Wait for file watcher to start + awaitFileSettingsWatcher(); + // Setup listener to wait for role mapping + var fileBasedRoleMappingsWrittenListener = setupClusterStateListener(masterNode, "everyone_kibana_alone"); + // Write role mappings + writeJSONFile(masterNode, TEST_JSON_WITH_ROLE_MAPPINGS, logger, versionCounter); + assertTrue(fileBasedRoleMappingsWrittenListener.v1().await(20, TimeUnit.SECONDS)); + waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION); + + // First migration is on a new index, so should skip all migrations. If we reset, it should re-trigger and run all migrations + resetMigration(); + + // Wait for the first migration to finish + waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION); + + assertAllRoleMappings( + "everyone_kibana_alone" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX, + "everyone_fleet_alone" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX, + "dont_clean_this_up" + ); + } + + public void testMigrationSuccessfulNoOverlap() throws Exception { + internalCluster().setBootstrapMasterNodeIndex(0); + ensureGreen(); + final String masterNode = internalCluster().getMasterName(); + + // Create a native role mapping to create security index and trigger migration (skipped initially) + createNativeRoleMapping("some_native_mapping"); + createNativeRoleMapping("some_other_native_mapping"); + assertAllRoleMappings("some_native_mapping", "some_other_native_mapping"); + + // Wait for file watcher to start + awaitFileSettingsWatcher(); + // Setup listener to wait for role mapping + var fileBasedRoleMappingsWrittenListener = setupClusterStateListener(masterNode, "everyone_kibana_alone"); + // Write role mappings with fallback name, this should block any security migration + writeJSONFile(masterNode, TEST_JSON_WITH_ROLE_MAPPINGS, logger, versionCounter); + assertTrue(fileBasedRoleMappingsWrittenListener.v1().await(20, TimeUnit.SECONDS)); + waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION); + + // First migration is on a new index, so should skip all migrations. If we reset, it should re-trigger and run all migrations + resetMigration(); + + // Wait for the first migration to finish + waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION); + + assertAllRoleMappings( + "everyone_kibana_alone" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX, + "everyone_fleet_alone" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX, + "some_native_mapping", + "some_other_native_mapping" + ); + } + + public void testMigrationSuccessfulNoNative() throws Exception { + internalCluster().setBootstrapMasterNodeIndex(0); + ensureGreen(); + final String masterNode = internalCluster().getMasterName(); + + // Create a native role mapping to create security index and trigger migration (skipped initially) + // Then delete it to test an empty role mapping store + createNativeRoleMapping("some_native_mapping"); + deleteNativeRoleMapping("some_native_mapping"); + // Wait for file watcher to start + awaitFileSettingsWatcher(); + // Setup listener to wait for role mapping + var fileBasedRoleMappingsWrittenListener = setupClusterStateListener(masterNode, "everyone_kibana_alone"); + // Write role mappings with fallback name, this should block any security migration + writeJSONFile(masterNode, TEST_JSON_WITH_ROLE_MAPPINGS, logger, versionCounter); + assertTrue(fileBasedRoleMappingsWrittenListener.v1().await(20, TimeUnit.SECONDS)); + waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION); + + // First migration is on a new index, so should skip all migrations. If we reset, it should re-trigger and run all migrations + resetMigration(); + + // Wait for the first migration to finish + waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION); + + assertAllRoleMappings( + "everyone_kibana_alone" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX, + "everyone_fleet_alone" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX + ); + } + + public void testMigrationFallbackNamePreCondition() throws Exception { + internalCluster().setBootstrapMasterNodeIndex(0); + ensureGreen(); + final String masterNode = internalCluster().getMasterName(); + // Wait for file watcher to start + awaitFileSettingsWatcher(); + + // Setup listener to wait for role mapping + var nameNotAvailableListener = setupClusterStateListener(masterNode, "name_not_available_after_deserialization"); + // Write role mappings with fallback name, this should block any security migration + writeJSONFile(masterNode, TEST_JSON_WITH_FALLBACK_NAME, logger, versionCounter); + assertTrue(nameNotAvailableListener.v1().await(20, TimeUnit.SECONDS)); + + // Create a native role mapping to create security index and trigger migration + createNativeRoleMapping("everyone_fleet_alone"); + waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION); + // First migration is on a new index, so should skip all migrations. If we reset, it should re-trigger and run all migrations + resetMigration(); + // Wait for the first migration to finish + waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION - 1); + + // Make sure migration didn't run yet (blocked by the fallback name) + assertMigrationLessThan(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION); + ClusterService clusterService = internalCluster().getInstance(ClusterService.class); + SecurityIndexManager.RoleMappingsCleanupMigrationStatus status = SecurityIndexManager.getRoleMappingsCleanupMigrationStatus( + clusterService.state(), + SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION - 1 + ); + assertThat(status, equalTo(SecurityIndexManager.RoleMappingsCleanupMigrationStatus.NOT_READY)); + + // Write file without fallback name in it to unblock migration + writeJSONFile(masterNode, TEST_JSON_WITH_ROLE_MAPPINGS, logger, versionCounter); + waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION); + } + + public void testSkipMigrationNoFileBasedMappings() throws Exception { + internalCluster().setBootstrapMasterNodeIndex(0); + ensureGreen(); + // Create a native role mapping to create security index and trigger migration (skipped initially) + createNativeRoleMapping("everyone_kibana_alone"); + createNativeRoleMapping("everyone_fleet_alone"); + assertAllRoleMappings("everyone_kibana_alone", "everyone_fleet_alone"); + + waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION); + + // First migration is on a new index, so should skip all migrations. If we reset, it should re-trigger and run all migrations + resetMigration(); + + // Wait for the first migration to finish + waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION); + + assertAllRoleMappings("everyone_kibana_alone", "everyone_fleet_alone"); + } + + public void testSkipMigrationEmptyFileBasedMappings() throws Exception { + internalCluster().setBootstrapMasterNodeIndex(0); + ensureGreen(); + final String masterNode = internalCluster().getMasterName(); + + // Wait for file watcher to start + awaitFileSettingsWatcher(); + // Setup listener to wait for any role mapping + var fileBasedRoleMappingsWrittenListener = setupClusterStateListener(masterNode); + // Write role mappings + writeJSONFile(masterNode, TEST_JSON_WITH_EMPTY_ROLE_MAPPINGS, logger, versionCounter); + assertTrue(fileBasedRoleMappingsWrittenListener.v1().await(20, TimeUnit.SECONDS)); + + // Create a native role mapping to create security index and trigger migration (skipped initially) + createNativeRoleMapping("everyone_kibana_alone"); + createNativeRoleMapping("everyone_fleet_alone"); + assertAllRoleMappings("everyone_kibana_alone", "everyone_fleet_alone"); + + waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION); + + // First migration is on a new index, so should skip all migrations. If we reset, it should re-trigger and run all migrations + resetMigration(); + + // Wait for the first migration to finish + waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION); + + assertAllRoleMappings("everyone_kibana_alone", "everyone_fleet_alone"); + } + + public void testNewIndexSkipMigration() { + internalCluster().setBootstrapMasterNodeIndex(0); + final String masterNode = internalCluster().getMasterName(); + ensureGreen(); + CountDownLatch awaitMigrations = awaitMigrationVersionUpdates( + masterNode, + SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION + ); + // Create a native role mapping to create security index and trigger migration + createNativeRoleMapping("everyone_kibana_alone"); + // Make sure no migration ran (set to current version without applying prior migrations) + safeAwait(awaitMigrations); + } + + /** + * Make sure all versions are applied to cluster state sequentially + */ + private CountDownLatch awaitMigrationVersionUpdates(String node, final int... versions) { + final ClusterService clusterService = internalCluster().clusterService(node); + final CountDownLatch allVersionsCountDown = new CountDownLatch(1); + final AtomicInteger currentVersionIdx = new AtomicInteger(0); + clusterService.addListener(new ClusterStateListener() { + @Override + public void clusterChanged(ClusterChangedEvent event) { + int currentMigrationVersion = getCurrentMigrationVersion(event.state()); + if (currentMigrationVersion > 0) { + assertThat(versions[currentVersionIdx.get()], lessThanOrEqualTo(currentMigrationVersion)); + if (versions[currentVersionIdx.get()] == currentMigrationVersion) { + currentVersionIdx.incrementAndGet(); + } + + if (currentVersionIdx.get() >= versions.length) { + clusterService.removeListener(this); + allVersionsCountDown.countDown(); + } + } + } + }); + + return allVersionsCountDown; + } + + private void assertAllRoleMappings(String... roleMappingNames) { + GetRoleMappingsResponse response = client().execute(GetRoleMappingsAction.INSTANCE, new GetRoleMappingsRequest()).actionGet(); + + assertTrue(response.hasMappings()); + assertThat(response.mappings().length, equalTo(roleMappingNames.length)); + + assertThat( + Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList(), + containsInAnyOrder( + roleMappingNames + + ) + ); + } + + private void awaitFileSettingsWatcher() throws Exception { + final String masterNode = internalCluster().getMasterName(); + FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode); + assertBusy(() -> assertTrue(masterFileSettingsService.watching())); + } + + private void resetMigration() { + client().execute( + UpdateIndexMigrationVersionAction.INSTANCE, + // -1 is a hack, since running a migration on version 0 on a new cluster will cause all migrations to be skipped (not needed) + new UpdateIndexMigrationVersionAction.Request(TimeValue.MAX_VALUE, -1, INTERNAL_SECURITY_MAIN_INDEX_7) + ).actionGet(); + } + + private void createNativeRoleMapping(String name) { + PutRoleMappingRequest request = new PutRoleMappingRequest(); + request.setName(name); + request.setRules(new FieldExpression("username", Collections.singletonList(new FieldExpression.FieldValue("*")))); + request.setRoles(List.of("superuser")); + + ActionFuture response = client().execute(PutRoleMappingAction.INSTANCE, request); + response.actionGet(); + } + + private void deleteNativeRoleMapping(String name) { + DeleteRoleMappingRequest request = new DeleteRoleMappingRequest(); + request.setName(name); + + ActionFuture response = client().execute(DeleteRoleMappingAction.INSTANCE, request); + response.actionGet(); + } + + private void assertMigrationVersionAtLeast(int expectedVersion) { + assertThat(getCurrentMigrationVersion(), greaterThanOrEqualTo(expectedVersion)); + } + + private void assertMigrationLessThan(int expectedVersion) { + assertThat(getCurrentMigrationVersion(), lessThan(expectedVersion)); + } + + private int getCurrentMigrationVersion(ClusterState state) { + IndexMetadata indexMetadata = state.metadata().getIndices().get(INTERNAL_SECURITY_MAIN_INDEX_7); + if (indexMetadata == null || indexMetadata.getCustomData(MIGRATION_VERSION_CUSTOM_KEY) == null) { + return 0; + } + return Integer.parseInt(indexMetadata.getCustomData(MIGRATION_VERSION_CUSTOM_KEY).get(MIGRATION_VERSION_CUSTOM_DATA_KEY)); + } + + private int getCurrentMigrationVersion() { + ClusterService clusterService = internalCluster().getInstance(ClusterService.class); + return getCurrentMigrationVersion(clusterService.state()); + } + + private void waitForMigrationCompletion(int version) throws Exception { + assertBusy(() -> assertMigrationVersionAtLeast(version)); + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatures.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatures.java index c1fe553f41334..d0292f32cd75f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatures.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatures.java @@ -17,13 +17,14 @@ import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MIGRATION_FRAMEWORK; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_PROFILE_ORIGIN_FEATURE; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_ROLES_METADATA_FLATTENED; +import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_ROLE_MAPPING_CLEANUP; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.VERSION_SECURITY_PROFILE_ORIGIN; public class SecurityFeatures implements FeatureSpecification { @Override public Set getFeatures() { - return Set.of(SECURITY_ROLES_METADATA_FLATTENED, SECURITY_MIGRATION_FRAMEWORK); + return Set.of(SECURITY_ROLE_MAPPING_CLEANUP, SECURITY_ROLES_METADATA_FLATTENED, SECURITY_MIGRATION_FRAMEWORK); } @Override diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index 6d9b0ef6aeebe..12ef800a7aae7 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -31,6 +31,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ReservedStateMetadata; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.core.TimeValue; @@ -46,7 +47,9 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata; import org.elasticsearch.xpack.security.SecurityFeatures; +import org.elasticsearch.xpack.security.action.rolemapping.ReservedRoleMappingAction; import java.time.Instant; import java.util.List; @@ -75,7 +78,7 @@ public class SecurityIndexManager implements ClusterStateListener { public static final String SECURITY_VERSION_STRING = "security-version"; - + protected static final String FILE_SETTINGS_METADATA_NAMESPACE = "file_settings"; private static final Logger logger = LogManager.getLogger(SecurityIndexManager.class); /** @@ -86,6 +89,13 @@ public enum Availability { PRIMARY_SHARDS } + public enum RoleMappingsCleanupMigrationStatus { + READY, + NOT_READY, + SKIP, + DONE + } + private final Client client; private final SystemIndexDescriptor systemIndexDescriptor; @@ -195,10 +205,6 @@ public boolean isMigrationsVersionAtLeast(Integer expectedMigrationsVersion) { return indexExists() && this.state.migrationsVersion.compareTo(expectedMigrationsVersion) >= 0; } - public boolean isCreatedOnLatestVersion() { - return this.state.createdOnLatestVersion; - } - public ElasticsearchException getUnavailableReason(Availability availability) { // ensure usage of a local copy so all checks execute against the same state! if (defensiveCopy == false) { @@ -261,6 +267,7 @@ private SystemIndexDescriptor.MappingsVersion getMinSecurityIndexMappingVersion( /** * Check if the index was created on the latest index version available in the cluster */ + private static boolean isCreatedOnLatestVersion(IndexMetadata indexMetadata) { final IndexVersion indexVersionCreated = indexMetadata != null ? SETTING_INDEX_VERSION_CREATED.get(indexMetadata.getSettings()) @@ -268,6 +275,50 @@ private static boolean isCreatedOnLatestVersion(IndexMetadata indexMetadata) { return indexVersionCreated != null && indexVersionCreated.onOrAfter(IndexVersion.current()); } + /** + * Check if a role mappings cleanup migration is needed or has already been performed and if the cluster is ready for a cleanup + * migration + * + * @param clusterState current cluster state + * @param migrationsVersion current migration version + * + * @return RoleMappingsCleanupMigrationStatus + */ + static RoleMappingsCleanupMigrationStatus getRoleMappingsCleanupMigrationStatus(ClusterState clusterState, int migrationsVersion) { + // Migration already finished + if (migrationsVersion >= SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION) { + return RoleMappingsCleanupMigrationStatus.DONE; + } + + ReservedStateMetadata fileSettingsMetadata = clusterState.metadata().reservedStateMetadata().get(FILE_SETTINGS_METADATA_NAMESPACE); + boolean hasFileSettingsMetadata = fileSettingsMetadata != null; + // If there is no fileSettingsMetadata, there should be no reserved state (this is to catch bugs related to + // name changes to FILE_SETTINGS_METADATA_NAMESPACE) + assert hasFileSettingsMetadata || clusterState.metadata().reservedStateMetadata().isEmpty() + : "ReservedStateMetadata contains unknown namespace"; + + // If no file based role mappings available -> migration not needed + if (hasFileSettingsMetadata == false || fileSettingsMetadata.keys(ReservedRoleMappingAction.NAME).isEmpty()) { + return RoleMappingsCleanupMigrationStatus.SKIP; + } + + RoleMappingMetadata roleMappingMetadata = RoleMappingMetadata.getFromClusterState(clusterState); + + // If there are file based role mappings, make sure they have the latest format (name available) and that they have all been + // synced to cluster state (same size as the reserved state keys) + if (roleMappingMetadata.getRoleMappings().size() == fileSettingsMetadata.keys(ReservedRoleMappingAction.NAME).size() + && roleMappingMetadata.hasAnyMappingWithFallbackName() == false) { + return RoleMappingsCleanupMigrationStatus.READY; + } + + // If none of the above conditions are met, wait for a state change to re-evaluate if the cluster is ready for migration + return RoleMappingsCleanupMigrationStatus.NOT_READY; + } + + public RoleMappingsCleanupMigrationStatus getRoleMappingsCleanupMigrationStatus() { + return state.roleMappingsCleanupMigrationStatus; + } + @Override public void clusterChanged(ClusterChangedEvent event) { if (event.state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) { @@ -285,8 +336,12 @@ public void clusterChanged(ClusterChangedEvent event) { Tuple available = checkIndexAvailable(event.state()); final boolean indexAvailableForWrite = available.v1(); final boolean indexAvailableForSearch = available.v2(); - final boolean mappingIsUpToDate = indexMetadata == null || checkIndexMappingUpToDate(event.state()); final int migrationsVersion = getMigrationVersionFromIndexMetadata(indexMetadata); + final RoleMappingsCleanupMigrationStatus roleMappingsCleanupMigrationStatus = getRoleMappingsCleanupMigrationStatus( + event.state(), + migrationsVersion + ); + final boolean mappingIsUpToDate = indexMetadata == null || checkIndexMappingUpToDate(event.state()); final SystemIndexDescriptor.MappingsVersion minClusterMappingVersion = getMinSecurityIndexMappingVersion(event.state()); final int indexMappingVersion = loadIndexMappingVersion(systemIndexDescriptor.getAliasName(), event.state()); final String concreteIndexName = indexMetadata == null @@ -315,6 +370,7 @@ public void clusterChanged(ClusterChangedEvent event) { indexAvailableForWrite, mappingIsUpToDate, createdOnLatestVersion, + roleMappingsCleanupMigrationStatus, migrationsVersion, minClusterMappingVersion, indexMappingVersion, @@ -474,7 +530,8 @@ private Tuple checkIndexAvailable(ClusterState state) { public boolean isEligibleSecurityMigration(SecurityMigrations.SecurityMigration securityMigration) { return state.securityFeatures.containsAll(securityMigration.nodeFeaturesRequired()) - && state.indexMappingVersion >= securityMigration.minMappingVersion(); + && state.indexMappingVersion >= securityMigration.minMappingVersion() + && securityMigration.checkPreConditions(state); } public boolean isReadyForSecurityMigration(SecurityMigrations.SecurityMigration securityMigration) { @@ -680,6 +737,10 @@ public void onFailure(Exception e) { } } + public boolean isCreatedOnLatestVersion() { + return state.createdOnLatestVersion; + } + /** * Return true if the state moves from an unhealthy ("RED") index state to a healthy ("non-RED") state. */ @@ -714,6 +775,7 @@ public static class State { null, null, null, + null, Set.of() ); public final Instant creationTime; @@ -722,6 +784,7 @@ public static class State { public final boolean indexAvailableForWrite; public final boolean mappingUpToDate; public final boolean createdOnLatestVersion; + public final RoleMappingsCleanupMigrationStatus roleMappingsCleanupMigrationStatus; public final Integer migrationsVersion; // Min mapping version supported by the descriptors in the cluster public final SystemIndexDescriptor.MappingsVersion minClusterMappingVersion; @@ -740,6 +803,7 @@ public State( boolean indexAvailableForWrite, boolean mappingUpToDate, boolean createdOnLatestVersion, + RoleMappingsCleanupMigrationStatus roleMappingsCleanupMigrationStatus, Integer migrationsVersion, SystemIndexDescriptor.MappingsVersion minClusterMappingVersion, Integer indexMappingVersion, @@ -756,6 +820,7 @@ public State( this.mappingUpToDate = mappingUpToDate; this.migrationsVersion = migrationsVersion; this.createdOnLatestVersion = createdOnLatestVersion; + this.roleMappingsCleanupMigrationStatus = roleMappingsCleanupMigrationStatus; this.minClusterMappingVersion = minClusterMappingVersion; this.indexMappingVersion = indexMappingVersion; this.concreteIndexName = concreteIndexName; @@ -776,6 +841,7 @@ public boolean equals(Object o) { && indexAvailableForWrite == state.indexAvailableForWrite && mappingUpToDate == state.mappingUpToDate && createdOnLatestVersion == state.createdOnLatestVersion + && roleMappingsCleanupMigrationStatus == state.roleMappingsCleanupMigrationStatus && Objects.equals(indexMappingVersion, state.indexMappingVersion) && Objects.equals(migrationsVersion, state.migrationsVersion) && Objects.equals(minClusterMappingVersion, state.minClusterMappingVersion) @@ -798,6 +864,7 @@ public int hashCode() { indexAvailableForWrite, mappingUpToDate, createdOnLatestVersion, + roleMappingsCleanupMigrationStatus, migrationsVersion, minClusterMappingVersion, indexMappingVersion, @@ -822,6 +889,8 @@ public String toString() { + mappingUpToDate + ", createdOnLatestVersion=" + createdOnLatestVersion + + ", roleMappingsCleanupMigrationStatus=" + + roleMappingsCleanupMigrationStatus + ", migrationsVersion=" + migrationsVersion + ", minClusterMappingVersion=" diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java index 5cd8cba763d3d..203dec9e25b91 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java @@ -11,6 +11,8 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.support.GroupedActionListener; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.internal.Client; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -20,20 +22,35 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingAction; +import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingRequestBuilder; +import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingResponse; +import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsAction; +import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequestBuilder; +import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsResponse; +import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; +import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; +import java.util.stream.Collectors; +import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.RoleMappingsCleanupMigrationStatus.READY; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.RoleMappingsCleanupMigrationStatus.SKIP; +import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SecurityMainIndexMappingVersion.ADD_MANAGE_ROLES_PRIVILEGE; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SecurityMainIndexMappingVersion.ADD_REMOTE_CLUSTER_AND_DESCRIPTION_FIELDS; -/** - * Interface for creating SecurityMigrations that will be automatically applied once to existing .security indices - * IMPORTANT: A new index version needs to be added to {@link org.elasticsearch.index.IndexVersions} for the migration to be triggered - */ public class SecurityMigrations { + /** + * Interface for creating SecurityMigrations that will be automatically applied once to existing .security indices + * IMPORTANT: A new index version needs to be added to {@link org.elasticsearch.index.IndexVersions} for the migration to be triggered + */ public interface SecurityMigration { /** * Method that will execute the actual migration - needs to be idempotent and non-blocking @@ -52,6 +69,16 @@ public interface SecurityMigration { */ Set nodeFeaturesRequired(); + /** + * Check that any pre-conditions are met before launching migration + * + * @param securityIndexManagerState current state of the security index + * @return true if pre-conditions met, otherwise false + */ + default boolean checkPreConditions(SecurityIndexManager.State securityIndexManagerState) { + return true; + } + /** * The min mapping version required to support this migration. This makes sure that the index has at least the min mapping that is * required to support the migration. @@ -62,63 +89,163 @@ public interface SecurityMigration { } public static final Integer ROLE_METADATA_FLATTENED_MIGRATION_VERSION = 1; + public static final Integer CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION = 2; + private static final Logger logger = LogManager.getLogger(SecurityMigration.class); public static final TreeMap MIGRATIONS_BY_VERSION = new TreeMap<>( - Map.of(ROLE_METADATA_FLATTENED_MIGRATION_VERSION, new SecurityMigration() { - private static final Logger logger = LogManager.getLogger(SecurityMigration.class); - - @Override - public void migrate(SecurityIndexManager indexManager, Client client, ActionListener listener) { - BoolQueryBuilder filterQuery = new BoolQueryBuilder().filter(QueryBuilders.termQuery("type", "role")) - .mustNot(QueryBuilders.existsQuery("metadata_flattened")); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(filterQuery).size(0).trackTotalHits(true); - SearchRequest countRequest = new SearchRequest(indexManager.getConcreteIndexName()); - countRequest.source(searchSourceBuilder); - - client.search(countRequest, ActionListener.wrap(response -> { - // If there are no roles, skip migration - if (response.getHits().getTotalHits().value() > 0) { - logger.info("Preparing to migrate [" + response.getHits().getTotalHits().value() + "] roles"); - updateRolesByQuery(indexManager, client, filterQuery, listener); - } else { - listener.onResponse(null); - } + Map.of( + ROLE_METADATA_FLATTENED_MIGRATION_VERSION, + new RoleMetadataFlattenedMigration(), + CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION, + new CleanupRoleMappingDuplicatesMigration() + ) + ); + + public static class RoleMetadataFlattenedMigration implements SecurityMigration { + @Override + public void migrate(SecurityIndexManager indexManager, Client client, ActionListener listener) { + BoolQueryBuilder filterQuery = new BoolQueryBuilder().filter(QueryBuilders.termQuery("type", "role")) + .mustNot(QueryBuilders.existsQuery("metadata_flattened")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(filterQuery).size(0).trackTotalHits(true); + SearchRequest countRequest = new SearchRequest(indexManager.getConcreteIndexName()); + countRequest.source(searchSourceBuilder); + + client.search(countRequest, ActionListener.wrap(response -> { + // If there are no roles, skip migration + if (response.getHits().getTotalHits().value() > 0) { + logger.info("Preparing to migrate [" + response.getHits().getTotalHits().value() + "] roles"); + updateRolesByQuery(indexManager, client, filterQuery, listener); + } else { + listener.onResponse(null); + } + }, listener::onFailure)); + } + + private void updateRolesByQuery( + SecurityIndexManager indexManager, + Client client, + BoolQueryBuilder filterQuery, + ActionListener listener + ) { + UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexManager.getConcreteIndexName()); + updateByQueryRequest.setQuery(filterQuery); + updateByQueryRequest.setScript( + new Script(ScriptType.INLINE, "painless", "ctx._source.metadata_flattened = ctx._source.metadata", Collections.emptyMap()) + ); + client.admin() + .cluster() + .execute(UpdateByQueryAction.INSTANCE, updateByQueryRequest, ActionListener.wrap(bulkByScrollResponse -> { + logger.info("Migrated [" + bulkByScrollResponse.getTotal() + "] roles"); + listener.onResponse(null); }, listener::onFailure)); + } + + @Override + public Set nodeFeaturesRequired() { + return Set.of(SecuritySystemIndices.SECURITY_ROLES_METADATA_FLATTENED); + } + + @Override + public int minMappingVersion() { + return ADD_REMOTE_CLUSTER_AND_DESCRIPTION_FIELDS.id(); + } + } + + public static class CleanupRoleMappingDuplicatesMigration implements SecurityMigration { + @Override + public void migrate(SecurityIndexManager indexManager, Client client, ActionListener listener) { + if (indexManager.getRoleMappingsCleanupMigrationStatus() == SKIP) { + listener.onResponse(null); + return; } + assert indexManager.getRoleMappingsCleanupMigrationStatus() == READY; - private void updateRolesByQuery( - SecurityIndexManager indexManager, - Client client, - BoolQueryBuilder filterQuery, - ActionListener listener - ) { - UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexManager.getConcreteIndexName()); - updateByQueryRequest.setQuery(filterQuery); - updateByQueryRequest.setScript( - new Script( - ScriptType.INLINE, - "painless", - "ctx._source.metadata_flattened = ctx._source.metadata", - Collections.emptyMap() - ) + getRoleMappings(client, ActionListener.wrap(roleMappings -> { + List roleMappingsToDelete = getDuplicateRoleMappingNames(roleMappings.mappings()); + if (roleMappingsToDelete.isEmpty() == false) { + logger.info("Found [" + roleMappingsToDelete.size() + "] role mapping(s) to cleanup in .security index."); + deleteNativeRoleMappings(client, roleMappingsToDelete, listener); + } else { + listener.onResponse(null); + } + }, listener::onFailure)); + } + + private void getRoleMappings(Client client, ActionListener listener) { + executeAsyncWithOrigin( + client, + SECURITY_ORIGIN, + GetRoleMappingsAction.INSTANCE, + new GetRoleMappingsRequestBuilder(client).request(), + listener + ); + } + + private void deleteNativeRoleMappings(Client client, List names, ActionListener listener) { + assert names.isEmpty() == false; + ActionListener groupListener = new GroupedActionListener<>( + names.size(), + ActionListener.wrap(responses -> { + long foundRoleMappings = responses.stream().filter(DeleteRoleMappingResponse::isFound).count(); + if (responses.size() > foundRoleMappings) { + logger.warn( + "[" + (responses.size() - foundRoleMappings) + "] Role mapping(s) not found during role mapping clean up." + ); + } + if (foundRoleMappings > 0) { + logger.info("Deleted [" + foundRoleMappings + "] duplicated role mapping(s) from .security index"); + } + listener.onResponse(null); + }, listener::onFailure) + ); + + for (String name : names) { + executeAsyncWithOrigin( + client, + SECURITY_ORIGIN, + DeleteRoleMappingAction.INSTANCE, + new DeleteRoleMappingRequestBuilder(client).name(name).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).request(), + groupListener ); - client.admin() - .cluster() - .execute(UpdateByQueryAction.INSTANCE, updateByQueryRequest, ActionListener.wrap(bulkByScrollResponse -> { - logger.info("Migrated [" + bulkByScrollResponse.getTotal() + "] roles"); - listener.onResponse(null); - }, listener::onFailure)); } - @Override - public Set nodeFeaturesRequired() { - return Set.of(SecuritySystemIndices.SECURITY_ROLES_METADATA_FLATTENED); - } + } - @Override - public int minMappingVersion() { - return ADD_REMOTE_CLUSTER_AND_DESCRIPTION_FIELDS.id(); - } - }) - ); + @Override + public boolean checkPreConditions(SecurityIndexManager.State securityIndexManagerState) { + // Block migration until expected role mappings are in cluster state and in the correct format or skip if no role mappings + // are expected + return securityIndexManagerState.roleMappingsCleanupMigrationStatus == READY + || securityIndexManagerState.roleMappingsCleanupMigrationStatus == SKIP; + } + + @Override + public Set nodeFeaturesRequired() { + return Set.of(SecuritySystemIndices.SECURITY_ROLE_MAPPING_CLEANUP); + } + + @Override + public int minMappingVersion() { + return ADD_MANAGE_ROLES_PRIVILEGE.id(); + } + + // Visible for testing + protected static List getDuplicateRoleMappingNames(ExpressionRoleMapping... roleMappings) { + // Partition role mappings on if they're cluster state role mappings (true) or native role mappings (false) + Map> partitionedRoleMappings = Arrays.stream(roleMappings) + .collect(Collectors.partitioningBy(ExpressionRoleMapping::isReadOnly)); + + Set clusterStateRoleMappings = partitionedRoleMappings.get(true) + .stream() + .map(ExpressionRoleMapping::getName) + .map(ExpressionRoleMapping::removeReadOnlySuffixIfPresent) + .collect(Collectors.toSet()); + + return partitionedRoleMappings.get(false) + .stream() + .map(ExpressionRoleMapping::getName) + .filter(clusterStateRoleMappings::contains) + .toList(); + } + } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java index 36ea14c6e101b..77c7d19e94a9b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java @@ -61,6 +61,7 @@ public class SecuritySystemIndices { public static final NodeFeature SECURITY_PROFILE_ORIGIN_FEATURE = new NodeFeature("security.security_profile_origin"); public static final NodeFeature SECURITY_MIGRATION_FRAMEWORK = new NodeFeature("security.migration_framework"); public static final NodeFeature SECURITY_ROLES_METADATA_FLATTENED = new NodeFeature("security.roles_metadata_flattened"); + public static final NodeFeature SECURITY_ROLE_MAPPING_CLEANUP = new NodeFeature("security.role_mapping_cleanup"); /** * Security managed index mappings used to be updated based on the product version. They are now updated based on per-index mappings diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index e1c3b936e5a32..cd6c88cf525af 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -2518,6 +2518,7 @@ private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { null, null, null, + null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java index 2254c78a2910c..75d5959f351f0 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java @@ -43,6 +43,7 @@ private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { null, null, null, + null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java index 38f01d4d18bc7..ca84a9189d90a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java @@ -415,6 +415,7 @@ private SecurityIndexManager.State indexState(boolean isUpToDate, ClusterHealthS null, null, null, + null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java index 9587533d87d86..da903ff7f7177 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java @@ -1702,6 +1702,7 @@ public SecurityIndexManager.State dummyIndexState(boolean isIndexUpToDate, Clust null, null, null, + null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java index f91cb567ba689..73a45dc20ac42 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java @@ -904,6 +904,7 @@ private SecurityIndexManager.State dummyState( null, null, null, + null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java index e3b00dfbcc6b8..d551dded4e566 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java @@ -63,6 +63,7 @@ public void testSecurityIndexStateChangeWillInvalidateAllRegisteredInvalidators( true, true, null, + null, new SystemIndexDescriptor.MappingsVersion(SecurityMainIndexMappingVersion.latest().id(), 0), null, ".security", diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java index 493483a5e4a1b..0b98a595a6ab9 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java @@ -23,6 +23,8 @@ import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata; +import org.elasticsearch.cluster.metadata.ReservedStateMetadata; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -51,8 +53,11 @@ import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; +import org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata; import org.elasticsearch.xpack.core.security.test.TestRestrictedIndices; import org.elasticsearch.xpack.security.SecurityFeatures; +import org.elasticsearch.xpack.security.action.rolemapping.ReservedRoleMappingAction; import org.elasticsearch.xpack.security.support.SecuritySystemIndices.SecurityMainIndexMappingVersion; import org.elasticsearch.xpack.security.test.SecurityTestUtils; import org.hamcrest.Matchers; @@ -70,6 +75,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.FILE_SETTINGS_METADATA_NAMESPACE; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; @@ -654,6 +660,138 @@ public int minMappingVersion() { })); } + public void testNotReadyForMigrationBecauseOfPrecondition() { + final ClusterState.Builder clusterStateBuilder = createClusterState( + TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7, + SecuritySystemIndices.SECURITY_MAIN_ALIAS, + IndexMetadata.State.OPEN + ); + clusterStateBuilder.nodeFeatures( + Map.of("1", new SecurityFeatures().getFeatures().stream().map(NodeFeature::id).collect(Collectors.toSet())) + ); + manager.clusterChanged(event(markShardsAvailable(clusterStateBuilder))); + assertFalse(manager.isReadyForSecurityMigration(new SecurityMigrations.SecurityMigration() { + @Override + public void migrate(SecurityIndexManager indexManager, Client client, ActionListener listener) { + listener.onResponse(null); + } + + @Override + public Set nodeFeaturesRequired() { + return Set.of(); + } + + @Override + public int minMappingVersion() { + return 0; + } + + @Override + public boolean checkPreConditions(SecurityIndexManager.State securityIndexManagerState) { + return false; + } + })); + } + + private ClusterState.Builder clusterStateBuilderForMigrationTesting() { + return createClusterState( + TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7, + SecuritySystemIndices.SECURITY_MAIN_ALIAS, + IndexMetadata.State.OPEN + ); + } + + public void testGetRoleMappingsCleanupMigrationStatus() { + { + assertThat( + SecurityIndexManager.getRoleMappingsCleanupMigrationStatus( + clusterStateBuilderForMigrationTesting().build(), + SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION + ), + equalTo(SecurityIndexManager.RoleMappingsCleanupMigrationStatus.DONE) + ); + } + { + // Migration should be skipped + ClusterState.Builder clusterStateBuilder = clusterStateBuilderForMigrationTesting(); + Metadata.Builder metadataBuilder = new Metadata.Builder(); + metadataBuilder.put(ReservedStateMetadata.builder(FILE_SETTINGS_METADATA_NAMESPACE).build()); + assertThat( + SecurityIndexManager.getRoleMappingsCleanupMigrationStatus(clusterStateBuilder.metadata(metadataBuilder).build(), 1), + equalTo(SecurityIndexManager.RoleMappingsCleanupMigrationStatus.SKIP) + ); + } + { + // Not ready for migration + ClusterState.Builder clusterStateBuilder = clusterStateBuilderForMigrationTesting(); + Metadata.Builder metadataBuilder = new Metadata.Builder(); + ReservedStateMetadata.Builder builder = ReservedStateMetadata.builder(FILE_SETTINGS_METADATA_NAMESPACE); + // File settings role mappings exist + ReservedStateHandlerMetadata reservedStateHandlerMetadata = new ReservedStateHandlerMetadata( + ReservedRoleMappingAction.NAME, + Set.of("role_mapping_1") + ); + builder.putHandler(reservedStateHandlerMetadata); + metadataBuilder.put(builder.build()); + + // No role mappings in cluster state yet + metadataBuilder.putCustom(RoleMappingMetadata.TYPE, new RoleMappingMetadata(Set.of())); + + assertThat( + SecurityIndexManager.getRoleMappingsCleanupMigrationStatus(clusterStateBuilder.metadata(metadataBuilder).build(), 1), + equalTo(SecurityIndexManager.RoleMappingsCleanupMigrationStatus.NOT_READY) + ); + } + { + // Old role mappings in cluster state + final ClusterState.Builder clusterStateBuilder = clusterStateBuilderForMigrationTesting(); + Metadata.Builder metadataBuilder = new Metadata.Builder(); + ReservedStateMetadata.Builder builder = ReservedStateMetadata.builder(FILE_SETTINGS_METADATA_NAMESPACE); + // File settings role mappings exist + ReservedStateHandlerMetadata reservedStateHandlerMetadata = new ReservedStateHandlerMetadata( + ReservedRoleMappingAction.NAME, + Set.of("role_mapping_1") + ); + builder.putHandler(reservedStateHandlerMetadata); + metadataBuilder.put(builder.build()); + + // Role mappings in cluster state with fallback name + metadataBuilder.putCustom( + RoleMappingMetadata.TYPE, + new RoleMappingMetadata(Set.of(new ExpressionRoleMapping(RoleMappingMetadata.FALLBACK_NAME, null, null, null, null, true))) + ); + + assertThat( + SecurityIndexManager.getRoleMappingsCleanupMigrationStatus(clusterStateBuilder.metadata(metadataBuilder).build(), 1), + equalTo(SecurityIndexManager.RoleMappingsCleanupMigrationStatus.NOT_READY) + ); + } + { + // Ready for migration + final ClusterState.Builder clusterStateBuilder = clusterStateBuilderForMigrationTesting(); + Metadata.Builder metadataBuilder = new Metadata.Builder(); + ReservedStateMetadata.Builder builder = ReservedStateMetadata.builder(FILE_SETTINGS_METADATA_NAMESPACE); + // File settings role mappings exist + ReservedStateHandlerMetadata reservedStateHandlerMetadata = new ReservedStateHandlerMetadata( + ReservedRoleMappingAction.NAME, + Set.of("role_mapping_1") + ); + builder.putHandler(reservedStateHandlerMetadata); + metadataBuilder.put(builder.build()); + + // Role mappings in cluster state + metadataBuilder.putCustom( + RoleMappingMetadata.TYPE, + new RoleMappingMetadata(Set.of(new ExpressionRoleMapping("role_mapping_1", null, null, null, null, true))) + ); + + assertThat( + SecurityIndexManager.getRoleMappingsCleanupMigrationStatus(clusterStateBuilder.metadata(metadataBuilder).build(), 1), + equalTo(SecurityIndexManager.RoleMappingsCleanupMigrationStatus.READY) + ); + } + } + public void testProcessClosedIndexState() { // Index initially exists final ClusterState.Builder indexAvailable = createClusterState( diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityMigrationsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityMigrationsTests.java new file mode 100644 index 0000000000000..3d3cc47b55cf6 --- /dev/null +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityMigrationsTests.java @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.security.support; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingAction; +import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingRequest; +import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingResponse; +import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsAction; +import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequest; +import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsResponse; +import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; +import org.junit.After; +import org.junit.Before; + +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class SecurityMigrationsTests extends ESTestCase { + private ThreadPool threadPool; + private Client client; + + public void testGetDuplicateRoleMappingNames() { + assertThat(SecurityMigrations.CleanupRoleMappingDuplicatesMigration.getDuplicateRoleMappingNames(), empty()); + assertThat( + SecurityMigrations.CleanupRoleMappingDuplicatesMigration.getDuplicateRoleMappingNames( + nativeRoleMapping("roleMapping1"), + nativeRoleMapping("roleMapping2") + ), + empty() + ); + assertThat( + SecurityMigrations.CleanupRoleMappingDuplicatesMigration.getDuplicateRoleMappingNames( + nativeRoleMapping("roleMapping1"), + reservedRoleMapping("roleMapping1") + ), + equalTo(List.of("roleMapping1")) + ); + + { + List duplicates = SecurityMigrations.CleanupRoleMappingDuplicatesMigration.getDuplicateRoleMappingNames( + nativeRoleMapping("roleMapping1"), + nativeRoleMapping("roleMapping2"), + reservedRoleMapping("roleMapping1"), + reservedRoleMapping("roleMapping2") + ); + assertThat(duplicates, hasSize(2)); + assertThat(duplicates, containsInAnyOrder("roleMapping1", "roleMapping2")); + } + { + List duplicates = SecurityMigrations.CleanupRoleMappingDuplicatesMigration.getDuplicateRoleMappingNames( + nativeRoleMapping("roleMapping1"), + nativeRoleMapping("roleMapping2"), + nativeRoleMapping("roleMapping3"), + reservedRoleMapping("roleMapping1"), + reservedRoleMapping("roleMapping2"), + reservedRoleMapping("roleMapping4") + ); + assertThat(duplicates, hasSize(2)); + assertThat(duplicates, containsInAnyOrder("roleMapping1", "roleMapping2")); + } + { + List duplicates = SecurityMigrations.CleanupRoleMappingDuplicatesMigration.getDuplicateRoleMappingNames( + nativeRoleMapping("roleMapping1" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX), + nativeRoleMapping("roleMapping2" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX), + nativeRoleMapping("roleMapping3"), + reservedRoleMapping("roleMapping1"), + reservedRoleMapping("roleMapping2"), + reservedRoleMapping("roleMapping3") + ); + assertThat(duplicates, hasSize(1)); + assertThat(duplicates, containsInAnyOrder("roleMapping3")); + } + } + + private static ExpressionRoleMapping reservedRoleMapping(String name) { + return new ExpressionRoleMapping( + name + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX, + null, + null, + null, + Map.of(ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_METADATA_FLAG, true), + true + ); + } + + private static ExpressionRoleMapping nativeRoleMapping(String name) { + return new ExpressionRoleMapping(name, null, null, null, randomBoolean() ? null : Map.of(), true); + } + + public void testCleanupRoleMappingDuplicatesMigrationPartialFailure() { + // Make sure migration continues even if a duplicate is not found + SecurityIndexManager securityIndexManager = mock(SecurityIndexManager.class); + when(securityIndexManager.getRoleMappingsCleanupMigrationStatus()).thenReturn( + SecurityIndexManager.RoleMappingsCleanupMigrationStatus.READY + ); + doAnswer(inv -> { + final Object[] args = inv.getArguments(); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) args[2]; + listener.onResponse( + new GetRoleMappingsResponse( + nativeRoleMapping("duplicate-0"), + reservedRoleMapping("duplicate-0"), + nativeRoleMapping("duplicate-1"), + reservedRoleMapping("duplicate-1"), + nativeRoleMapping("duplicate-2"), + reservedRoleMapping("duplicate-2") + ) + ); + return null; + }).when(client).execute(eq(GetRoleMappingsAction.INSTANCE), any(GetRoleMappingsRequest.class), any()); + + final boolean[] duplicatesDeleted = new boolean[3]; + doAnswer(inv -> { + final Object[] args = inv.getArguments(); + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) args[2]; + DeleteRoleMappingRequest request = (DeleteRoleMappingRequest) args[1]; + if (request.getName().equals("duplicate-0")) { + duplicatesDeleted[0] = true; + } + if (request.getName().equals("duplicate-1")) { + if (randomBoolean()) { + listener.onResponse(new DeleteRoleMappingResponse(false)); + } else { + listener.onFailure(new IllegalStateException("bad state")); + } + } + if (request.getName().equals("duplicate-2")) { + duplicatesDeleted[2] = true; + } + return null; + }).when(client).execute(eq(DeleteRoleMappingAction.INSTANCE), any(DeleteRoleMappingRequest.class), any()); + + SecurityMigrations.SecurityMigration securityMigration = new SecurityMigrations.CleanupRoleMappingDuplicatesMigration(); + securityMigration.migrate(securityIndexManager, client, ActionListener.noop()); + + assertTrue(duplicatesDeleted[0]); + assertFalse(duplicatesDeleted[1]); + assertTrue(duplicatesDeleted[2]); + } + + @Before + public void createClientAndThreadPool() { + threadPool = new TestThreadPool("cleanup role mappings test pool"); + client = mock(Client.class); + when(client.threadPool()).thenReturn(threadPool); + } + + @After + public void stopThreadPool() { + terminate(threadPool); + } + +} diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle index b9b0531fa5b68..38fbf99068a9b 100644 --- a/x-pack/qa/rolling-upgrade/build.gradle +++ b/x-pack/qa/rolling-upgrade/build.gradle @@ -88,6 +88,8 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> keystore 'xpack.watcher.encryption_key', file("${project.projectDir}/src/test/resources/system_key") setting 'xpack.watcher.encrypt_sensitive_data', 'true' + extraConfigFile 'operator/settings.json', file("${project.projectDir}/src/test/resources/operator_defined_role_mappings.json") + // Old versions of the code contain an invalid assertion that trips // during tests. Versions 5.6.9 and 6.2.4 have been fixed by removing // the assertion, but this is impossible for released versions. diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java index 4324aed5fee18..b17644cd1c2a9 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java @@ -9,11 +9,13 @@ import org.elasticsearch.Build; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.Booleans; +import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.test.SecuritySettingsSourceField; import org.junit.Before; @@ -21,6 +23,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; public abstract class AbstractUpgradeTestCase extends ESRestTestCase { @@ -149,4 +152,22 @@ public void setupForTests() throws Exception { } }); } + + protected static void waitForSecurityMigrationCompletion(RestClient adminClient, int version) throws Exception { + final Request request = new Request("GET", "_cluster/state/metadata/.security-7"); + assertBusy(() -> { + Map indices = new XContentTestUtils.JsonMapView(entityAsMap(adminClient.performRequest(request))).get( + "metadata.indices" + ); + assertNotNull(indices); + assertTrue(indices.containsKey(".security-7")); + // JsonMapView doesn't support . prefixed indices (splits on .) + @SuppressWarnings("unchecked") + String responseVersion = new XContentTestUtils.JsonMapView((Map) indices.get(".security-7")).get( + "migration_version.version" + ); + assertNotNull(responseVersion); + assertTrue(Integer.parseInt(responseVersion) >= version); + }); + } } diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRoleMappingCleanupIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRoleMappingCleanupIT.java new file mode 100644 index 0000000000000..82d4050c044b1 --- /dev/null +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRoleMappingCleanupIT.java @@ -0,0 +1,146 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.upgrades; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.TransportVersions.V_8_15_0; +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.containsInAnyOrder; + +public class SecurityIndexRoleMappingCleanupIT extends AbstractUpgradeTestCase { + + public void testCleanupDuplicateMappings() throws Exception { + if (CLUSTER_TYPE == ClusterType.OLD) { + // If we're in a state where the same operator-defined role mappings can exist both in cluster state and the native store + // (V_8_15_0 transport added to security.role_mapping_cleanup feature added), create a state + // where the native store will need to be cleaned up + assumeTrue( + "Cleanup only needed before security.role_mapping_cleanup feature available in cluster", + clusterHasFeature("security.role_mapping_cleanup") == false + ); + assumeTrue( + "If role mappings are in cluster state but cleanup has not been performed yet, create duplicated role mappings", + minimumTransportVersion().onOrAfter(V_8_15_0) + ); + // Since the old cluster has role mappings in cluster state, but doesn't check duplicates, create duplicates + createNativeRoleMapping("operator_role_mapping_1", Map.of("meta", "test"), true); + createNativeRoleMapping("operator_role_mapping_2", Map.of("meta", "test"), true); + } else if (CLUSTER_TYPE == ClusterType.MIXED) { + // Create a native role mapping that doesn't conflict with anything before the migration run + createNativeRoleMapping("no_name_conflict", Map.of("meta", "test")); + } else if (CLUSTER_TYPE == ClusterType.UPGRADED) { + waitForSecurityMigrationCompletion(adminClient(), 2); + assertAllRoleMappings( + client(), + "operator_role_mapping_1" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX, + "operator_role_mapping_2" + ExpressionRoleMapping.READ_ONLY_ROLE_MAPPING_SUFFIX, + "no_name_conflict" + ); + // In the old cluster we might have created these (depending on the node features), so make sure they were removed + assertFalse(roleMappingExistsInSecurityIndex("operator_role_mapping_1")); + assertFalse(roleMappingExistsInSecurityIndex("operator_role_mapping_2")); + assertTrue(roleMappingExistsInSecurityIndex("no_name_conflict")); + // Make sure we can create and delete a conflicting role mapping again + createNativeRoleMapping("operator_role_mapping_1", Map.of("meta", "test"), true); + deleteNativeRoleMapping("operator_role_mapping_1", true); + } + } + + @SuppressWarnings("unchecked") + private boolean roleMappingExistsInSecurityIndex(String mappingName) throws IOException { + final Request request = new Request("POST", "/.security/_search"); + request.setJsonEntity(String.format(Locale.ROOT, """ + {"query":{"bool":{"must":[{"term":{"_id":"%s_%s"}}]}}}""", "role-mapping", mappingName)); + + request.setOptions( + expectWarnings( + "this request accesses system indices: [.security-7]," + + " but in a future major version, direct access to system indices will be prevented by default" + ) + ); + + Response response = adminClient().performRequest(request); + assertOK(response); + final Map responseMap = responseAsMap(response); + + Map hits = ((Map) responseMap.get("hits")); + return ((List) hits.get("hits")).isEmpty() == false; + } + + private void createNativeRoleMapping(String roleMappingName, Map metadata) throws IOException { + createNativeRoleMapping(roleMappingName, metadata, false); + } + + private void createNativeRoleMapping(String roleMappingName, Map metadata, boolean expectWarning) throws IOException { + final Request request = new Request("POST", "/_security/role_mapping/" + roleMappingName); + if (expectWarning) { + request.setOptions( + expectWarnings( + "A read-only role mapping with the same name [" + + roleMappingName + + "] has been previously defined in a configuration file. " + + "Both role mappings will be used to determine role assignments." + ) + ); + } + + BytesReference source = BytesReference.bytes( + jsonBuilder().map( + Map.of( + ExpressionRoleMapping.Fields.ROLES.getPreferredName(), + List.of("superuser"), + ExpressionRoleMapping.Fields.ENABLED.getPreferredName(), + true, + ExpressionRoleMapping.Fields.RULES.getPreferredName(), + Map.of("field", Map.of("username", "role-mapping-test-user")), + RoleDescriptor.Fields.METADATA.getPreferredName(), + metadata + ) + ) + ); + request.setJsonEntity(source.utf8ToString()); + assertOK(client().performRequest(request)); + } + + private void deleteNativeRoleMapping(String roleMappingName, boolean expectWarning) throws IOException { + final Request request = new Request("DELETE", "/_security/role_mapping/" + roleMappingName); + if (expectWarning) { + request.setOptions( + expectWarnings( + "A read-only role mapping with the same name [" + + roleMappingName + + "] has previously been defined in a configuration file. " + + "The native role mapping was deleted, but the read-only mapping will remain active " + + "and will be used to determine role assignments." + ) + ); + } + assertOK(client().performRequest(request)); + } + + private void assertAllRoleMappings(RestClient client, String... roleNames) throws IOException { + Request request = new Request("GET", "/_security/role_mapping"); + Response response = client.performRequest(request); + assertOK(response); + Map responseMap = responseAsMap(response); + + assertThat(responseMap.keySet(), containsInAnyOrder(roleNames)); + assertThat(responseMap.size(), is(roleNames.length)); + } +} diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRolesMetadataMigrationIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRolesMetadataMigrationIT.java index d31130e970f03..6c34e68297aa0 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRolesMetadataMigrationIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRolesMetadataMigrationIT.java @@ -58,7 +58,7 @@ public void testRoleMigration() throws Exception { } else if (CLUSTER_TYPE == ClusterType.UPGRADED) { createRoleWithMetadata(upgradedTestRole, Map.of("meta", "test")); assertTrue(canRolesBeMigrated()); - waitForMigrationCompletion(adminClient()); + waitForSecurityMigrationCompletion(adminClient(), 1); assertMigratedDocInSecurityIndex(oldTestRole, "meta", "test"); assertMigratedDocInSecurityIndex(mixed1TestRole, "meta", "test"); assertMigratedDocInSecurityIndex(mixed2TestRole, "meta", "test"); @@ -136,23 +136,6 @@ private static void assertNoMigration(RestClient adminClient) throws Exception { ); } - @SuppressWarnings("unchecked") - private static void waitForMigrationCompletion(RestClient adminClient) throws Exception { - final Request request = new Request("GET", "_cluster/state/metadata/" + INTERNAL_SECURITY_MAIN_INDEX_7); - assertBusy(() -> { - Response response = adminClient.performRequest(request); - assertOK(response); - Map responseMap = responseAsMap(response); - Map indicesMetadataMap = (Map) ((Map) responseMap.get("metadata")).get( - "indices" - ); - assertTrue(indicesMetadataMap.containsKey(INTERNAL_SECURITY_MAIN_INDEX_7)); - assertTrue( - ((Map) indicesMetadataMap.get(INTERNAL_SECURITY_MAIN_INDEX_7)).containsKey(MIGRATION_VERSION_CUSTOM_KEY) - ); - }); - } - private void createRoleWithMetadata(String roleName, Map metadata) throws IOException { final Request request = new Request("POST", "/_security/role/" + roleName); BytesReference source = BytesReference.bytes( diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/operator_defined_role_mappings.json b/x-pack/qa/rolling-upgrade/src/test/resources/operator_defined_role_mappings.json new file mode 100644 index 0000000000000..d897cabb8ab01 --- /dev/null +++ b/x-pack/qa/rolling-upgrade/src/test/resources/operator_defined_role_mappings.json @@ -0,0 +1,38 @@ +{ + "metadata": { + "version": "2", + "compatibility": "8.4.0" + }, + "state": { + "role_mappings": { + "operator_role_mapping_1": { + "enabled": true, + "roles": [ + "kibana_user" + ], + "metadata": { + "from_file": true + }, + "rules": { + "field": { + "username": "role-mapping-test-user" + } + } + }, + "operator_role_mapping_2": { + "enabled": true, + "roles": [ + "fleet_user" + ], + "metadata": { + "from_file": true + }, + "rules": { + "field": { + "username": "role-mapping-test-user" + } + } + } + } + } +} From f395f113e9fdf688f11dd9d272496913986b87fc Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 29 Oct 2024 11:36:35 +0000 Subject: [PATCH 04/18] Increase minimum threshold in shard balancer (#115831) Support for thresholds between 0.0 and 1.0 was deprecated in #92100. This commit removes this support in 9.0. --- docs/changelog/115831.yaml | 13 +++++++ .../upgrades/FullClusterRestartIT.java | 35 +++++++++++++++++++ .../allocator/BalancedShardsAllocator.java | 31 ++-------------- .../allocation/BalancedSingleShardTests.java | 19 +++------- .../BalancedShardsAllocatorTests.java | 11 ++---- 5 files changed, 58 insertions(+), 51 deletions(-) create mode 100644 docs/changelog/115831.yaml diff --git a/docs/changelog/115831.yaml b/docs/changelog/115831.yaml new file mode 100644 index 0000000000000..18442ec3b97e6 --- /dev/null +++ b/docs/changelog/115831.yaml @@ -0,0 +1,13 @@ +pr: 115831 +summary: Increase minimum threshold in shard balancer +area: Allocation +type: breaking +issues: [] +breaking: + title: Minimum shard balancer threshold is now 1.0 + area: Cluster and node setting + details: >- + Earlier versions of {es} accepted any non-negative value for `cluster.routing.allocation.balance.threshold`, but values smaller than + `1.0` do not make sense and have been ignored since version 8.6.1. From 9.0.0 these nonsensical values are now forbidden. + impact: Do not set `cluster.routing.allocation.balance.threshold` to a value less than `1.0`. + notable: false diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 92a704f793dc2..fcca3f9a4700c 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -16,9 +16,11 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.Build; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.WarningsHandler; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.common.Strings; @@ -27,6 +29,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Booleans; import org.elasticsearch.core.CheckedFunction; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -72,6 +75,7 @@ import static java.util.stream.Collectors.toList; import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SYSTEM_INDEX_ENFORCEMENT_INDEX_VERSION; import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING; +import static org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator.THRESHOLD_SETTING; import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; @@ -1949,4 +1953,35 @@ public static void assertNumHits(String index, int numHits, int totalShards) thr assertThat(XContentMapValues.extractValue("_shards.successful", resp), equalTo(totalShards)); assertThat(extractTotalHits(resp), equalTo(numHits)); } + + @UpdateForV10(owner = UpdateForV10.Owner.DISTRIBUTED_COORDINATION) // this test is just about v8->v9 upgrades, remove it in v10 + public void testBalancedShardsAllocatorThreshold() throws Exception { + assumeTrue("test only applies for v8->v9 upgrades", getOldClusterTestVersion().getMajor() == 8); + + final var chosenValue = randomFrom("0", "0.1", "0.5", "0.999"); + + if (isRunningAgainstOldCluster()) { + final var request = newXContentRequest( + HttpMethod.PUT, + "/_cluster/settings", + (builder, params) -> builder.startObject("persistent").field(THRESHOLD_SETTING.getKey(), chosenValue).endObject() + ); + request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE)); + assertOK(client().performRequest(request)); + } + + final var clusterSettingsResponse = ObjectPath.createFromResponse( + client().performRequest(new Request("GET", "/_cluster/settings")) + ); + + final var settingsPath = "persistent." + THRESHOLD_SETTING.getKey(); + final var settingValue = clusterSettingsResponse.evaluate(settingsPath); + + if (isRunningAgainstOldCluster()) { + assertEquals(chosenValue, settingValue); + } else { + assertNull(settingValue); + assertNotNull(clusterSettingsResponse.evaluate("persistent.archived." + THRESHOLD_SETTING.getKey())); + } + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java index 840aa3a3c1d3f..108bb83d90871 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java @@ -32,8 +32,6 @@ import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.cluster.routing.allocation.decider.Decision; import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type; -import org.elasticsearch.common.logging.DeprecationCategory; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; @@ -41,7 +39,6 @@ import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.Tuple; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.gateway.PriorityComparator; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.injection.guice.Inject; @@ -109,7 +106,7 @@ public class BalancedShardsAllocator implements ShardsAllocator { public static final Setting THRESHOLD_SETTING = Setting.floatSetting( "cluster.routing.allocation.balance.threshold", 1.0f, - 0.0f, + 1.0f, Property.Dynamic, Property.NodeScope ); @@ -140,34 +137,10 @@ public BalancedShardsAllocator(ClusterSettings clusterSettings, WriteLoadForecas clusterSettings.initializeAndWatch(INDEX_BALANCE_FACTOR_SETTING, value -> this.indexBalanceFactor = value); clusterSettings.initializeAndWatch(WRITE_LOAD_BALANCE_FACTOR_SETTING, value -> this.writeLoadBalanceFactor = value); clusterSettings.initializeAndWatch(DISK_USAGE_BALANCE_FACTOR_SETTING, value -> this.diskUsageBalanceFactor = value); - clusterSettings.initializeAndWatch(THRESHOLD_SETTING, value -> this.threshold = ensureValidThreshold(value)); + clusterSettings.initializeAndWatch(THRESHOLD_SETTING, value -> this.threshold = value); this.writeLoadForecaster = writeLoadForecaster; } - /** - * Clamp threshold to be at least 1, and log a critical deprecation warning if smaller values are given. - * - * Once {@link org.elasticsearch.Version#V_7_17_0} goes out of scope, start to properly reject such bad values. - */ - @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) - private static float ensureValidThreshold(float threshold) { - if (1.0f <= threshold) { - return threshold; - } else { - DeprecationLogger.getLogger(BalancedShardsAllocator.class) - .critical( - DeprecationCategory.SETTINGS, - "balance_threshold_too_small", - "ignoring value [{}] for [{}] since it is smaller than 1.0; " - + "setting [{}] to a value smaller than 1.0 will be forbidden in a future release", - threshold, - THRESHOLD_SETTING.getKey(), - THRESHOLD_SETTING.getKey() - ); - return 1.0f; - } - } - @Override public void allocate(RoutingAllocation allocation) { assert allocation.ignoreDisable() == false; diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java index 41207a2d968b8..9a769567bee1c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java @@ -246,7 +246,7 @@ public void testNodeDecisionsRanking() { // return the same ranking as the current node ClusterState clusterState = ClusterStateCreationUtils.state(randomIntBetween(1, 10), new String[] { "idx" }, 1); ShardRouting shardToRebalance = clusterState.routingTable().index("idx").shardsWithState(ShardRoutingState.STARTED).get(0); - MoveDecision decision = executeRebalanceFor(shardToRebalance, clusterState, emptySet(), -1); + MoveDecision decision = executeRebalanceFor(shardToRebalance, clusterState, emptySet()); int currentRanking = decision.getCurrentNodeRanking(); assertEquals(1, currentRanking); for (NodeAllocationResult result : decision.getNodeDecisions()) { @@ -258,7 +258,7 @@ public void testNodeDecisionsRanking() { clusterState = ClusterStateCreationUtils.state(1, new String[] { "idx" }, randomIntBetween(2, 10)); shardToRebalance = clusterState.routingTable().index("idx").shardsWithState(ShardRoutingState.STARTED).get(0); clusterState = addNodesToClusterState(clusterState, randomIntBetween(1, 10)); - decision = executeRebalanceFor(shardToRebalance, clusterState, emptySet(), 0.01f); + decision = executeRebalanceFor(shardToRebalance, clusterState, emptySet()); for (NodeAllocationResult result : decision.getNodeDecisions()) { assertThat(result.getWeightRanking(), lessThan(decision.getCurrentNodeRanking())); } @@ -285,7 +285,7 @@ public void testNodeDecisionsRanking() { } } clusterState = addNodesToClusterState(clusterState, 1); - decision = executeRebalanceFor(shardToRebalance, clusterState, emptySet(), 0.01f); + decision = executeRebalanceFor(shardToRebalance, clusterState, emptySet()); for (NodeAllocationResult result : decision.getNodeDecisions()) { if (result.getWeightRanking() < decision.getCurrentNodeRanking()) { // highest ranked node should not be any of the initial nodes @@ -298,22 +298,13 @@ public void testNodeDecisionsRanking() { assertTrue(nodesWithTwoShards.contains(result.getNode().getId())); } } - - assertCriticalWarnings(""" - ignoring value [0.01] for [cluster.routing.allocation.balance.threshold] since it is smaller than 1.0; setting \ - [cluster.routing.allocation.balance.threshold] to a value smaller than 1.0 will be forbidden in a future release"""); } private MoveDecision executeRebalanceFor( final ShardRouting shardRouting, final ClusterState clusterState, - final Set noDecisionNodes, - final float threshold + final Set noDecisionNodes ) { - Settings settings = Settings.EMPTY; - if (Float.compare(-1.0f, threshold) != 0) { - settings = Settings.builder().put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), threshold).build(); - } AllocationDecider allocationDecider = new AllocationDecider() { @Override public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { @@ -329,7 +320,7 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca return Decision.YES; } }; - BalancedShardsAllocator allocator = new BalancedShardsAllocator(settings); + BalancedShardsAllocator allocator = new BalancedShardsAllocator(Settings.EMPTY); RoutingAllocation routingAllocation = newRoutingAllocation( new AllocationDeciders(Arrays.asList(allocationDecider, rebalanceDecider)), clusterState diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocatorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocatorTests.java index 8392b6fe3e148..98c3451329f52 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocatorTests.java @@ -441,15 +441,10 @@ public void testGetIndexDiskUsageInBytes() { public void testThresholdLimit() { final var badValue = (float) randomDoubleBetween(0.0, Math.nextDown(1.0f), true); - assertEquals( - 1.0f, - new BalancedShardsAllocator(Settings.builder().put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), badValue).build()) - .getThreshold(), - 0.0f + expectThrows( + IllegalArgumentException.class, + () -> new BalancedShardsAllocator(Settings.builder().put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), badValue).build()) ); - assertCriticalWarnings("ignoring value [" + badValue + """ - ] for [cluster.routing.allocation.balance.threshold] since it is smaller than 1.0; setting \ - [cluster.routing.allocation.balance.threshold] to a value smaller than 1.0 will be forbidden in a future release"""); final var goodValue = (float) randomDoubleBetween(1.0, 10.0, true); assertEquals( From d18824d4e6bd0beb2e8e5ec854caef64a31594b6 Mon Sep 17 00:00:00 2001 From: Jan Kuipers <148754765+jan-elastic@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:47:32 +0100 Subject: [PATCH 05/18] Set assignment state to "started" in case of zero allocations (#115824) --- .../core/ml/inference/assignment/TrainedModelAssignment.java | 3 +++ .../ml/inference/assignment/TrainedModelAssignmentTests.java | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java index 06c3f75587d62..efd07cceae09b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java @@ -533,6 +533,9 @@ public AssignmentState calculateAssignmentState() { if (assignmentState.equals(AssignmentState.STOPPING)) { return assignmentState; } + if (taskParams.getNumberOfAllocations() == 0) { + return AssignmentState.STARTED; + } if (nodeRoutingTable.values().stream().anyMatch(r -> r.getState().equals(RoutingState.STARTED))) { return AssignmentState.STARTED; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java index c3b6e0089b4ae..dc0a8b52e585a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java @@ -172,6 +172,11 @@ public void testCalculateAssignmentState_GivenNoStartedAssignments() { assertThat(builder.calculateAssignmentState(), equalTo(AssignmentState.STARTING)); } + public void testCalculateAssignmentState_GivenNumAllocationsIsZero() { + TrainedModelAssignment.Builder builder = TrainedModelAssignment.Builder.empty(randomTaskParams(0), null); + assertThat(builder.calculateAssignmentState(), equalTo(AssignmentState.STARTED)); + } + public void testCalculateAssignmentState_GivenOneStartedAssignment() { TrainedModelAssignment.Builder builder = TrainedModelAssignment.Builder.empty(randomTaskParams(5), null); builder.addRoutingEntry("node-1", new RoutingInfo(4, 4, RoutingState.STARTING, "")); From 6742147d6ada3af42ff73f03eb45fd2486cb64cc Mon Sep 17 00:00:00 2001 From: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com> Date: Tue, 29 Oct 2024 08:59:19 -0400 Subject: [PATCH 06/18] [Inference API] Improve chunked results error message (#115807) * Improve chunked results error message * Update RestStatus to conflict * precommit * Update docs/changelog/115807.yaml --- docs/changelog/115807.yaml | 5 +++++ .../xpack/core/inference/results/ResultUtils.java | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/115807.yaml diff --git a/docs/changelog/115807.yaml b/docs/changelog/115807.yaml new file mode 100644 index 0000000000000..d17cabca4bd03 --- /dev/null +++ b/docs/changelog/115807.yaml @@ -0,0 +1,5 @@ +pr: 115807 +summary: "[Inference API] Improve chunked results error message" +area: Machine Learning +type: enhancement +issues: [] diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ResultUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ResultUtils.java index 4fe2c9ae486f1..eb68af7589717 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ResultUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ResultUtils.java @@ -14,8 +14,9 @@ public class ResultUtils { public static ElasticsearchStatusException createInvalidChunkedResultException(String expectedResultName, String receivedResultName) { return new ElasticsearchStatusException( - "Expected a chunked inference [{}] received [{}]", - RestStatus.INTERNAL_SERVER_ERROR, + "Received incompatible results. Check that your model_id matches the task_type of this endpoint. " + + "Expected chunked output of type [{}] but received [{}].", + RestStatus.CONFLICT, expectedResultName, receivedResultName ); From 78a531bf4eed313d44b80638ddf015cc586ee2b6 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 29 Oct 2024 14:11:14 +0100 Subject: [PATCH 07/18] Catch and handle disconnect exceptions in search (#115836) Getting a connection can throw an exception for a disconnected node. We failed to handle these in the adjusted spots, leading to a phase failure (and possible memory leaks for outstanding operations) instead of correctly recording a per-shard failure. --- docs/changelog/115836.yaml | 5 ++ .../action/search/DfsQueryPhase.java | 32 +++++++--- .../action/search/FetchSearchPhase.java | 61 +++++++++++-------- .../action/search/RankFeaturePhase.java | 55 ++++++++++------- .../SearchDfsQueryThenFetchAsyncAction.java | 14 +++-- .../SearchQueryThenFetchAsyncAction.java | 9 ++- 6 files changed, 111 insertions(+), 65 deletions(-) create mode 100644 docs/changelog/115836.yaml diff --git a/docs/changelog/115836.yaml b/docs/changelog/115836.yaml new file mode 100644 index 0000000000000..f6da638f1feff --- /dev/null +++ b/docs/changelog/115836.yaml @@ -0,0 +1,5 @@ +pr: 115836 +summary: Catch and handle disconnect exceptions in search +area: Search +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java index e0e240be0377a..93c8d66447e34 100644 --- a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java @@ -84,15 +84,20 @@ public void run() { for (final DfsSearchResult dfsResult : searchResults) { final SearchShardTarget shardTarget = dfsResult.getSearchShardTarget(); - Transport.Connection connection = context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId()); - ShardSearchRequest shardRequest = rewriteShardSearchRequest(dfsResult.getShardSearchRequest()); + final int shardIndex = dfsResult.getShardIndex(); QuerySearchRequest querySearchRequest = new QuerySearchRequest( - context.getOriginalIndices(dfsResult.getShardIndex()), + context.getOriginalIndices(shardIndex), dfsResult.getContextId(), - shardRequest, + rewriteShardSearchRequest(dfsResult.getShardSearchRequest()), dfs ); - final int shardIndex = dfsResult.getShardIndex(); + final Transport.Connection connection; + try { + connection = context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId()); + } catch (Exception e) { + shardFailure(e, querySearchRequest, shardIndex, shardTarget, counter); + return; + } searchTransportService.sendExecuteQuery( connection, querySearchRequest, @@ -112,10 +117,7 @@ protected void innerOnResponse(QuerySearchResult response) { @Override public void onFailure(Exception exception) { try { - context.getLogger() - .debug(() -> "[" + querySearchRequest.contextId() + "] Failed to execute query phase", exception); - progressListener.notifyQueryFailure(shardIndex, shardTarget, exception); - counter.onFailure(shardIndex, shardTarget, exception); + shardFailure(exception, querySearchRequest, shardIndex, shardTarget, counter); } finally { if (context.isPartOfPointInTime(querySearchRequest.contextId()) == false) { // the query might not have been executed at all (for example because thread pool rejected @@ -134,6 +136,18 @@ public void onFailure(Exception exception) { } } + private void shardFailure( + Exception exception, + QuerySearchRequest querySearchRequest, + int shardIndex, + SearchShardTarget shardTarget, + CountedCollector counter + ) { + context.getLogger().debug(() -> "[" + querySearchRequest.contextId() + "] Failed to execute query phase", exception); + progressListener.notifyQueryFailure(shardIndex, shardTarget, exception); + counter.onFailure(shardIndex, shardTarget, exception); + } + // package private for testing ShardSearchRequest rewriteShardSearchRequest(ShardSearchRequest request) { SearchSourceBuilder source = request.source(); diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java index 99b24bd483fb4..29aba0eee1f55 100644 --- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java @@ -21,6 +21,7 @@ import org.elasticsearch.search.internal.ShardSearchContextId; import org.elasticsearch.search.rank.RankDoc; import org.elasticsearch.search.rank.RankDocShardInfo; +import org.elasticsearch.transport.Transport; import java.util.ArrayList; import java.util.HashMap; @@ -214,9 +215,41 @@ private void executeFetch( final ShardSearchContextId contextId = shardPhaseResult.queryResult() != null ? shardPhaseResult.queryResult().getContextId() : shardPhaseResult.rankFeatureResult().getContextId(); + var listener = new SearchActionListener(shardTarget, shardIndex) { + @Override + public void innerOnResponse(FetchSearchResult result) { + try { + progressListener.notifyFetchResult(shardIndex); + counter.onResult(result); + } catch (Exception e) { + context.onPhaseFailure(FetchSearchPhase.this, "", e); + } + } + + @Override + public void onFailure(Exception e) { + try { + logger.debug(() -> "[" + contextId + "] Failed to execute fetch phase", e); + progressListener.notifyFetchFailure(shardIndex, shardTarget, e); + counter.onFailure(shardIndex, shardTarget, e); + } finally { + // the search context might not be cleared on the node where the fetch was executed for example + // because the action was rejected by the thread pool. in this case we need to send a dedicated + // request to clear the search context. + releaseIrrelevantSearchContext(shardPhaseResult, context); + } + } + }; + final Transport.Connection connection; + try { + connection = context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId()); + } catch (Exception e) { + listener.onFailure(e); + return; + } context.getSearchTransport() .sendExecuteFetch( - context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId()), + connection, new ShardFetchSearchRequest( context.getOriginalIndices(shardPhaseResult.getShardIndex()), contextId, @@ -228,31 +261,7 @@ private void executeFetch( aggregatedDfs ), context.getTask(), - new SearchActionListener<>(shardTarget, shardIndex) { - @Override - public void innerOnResponse(FetchSearchResult result) { - try { - progressListener.notifyFetchResult(shardIndex); - counter.onResult(result); - } catch (Exception e) { - context.onPhaseFailure(FetchSearchPhase.this, "", e); - } - } - - @Override - public void onFailure(Exception e) { - try { - logger.debug(() -> "[" + contextId + "] Failed to execute fetch phase", e); - progressListener.notifyFetchFailure(shardIndex, shardTarget, e); - counter.onFailure(shardIndex, shardTarget, e); - } finally { - // the search context might not be cleared on the node where the fetch was executed for example - // because the action was rejected by the thread pool. in this case we need to send a dedicated - // request to clear the search context. - releaseIrrelevantSearchContext(shardPhaseResult, context); - } - } - } + listener ); } diff --git a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java index dd3c28bba0fce..e37d6d1729f9f 100644 --- a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java @@ -24,6 +24,7 @@ import org.elasticsearch.search.rank.feature.RankFeatureDoc; import org.elasticsearch.search.rank.feature.RankFeatureResult; import org.elasticsearch.search.rank.feature.RankFeatureShardRequest; +import org.elasticsearch.transport.Transport; import java.util.List; @@ -131,9 +132,38 @@ private void executeRankFeatureShardPhase( final SearchShardTarget shardTarget = queryResult.queryResult().getSearchShardTarget(); final ShardSearchContextId contextId = queryResult.queryResult().getContextId(); final int shardIndex = queryResult.getShardIndex(); + var listener = new SearchActionListener(shardTarget, shardIndex) { + @Override + protected void innerOnResponse(RankFeatureResult response) { + try { + progressListener.notifyRankFeatureResult(shardIndex); + rankRequestCounter.onResult(response); + } catch (Exception e) { + context.onPhaseFailure(RankFeaturePhase.this, "", e); + } + } + + @Override + public void onFailure(Exception e) { + try { + logger.debug(() -> "[" + contextId + "] Failed to execute rank phase", e); + progressListener.notifyRankFeatureFailure(shardIndex, shardTarget, e); + rankRequestCounter.onFailure(shardIndex, shardTarget, e); + } finally { + releaseIrrelevantSearchContext(queryResult, context); + } + } + }; + final Transport.Connection connection; + try { + connection = context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId()); + } catch (Exception e) { + listener.onFailure(e); + return; + } context.getSearchTransport() .sendExecuteRankFeature( - context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId()), + connection, new RankFeatureShardRequest( context.getOriginalIndices(queryResult.getShardIndex()), queryResult.getContextId(), @@ -141,28 +171,7 @@ private void executeRankFeatureShardPhase( entry ), context.getTask(), - new SearchActionListener<>(shardTarget, shardIndex) { - @Override - protected void innerOnResponse(RankFeatureResult response) { - try { - progressListener.notifyRankFeatureResult(shardIndex); - rankRequestCounter.onResult(response); - } catch (Exception e) { - context.onPhaseFailure(RankFeaturePhase.this, "", e); - } - } - - @Override - public void onFailure(Exception e) { - try { - logger.debug(() -> "[" + contextId + "] Failed to execute rank phase", e); - progressListener.notifyRankFeatureFailure(shardIndex, shardTarget, e); - rankRequestCounter.onFailure(shardIndex, shardTarget, e); - } finally { - releaseIrrelevantSearchContext(queryResult, context); - } - } - } + listener ); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java index 5b7ee04d020fc..26eb266cd457e 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java @@ -87,12 +87,14 @@ protected void executePhaseOnShard( final SearchShardTarget shard, final SearchActionListener listener ) { - getSearchTransport().sendExecuteDfs( - getConnection(shard.getClusterAlias(), shard.getNodeId()), - buildShardSearchRequest(shardIt, listener.requestIndex), - getTask(), - listener - ); + final Transport.Connection connection; + try { + connection = getConnection(shard.getClusterAlias(), shard.getNodeId()); + } catch (Exception e) { + listener.onFailure(e); + return; + } + getSearchTransport().sendExecuteDfs(connection, buildShardSearchRequest(shardIt, listener.requestIndex), getTask(), listener); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java index e0ad4691fa991..33b2cdf74cd79 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java @@ -94,8 +94,15 @@ protected void executePhaseOnShard( final SearchShardTarget shard, final SearchActionListener listener ) { + final Transport.Connection connection; + try { + connection = getConnection(shard.getClusterAlias(), shard.getNodeId()); + } catch (Exception e) { + listener.onFailure(e); + return; + } ShardSearchRequest request = rewriteShardSearchRequest(super.buildShardSearchRequest(shardIt, listener.requestIndex)); - getSearchTransport().sendExecuteQuery(getConnection(shard.getClusterAlias(), shard.getNodeId()), request, getTask(), listener); + getSearchTransport().sendExecuteQuery(connection, request, getTask(), listener); } @Override From 61829213cf34fc307395d1f1475f652427db6518 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 29 Oct 2024 17:14:50 +0100 Subject: [PATCH 08/18] Tweak Logsdb* and TsdbIndexingRollingUpgradeIT (#115850) Adjust assertion to more losely detect an error that can be ignored. Closes #115817 --- .../upgrades/LogsdbIndexingRollingUpgradeIT.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsdbIndexingRollingUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsdbIndexingRollingUpgradeIT.java index 226cb3dda2ba1..9cb91438e09c0 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsdbIndexingRollingUpgradeIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsdbIndexingRollingUpgradeIT.java @@ -28,10 +28,7 @@ import static org.elasticsearch.upgrades.LogsIndexModeRollingUpgradeIT.enableLogsdbByDefault; import static org.elasticsearch.upgrades.LogsIndexModeRollingUpgradeIT.getWriteBackingIndex; import static org.elasticsearch.upgrades.TsdbIT.formatInstant; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.*; public class LogsdbIndexingRollingUpgradeIT extends AbstractRollingUpgradeTestCase { @@ -240,7 +237,7 @@ protected static void startTrial() throws IOException { } catch (ResponseException e) { var responseBody = entityAsMap(e.getResponse()); String error = ObjectPath.evaluate(responseBody, "error_message"); - assertThat(error, equalTo("Trial was already activated.")); + assertThat(error, containsString("Trial was already activated.")); } } From 23e1116adb1b266157b423d36fb05799a62b79ba Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 29 Oct 2024 10:57:12 -0600 Subject: [PATCH 09/18] Ensure thread context set for streaming (#115683) Currently the thread context is lost between streaming context switches. This commit ensures that each time the thread context is properly set before providing new data to the stream. --- .../netty4/Netty4HttpPipeliningHandler.java | 5 +- .../netty4/Netty4HttpRequestBodyStream.java | 33 ++++--- .../Netty4HttpRequestBodyStreamTests.java | 69 ++++++++++++- .../action/bulk/IncrementalBulkService.java | 99 ++++++++----------- .../elasticsearch/node/NodeConstruction.java | 6 +- .../elasticsearch/rest/BaseRestHandler.java | 1 + .../rest/action/document/RestBulkAction.java | 3 +- .../action/ActionModuleTests.java | 10 +- .../AbstractHttpServerTransportTests.java | 2 +- .../action/document/RestBulkActionTests.java | 14 ++- .../xpack/security/SecurityTests.java | 2 +- 11 files changed, 148 insertions(+), 96 deletions(-) diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java index b08c93a4dc240..1a391a05add58 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java @@ -137,7 +137,10 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { netty4HttpRequest = new Netty4HttpRequest(readSequence++, fullHttpRequest); currentRequestStream = null; } else { - var contentStream = new Netty4HttpRequestBodyStream(ctx.channel()); + var contentStream = new Netty4HttpRequestBodyStream( + ctx.channel(), + serverTransport.getThreadPool().getThreadContext() + ); currentRequestStream = contentStream; netty4HttpRequest = new Netty4HttpRequest(readSequence++, request, contentStream); } diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java index 9a0dc09b7566c..238faa7a9237e 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java @@ -16,6 +16,7 @@ import io.netty.handler.codec.http.HttpContent; import io.netty.handler.codec.http.LastHttpContent; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.Releasables; import org.elasticsearch.http.HttpBody; import org.elasticsearch.transport.netty4.Netty4Utils; @@ -34,14 +35,18 @@ public class Netty4HttpRequestBodyStream implements HttpBody.Stream { private final Channel channel; private final ChannelFutureListener closeListener = future -> doClose(); private final List tracingHandlers = new ArrayList<>(4); + private final ThreadContext threadContext; private ByteBuf buf; private boolean hasLast = false; private boolean requested = false; private boolean closing = false; private HttpBody.ChunkHandler handler; + private ThreadContext.StoredContext requestContext; - public Netty4HttpRequestBodyStream(Channel channel) { + public Netty4HttpRequestBodyStream(Channel channel, ThreadContext threadContext) { this.channel = channel; + this.threadContext = threadContext; + this.requestContext = threadContext.newStoredContext(); Netty4Utils.addListener(channel.closeFuture(), closeListener); channel.config().setAutoRead(false); } @@ -66,6 +71,7 @@ public void addTracingHandler(ChunkHandler chunkHandler) { public void next() { assert closing == false : "cannot request next chunk on closing stream"; assert handler != null : "handler must be set before requesting next chunk"; + requestContext = threadContext.newStoredContext(); channel.eventLoop().submit(() -> { requested = true; if (buf == null) { @@ -108,11 +114,6 @@ private void addChunk(ByteBuf chunk) { } } - // visible for test - Channel channel() { - return channel; - } - // visible for test ByteBuf buf() { return buf; @@ -129,10 +130,12 @@ private void send() { var bytesRef = Netty4Utils.toReleasableBytesReference(buf); requested = false; buf = null; - for (var tracer : tracingHandlers) { - tracer.onNext(bytesRef, hasLast); + try (var ignored = threadContext.restoreExistingContext(requestContext)) { + for (var tracer : tracingHandlers) { + tracer.onNext(bytesRef, hasLast); + } + handler.onNext(bytesRef, hasLast); } - handler.onNext(bytesRef, hasLast); if (hasLast) { channel.config().setAutoRead(true); channel.closeFuture().removeListener(closeListener); @@ -150,11 +153,13 @@ public void close() { private void doClose() { closing = true; - for (var tracer : tracingHandlers) { - Releasables.closeExpectNoException(tracer); - } - if (handler != null) { - handler.close(); + try (var ignored = threadContext.restoreExistingContext(requestContext)) { + for (var tracer : tracingHandlers) { + Releasables.closeExpectNoException(tracer); + } + if (handler != null) { + handler.close(); + } } if (buf != null) { buf.release(); diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java index f495883631a4e..5ff5a27e2d551 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java @@ -19,24 +19,33 @@ import io.netty.handler.flow.FlowControlHandler; import org.elasticsearch.common.bytes.ReleasableBytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.http.HttpBody; import org.elasticsearch.test.ESTestCase; import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.hamcrest.Matchers.hasEntry; public class Netty4HttpRequestBodyStreamTests extends ESTestCase { - EmbeddedChannel channel; - Netty4HttpRequestBodyStream stream; + private final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + private EmbeddedChannel channel; + private Netty4HttpRequestBodyStream stream; static HttpBody.ChunkHandler discardHandler = (chunk, isLast) -> chunk.close(); @Override public void setUp() throws Exception { super.setUp(); channel = new EmbeddedChannel(); - stream = new Netty4HttpRequestBodyStream(channel); + threadContext.putHeader("header1", "value1"); + stream = new Netty4HttpRequestBodyStream(channel, threadContext); stream.setHandler(discardHandler); // set default handler, each test might override one channel.pipeline().addLast(new SimpleChannelInboundHandler(false) { @Override @@ -118,6 +127,60 @@ public void testReadFromChannel() { assertTrue("should receive last content", gotLast.get()); } + public void testReadFromHasCorrectThreadContext() throws InterruptedException { + var gotLast = new AtomicBoolean(false); + AtomicReference> headers = new AtomicReference<>(); + stream.setHandler(new HttpBody.ChunkHandler() { + @Override + public void onNext(ReleasableBytesReference chunk, boolean isLast) { + headers.set(threadContext.getHeaders()); + gotLast.set(isLast); + chunk.close(); + } + + @Override + public void close() { + headers.set(threadContext.getHeaders()); + } + }); + channel.pipeline().addFirst(new FlowControlHandler()); // block all incoming messages, need explicit channel.read() + var chunkSize = 1024; + + channel.writeInbound(randomContent(chunkSize)); + channel.writeInbound(randomLastContent(chunkSize)); + + threadContext.putHeader("header2", "value2"); + stream.next(); + + Thread thread = new Thread(() -> channel.runPendingTasks()); + thread.start(); + thread.join(); + + assertThat(headers.get(), hasEntry("header1", "value1")); + assertThat(headers.get(), hasEntry("header2", "value2")); + + threadContext.putHeader("header3", "value3"); + stream.next(); + + thread = new Thread(() -> channel.runPendingTasks()); + thread.start(); + thread.join(); + + assertThat(headers.get(), hasEntry("header1", "value1")); + assertThat(headers.get(), hasEntry("header2", "value2")); + assertThat(headers.get(), hasEntry("header3", "value3")); + + assertTrue("should receive last content", gotLast.get()); + + headers.set(new HashMap<>()); + + stream.close(); + + assertThat(headers.get(), hasEntry("header1", "value1")); + assertThat(headers.get(), hasEntry("header2", "value2")); + assertThat(headers.get(), hasEntry("header3", "value3")); + } + HttpContent randomContent(int size, boolean isLast) { var buf = Unpooled.wrappedBuffer(randomByteArrayOfLength(size)); if (isLast) { diff --git a/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java b/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java index 2e7c87301b2f6..6ce198260ba3c 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; -import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; @@ -43,12 +42,10 @@ public class IncrementalBulkService { private final Client client; private final AtomicBoolean enabledForTests = new AtomicBoolean(true); private final IndexingPressure indexingPressure; - private final ThreadContext threadContext; - public IncrementalBulkService(Client client, IndexingPressure indexingPressure, ThreadContext threadContext) { + public IncrementalBulkService(Client client, IndexingPressure indexingPressure) { this.client = client; this.indexingPressure = indexingPressure; - this.threadContext = threadContext; } public Handler newBulkRequest() { @@ -58,7 +55,7 @@ public Handler newBulkRequest() { public Handler newBulkRequest(@Nullable String waitForActiveShards, @Nullable TimeValue timeout, @Nullable String refresh) { ensureEnabled(); - return new Handler(client, threadContext, indexingPressure, waitForActiveShards, timeout, refresh); + return new Handler(client, indexingPressure, waitForActiveShards, timeout, refresh); } private void ensureEnabled() { @@ -94,7 +91,6 @@ public static class Handler implements Releasable { public static final BulkRequest.IncrementalState EMPTY_STATE = new BulkRequest.IncrementalState(Collections.emptyMap(), true); private final Client client; - private final ThreadContext threadContext; private final IndexingPressure indexingPressure; private final ActiveShardCount waitForActiveShards; private final TimeValue timeout; @@ -106,22 +102,18 @@ public static class Handler implements Releasable { private boolean globalFailure = false; private boolean incrementalRequestSubmitted = false; private boolean bulkInProgress = false; - private ThreadContext.StoredContext requestContext; private Exception bulkActionLevelFailure = null; private long currentBulkSize = 0L; private BulkRequest bulkRequest = null; protected Handler( Client client, - ThreadContext threadContext, IndexingPressure indexingPressure, @Nullable String waitForActiveShards, @Nullable TimeValue timeout, @Nullable String refresh ) { this.client = client; - this.threadContext = threadContext; - this.requestContext = threadContext.newStoredContext(); this.indexingPressure = indexingPressure; this.waitForActiveShards = waitForActiveShards != null ? ActiveShardCount.parseString(waitForActiveShards) : null; this.timeout = timeout; @@ -141,31 +133,28 @@ public void addItems(List> items, Releasable releasable, Runn if (shouldBackOff()) { final boolean isFirstRequest = incrementalRequestSubmitted == false; incrementalRequestSubmitted = true; - try (var ignored = threadContext.restoreExistingContext(requestContext)) { - final ArrayList toRelease = new ArrayList<>(releasables); - releasables.clear(); - bulkInProgress = true; - client.bulk(bulkRequest, ActionListener.runAfter(new ActionListener<>() { - - @Override - public void onResponse(BulkResponse bulkResponse) { - handleBulkSuccess(bulkResponse); - createNewBulkRequest( - new BulkRequest.IncrementalState(bulkResponse.getIncrementalState().shardLevelFailures(), true) - ); - } - - @Override - public void onFailure(Exception e) { - handleBulkFailure(isFirstRequest, e); - } - }, () -> { - bulkInProgress = false; - requestContext = threadContext.newStoredContext(); - toRelease.forEach(Releasable::close); - nextItems.run(); - })); - } + final ArrayList toRelease = new ArrayList<>(releasables); + releasables.clear(); + bulkInProgress = true; + client.bulk(bulkRequest, ActionListener.runAfter(new ActionListener<>() { + + @Override + public void onResponse(BulkResponse bulkResponse) { + handleBulkSuccess(bulkResponse); + createNewBulkRequest( + new BulkRequest.IncrementalState(bulkResponse.getIncrementalState().shardLevelFailures(), true) + ); + } + + @Override + public void onFailure(Exception e) { + handleBulkFailure(isFirstRequest, e); + } + }, () -> { + bulkInProgress = false; + toRelease.forEach(Releasable::close); + nextItems.run(); + })); } else { nextItems.run(); } @@ -187,28 +176,26 @@ public void lastItems(List> items, Releasable releasable, Act } else { assert bulkRequest != null; if (internalAddItems(items, releasable)) { - try (var ignored = threadContext.restoreExistingContext(requestContext)) { - final ArrayList toRelease = new ArrayList<>(releasables); - releasables.clear(); - // We do not need to set this back to false as this will be the last request. - bulkInProgress = true; - client.bulk(bulkRequest, ActionListener.runBefore(new ActionListener<>() { - - private final boolean isFirstRequest = incrementalRequestSubmitted == false; - - @Override - public void onResponse(BulkResponse bulkResponse) { - handleBulkSuccess(bulkResponse); - listener.onResponse(combineResponses()); - } + final ArrayList toRelease = new ArrayList<>(releasables); + releasables.clear(); + // We do not need to set this back to false as this will be the last request. + bulkInProgress = true; + client.bulk(bulkRequest, ActionListener.runBefore(new ActionListener<>() { + + private final boolean isFirstRequest = incrementalRequestSubmitted == false; + + @Override + public void onResponse(BulkResponse bulkResponse) { + handleBulkSuccess(bulkResponse); + listener.onResponse(combineResponses()); + } - @Override - public void onFailure(Exception e) { - handleBulkFailure(isFirstRequest, e); - errorResponse(listener); - } - }, () -> toRelease.forEach(Releasable::close))); - } + @Override + public void onFailure(Exception e) { + handleBulkFailure(isFirstRequest, e); + errorResponse(listener); + } + }, () -> toRelease.forEach(Releasable::close))); } else { errorResponse(listener); } diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index 0a88a202ac8d3..5354b1097326b 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -915,11 +915,7 @@ private void construct( terminationHandler = getSinglePlugin(terminationHandlers, TerminationHandler.class).orElse(null); final IndexingPressure indexingLimits = new IndexingPressure(settings); - final IncrementalBulkService incrementalBulkService = new IncrementalBulkService( - client, - indexingLimits, - threadPool.getThreadContext() - ); + final IncrementalBulkService incrementalBulkService = new IncrementalBulkService(client, indexingLimits); ActionModule actionModule = new ActionModule( settings, diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java index c8cf0bf93879b..f1b59ed14cefb 100644 --- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java @@ -125,6 +125,7 @@ public final void handleRequest(RestRequest request, RestChannel channel, NodeCl if (request.isStreamedContent()) { assert action instanceof RequestBodyChunkConsumer; var chunkConsumer = (RequestBodyChunkConsumer) action; + request.contentStream().setHandler(new HttpBody.ChunkHandler() { @Override public void onNext(ReleasableBytesReference chunk, boolean isLast) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java index 1e80e6de60d65..7b82481d3d283 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java @@ -173,8 +173,7 @@ static class ChunkHandler implements BaseRestHandler.RequestBodyChunkConsumer { this.defaultListExecutedPipelines = request.paramAsBoolean("list_executed_pipelines", false); this.defaultRequireAlias = request.paramAsBoolean(DocWriteRequest.REQUIRE_ALIAS, false); this.defaultRequireDataStream = request.paramAsBoolean(DocWriteRequest.REQUIRE_DATA_STREAM, false); - // TODO: Fix type deprecation logging - this.parser = new BulkRequestParser(false, request.getRestApiVersion()); + this.parser = new BulkRequestParser(true, request.getRestApiVersion()); this.handlerSupplier = handlerSupplier; } diff --git a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java index 871062a687429..8d3561f2179cd 100644 --- a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java +++ b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java @@ -131,7 +131,7 @@ public void testSetupRestHandlerContainsKnownBuiltin() { null, List.of(), RestExtension.allowAll(), - new IncrementalBulkService(null, null, new ThreadContext(Settings.EMPTY)) + new IncrementalBulkService(null, null) ); actionModule.initRestHandlers(null, null); // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail @@ -196,7 +196,7 @@ public String getName() { null, List.of(), RestExtension.allowAll(), - new IncrementalBulkService(null, null, new ThreadContext(Settings.EMPTY)) + new IncrementalBulkService(null, null) ); Exception e = expectThrows(IllegalArgumentException.class, () -> actionModule.initRestHandlers(null, null)); assertThat(e.getMessage(), startsWith("Cannot replace existing handler for [/_nodes] for method: GET")); @@ -254,7 +254,7 @@ public List getRestHandlers( null, List.of(), RestExtension.allowAll(), - new IncrementalBulkService(null, null, new ThreadContext(Settings.EMPTY)) + new IncrementalBulkService(null, null) ); actionModule.initRestHandlers(null, null); // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail @@ -305,7 +305,7 @@ public void test3rdPartyHandlerIsNotInstalled() { null, List.of(), RestExtension.allowAll(), - new IncrementalBulkService(null, null, new ThreadContext(Settings.EMPTY)) + new IncrementalBulkService(null, null) ) ); assertThat( @@ -347,7 +347,7 @@ public void test3rdPartyRestControllerIsNotInstalled() { null, List.of(), RestExtension.allowAll(), - new IncrementalBulkService(null, null, new ThreadContext(Settings.EMPTY)) + new IncrementalBulkService(null, null) ) ); assertThat( diff --git a/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java b/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java index 77133516f37d5..cf623e77f740a 100644 --- a/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java +++ b/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java @@ -1179,7 +1179,7 @@ public Collection getRestHeaders() { null, List.of(), RestExtension.allowAll(), - new IncrementalBulkService(null, null, new ThreadContext(Settings.EMPTY)) + new IncrementalBulkService(null, null) ); } diff --git a/server/src/test/java/org/elasticsearch/rest/action/document/RestBulkActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/document/RestBulkActionTests.java index d3cd6dd9ca420..25cfd1e56514c 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/document/RestBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/document/RestBulkActionTests.java @@ -20,8 +20,6 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.ReleasableBytesReference; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.Releasable; import org.elasticsearch.http.HttpBody; import org.elasticsearch.index.IndexVersion; @@ -67,7 +65,7 @@ public void bulk(BulkRequest request, ActionListener listener) { params.put("pipeline", "timestamps"); new RestBulkAction( settings(IndexVersion.current()).build(), - new IncrementalBulkService(mock(Client.class), mock(IndexingPressure.class), new ThreadContext(Settings.EMPTY)) + new IncrementalBulkService(mock(Client.class), mock(IndexingPressure.class)) ).handleRequest( new FakeRestRequest.Builder(xContentRegistry()).withPath("my_index/_bulk").withParams(params).withContent(new BytesArray(""" {"index":{"_id":"1"}} @@ -102,7 +100,7 @@ public void bulk(BulkRequest request, ActionListener listener) { { new RestBulkAction( settings(IndexVersion.current()).build(), - new IncrementalBulkService(mock(Client.class), mock(IndexingPressure.class), new ThreadContext(Settings.EMPTY)) + new IncrementalBulkService(mock(Client.class), mock(IndexingPressure.class)) ).handleRequest( new FakeRestRequest.Builder(xContentRegistry()).withPath("my_index/_bulk") .withParams(params) @@ -126,7 +124,7 @@ public void bulk(BulkRequest request, ActionListener listener) { bulkCalled.set(false); new RestBulkAction( settings(IndexVersion.current()).build(), - new IncrementalBulkService(mock(Client.class), mock(IndexingPressure.class), new ThreadContext(Settings.EMPTY)) + new IncrementalBulkService(mock(Client.class), mock(IndexingPressure.class)) ).handleRequest( new FakeRestRequest.Builder(xContentRegistry()).withPath("my_index/_bulk") .withParams(params) @@ -149,7 +147,7 @@ public void bulk(BulkRequest request, ActionListener listener) { bulkCalled.set(false); new RestBulkAction( settings(IndexVersion.current()).build(), - new IncrementalBulkService(mock(Client.class), mock(IndexingPressure.class), new ThreadContext(Settings.EMPTY)) + new IncrementalBulkService(mock(Client.class), mock(IndexingPressure.class)) ).handleRequest( new FakeRestRequest.Builder(xContentRegistry()).withPath("my_index/_bulk") .withParams(params) @@ -173,7 +171,7 @@ public void bulk(BulkRequest request, ActionListener listener) { bulkCalled.set(false); new RestBulkAction( settings(IndexVersion.current()).build(), - new IncrementalBulkService(mock(Client.class), mock(IndexingPressure.class), new ThreadContext(Settings.EMPTY)) + new IncrementalBulkService(mock(Client.class), mock(IndexingPressure.class)) ).handleRequest( new FakeRestRequest.Builder(xContentRegistry()).withPath("my_index/_bulk") .withParams(params) @@ -229,7 +227,7 @@ public void next() { RestBulkAction.ChunkHandler chunkHandler = new RestBulkAction.ChunkHandler( true, request, - () -> new IncrementalBulkService.Handler(null, new ThreadContext(Settings.EMPTY), null, null, null, null) { + () -> new IncrementalBulkService.Handler(null, null, null, null, null) { @Override public void addItems(List> items, Releasable releasable, Runnable nextItems) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java index 8d580f10e5137..c0e55992df88f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java @@ -824,7 +824,7 @@ public void testSecurityRestHandlerInterceptorCanBeInstalled() throws IllegalAcc null, List.of(), RestExtension.allowAll(), - new IncrementalBulkService(null, null, new ThreadContext(Settings.EMPTY)) + new IncrementalBulkService(null, null) ); actionModule.initRestHandlers(null, null); From 812e43849235507409880cdeba53707c3f22ddcc Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 29 Oct 2024 14:11:37 -0400 Subject: [PATCH 10/18] [CI] Stop gradle processes between tasks in packer cache --- .buildkite/packer_cache.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/packer_cache.sh b/.buildkite/packer_cache.sh index 01e1ad5cd7823..e4a80d439741d 100755 --- a/.buildkite/packer_cache.sh +++ b/.buildkite/packer_cache.sh @@ -30,5 +30,7 @@ for branch in "${branches[@]}"; do export JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA" "checkout/${branch}/gradlew" --project-dir "$CHECKOUT_DIR" --parallel -s resolveAllDependencies -Dorg.gradle.warning.mode=none -DisCI --max-workers=4 + "checkout/${branch}/gradlew" --stop + pkill -f '.*GradleDaemon.*' rm -rf "checkout/${branch}" done From 18c3bcbd6cda661a179f7eacd29d13d592d692a5 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Tue, 29 Oct 2024 19:40:23 +0100 Subject: [PATCH 11/18] Fix cloud deploy PR job after removing cloud image (#115857) * Fix cloud deploy PR job after removing cloud image * Fix task name for building cloud ess docker image in cloud deploy --- .buildkite/scripts/cloud-deploy.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/cloud-deploy.sh b/.buildkite/scripts/cloud-deploy.sh index 2b98aa224406b..045b05ce16dee 100755 --- a/.buildkite/scripts/cloud-deploy.sh +++ b/.buildkite/scripts/cloud-deploy.sh @@ -2,11 +2,11 @@ set -euo pipefail -.ci/scripts/run-gradle.sh buildCloudDockerImage +.ci/scripts/run-gradle.sh buildCloudEssDockerImage ES_VERSION=$(grep 'elasticsearch' build-tools-internal/version.properties | awk '{print $3}') -DOCKER_TAG="docker.elastic.co/elasticsearch-ci/elasticsearch-cloud:${ES_VERSION}-${BUILDKITE_COMMIT:0:7}" -docker tag elasticsearch-cloud:test "$DOCKER_TAG" +DOCKER_TAG="docker.elastic.co/elasticsearch-ci/elasticsearch-cloud-ess:${ES_VERSION}-${BUILDKITE_COMMIT:0:7}" +docker tag elasticsearch-cloud-ess:test "$DOCKER_TAG" echo "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin docker.elastic.co unset DOCKER_REGISTRY_USERNAME DOCKER_REGISTRY_PASSWORD From 853f51fa05b1808d8741a20e2f009b06cbcf87b9 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 30 Oct 2024 05:53:18 +1100 Subject: [PATCH 12/18] Mute org.elasticsearch.reservedstate.service.FileSettingsServiceTests testProcessFileChanges #115280 --- muted-tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/muted-tests.yml b/muted-tests.yml index 419e8fbb68566..22e57a524f0bc 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -287,6 +287,9 @@ tests: - class: org.elasticsearch.search.StressSearchServiceReaperIT method: testStressReaper issue: https://github.com/elastic/elasticsearch/issues/115816 +- class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests + method: testProcessFileChanges + issue: https://github.com/elastic/elasticsearch/issues/115280 # Examples: # @@ -325,4 +328,4 @@ tests: # issue: "https://github.com/elastic/elasticsearch/..." # - class: "org.elasticsearch.xpack.esql.**" # method: "test {union_types.MultiIndexIpStringStatsInline *}" -# issue: "https://github.com/elastic/elasticsearch/..." \ No newline at end of file +# issue: "https://github.com/elastic/elasticsearch/..." From 06eb0727c22db5559cc9cc9da46ba423bd3663c5 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Tue, 29 Oct 2024 21:12:43 +0200 Subject: [PATCH 13/18] Use flattened names in ignored source (#115822) * Use flattened names in ignored source * spotless * fix rest compat * fix unittests * expand dots --- rest-api-spec/build.gradle | 4 + .../indices.create/20_synthetic_source.yml | 20 +-- .../21_synthetic_source_stored.yml | 49 +++++++ .../index/mapper/DocumentParserContext.java | 6 +- .../mapper/DotExpandingXContentParser.java | 4 + .../index/mapper/XContentDataHelper.java | 15 +- .../mapper/IgnoredSourceFieldMapperTests.java | 130 ++++++++++++++++++ 7 files changed, 211 insertions(+), 17 deletions(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 6cc2028bffa39..b9064ab1d79ad 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -59,6 +59,10 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> task.replaceValueInMatch("profile.shards.0.dfs.knn.0.query.0.description", "DocAndScoreQuery[0,...][0.009673266,...],0.009673266", "dfs knn vector profiling with vector_operations_count") task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility") task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility") + task.skipTest("indices.create/20_synthetic_source/object with dynamic override", "temporary until backported") + task.skipTest("indices.create/20_synthetic_source/object with unmapped fields", "temporary until backported") + task.skipTest("indices.create/20_synthetic_source/empty object with unmapped fields", "temporary until backported") + task.skipTest("indices.create/20_synthetic_source/nested object with unmapped fields", "temporary until backported") task.skipTest("indices.create/21_synthetic_source_stored/object param - nested object with stored array", "temporary until backported") task.skipTest("cat.aliases/10_basic/Deprecated local parameter", "CAT APIs not covered by compatibility policy") }) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index a871d2ac0ae15..258dfeb57e00c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -1,6 +1,6 @@ object with unmapped fields: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -41,13 +41,13 @@ object with unmapped fields: - match: { hits.hits.0._source.some_string: AaAa } - match: { hits.hits.0._source.some_int: 1000 } - match: { hits.hits.0._source.some_double: 123.456789 } - - match: { hits.hits.0._source.a.very.deeply.nested.field: AAAA } + - match: { hits.hits.0._source.a: { very.deeply.nested.field: AAAA } } - match: { hits.hits.0._source.some_bool: true } - match: { hits.hits.1._source.name: bbbb } - match: { hits.hits.1._source.some_string: BbBb } - match: { hits.hits.1._source.some_int: 2000 } - match: { hits.hits.1._source.some_double: 321.987654 } - - match: { hits.hits.1._source.a.very.deeply.nested.field: BBBB } + - match: { hits.hits.1._source.a: { very.deeply.nested.field: BBBB } } --- @@ -100,7 +100,7 @@ unmapped arrays: --- nested object with unmapped fields: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -143,16 +143,16 @@ nested object with unmapped fields: - match: { hits.total.value: 2 } - match: { hits.hits.0._source.path.to.name: aaaa } - match: { hits.hits.0._source.path.to.surname: AaAa } - - match: { hits.hits.0._source.path.some.other.name: AaAaAa } + - match: { hits.hits.0._source.path.some.other\.name: AaAaAa } - match: { hits.hits.1._source.path.to.name: bbbb } - match: { hits.hits.1._source.path.to.surname: BbBb } - - match: { hits.hits.1._source.path.some.other.name: BbBbBb } + - match: { hits.hits.1._source.path.some.other\.name: BbBbBb } --- empty object with unmapped fields: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -191,7 +191,7 @@ empty object with unmapped fields: - match: { hits.total.value: 1 } - match: { hits.hits.0._source.path.to.surname: AaAa } - - match: { hits.hits.0._source.path.some.other.name: AaAaAa } + - match: { hits.hits.0._source.path.some.other\.name: AaAaAa } --- @@ -434,7 +434,7 @@ mixed disabled and enabled objects: --- object with dynamic override: - requires: - cluster_features: ["mapper.ignored_source.dont_expand_dots"] + cluster_features: ["mapper.ignored_source.dont_expand_dots", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -475,7 +475,7 @@ object with dynamic override: - match: { hits.hits.0._source.path_no.to: { a.very.deeply.nested.field: A } } - match: { hits.hits.0._source.path_runtime.name: bar } - match: { hits.hits.0._source.path_runtime.some_int: 20 } - - match: { hits.hits.0._source.path_runtime.to.a.very.deeply.nested.field: B } + - match: { hits.hits.0._source.path_runtime.to: { a.very.deeply.nested.field: B } } --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index 6a4e92f694220..f3545bb0a3f0e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -1249,3 +1249,52 @@ index param - nested object with stored array: - match: { hits.hits.1._source.nested.0.b.1.c: 300 } - match: { hits.hits.1._source.nested.1.b.0.c: 40 } - match: { hits.hits.1._source.nested.1.b.1.c: 400 } + + +--- +index param - flattened fields: + - requires: + cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] + reason: requires keeping array source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + synthetic_source_keep: arrays + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + outer: + properties: + inner: + type: object + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "name": "A", "outer": { "inner": [ { "a.b": "AA", "a.c": "AAA" } ] } }' + - '{ "create": { } }' + - '{ "name": "B", "outer": { "inner": [ { "a.x.y.z": "BB", "a.z.y.x": "BBB" } ] } }' + + + - match: { errors: false } + + - do: + search: + index: test + sort: name + - match: { hits.total.value: 2 } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.outer.inner: [{ a.b: AA, a.c: AAA }] } + - match: { hits.hits.1._source.name: B } + - match: { hits.hits.1._source.outer.inner: [{ a.x.y.z: BB, a.z.y.x: BBB }] } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 3b1f1a6d2809a..c884d68c8f0ee 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -528,11 +528,7 @@ public final boolean addDynamicMapper(Mapper mapper) { if (canAddIgnoredField()) { try { addIgnoredField( - IgnoredSourceFieldMapper.NameValue.fromContext( - this, - mapper.fullPath(), - XContentDataHelper.encodeToken(parser()) - ) + IgnoredSourceFieldMapper.NameValue.fromContext(this, mapper.fullPath(), encodeFlattenedToken()) ); } catch (IOException e) { throw new IllegalArgumentException("failed to parse field [" + mapper.fullPath() + " ]", e); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DotExpandingXContentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DotExpandingXContentParser.java index fc003e709cbca..42784e0974417 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DotExpandingXContentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DotExpandingXContentParser.java @@ -34,6 +34,10 @@ */ class DotExpandingXContentParser extends FilterXContentParserWrapper { + static boolean isInstance(XContentParser parser) { + return parser instanceof WrappingParser; + } + private static final class WrappingParser extends FilterXContentParser { private final ContentPath contentPath; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java index 8bacaf8505f91..dee5ff92040a9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java @@ -221,8 +221,11 @@ static Tuple cloneSubContextWithParser(Do private static Tuple cloneSubContextParserConfiguration(DocumentParserContext context) throws IOException { XContentParser parser = context.parser(); + var oldValue = context.path().isWithinLeafObject(); + context.path().setWithinLeafObject(true); XContentBuilder builder = XContentBuilder.builder(parser.contentType().xContent()); builder.copyCurrentStructure(parser); + context.path().setWithinLeafObject(oldValue); XContentParserConfiguration configuration = XContentParserConfiguration.EMPTY.withRegistry(parser.getXContentRegistry()) .withDeprecationHandler(parser.getDeprecationHandler()) @@ -235,9 +238,17 @@ private static DocumentParserContext cloneDocumentParserContext( XContentParserConfiguration configuration, XContentBuilder builder ) throws IOException { - DocumentParserContext subcontext = context.switchParser( - XContentHelper.createParserNotCompressed(configuration, BytesReference.bytes(builder), context.parser().contentType()) + XContentParser newParser = XContentHelper.createParserNotCompressed( + configuration, + BytesReference.bytes(builder), + context.parser().contentType() ); + if (DotExpandingXContentParser.isInstance(context.parser())) { + // If we performed dot expanding originally we need to continue to do so when we replace the parser. + newParser = DotExpandingXContentParser.expandDots(newParser, context.path()); + } + + DocumentParserContext subcontext = context.switchParser(newParser); subcontext.setRecordedSource(); // Avoids double-storing parts of the source for the same parser subtree. subcontext.parser().nextToken(); return subcontext; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 7a4ce8bcb03fa..884372d249287 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -2075,6 +2075,136 @@ public void testDisabledObjectWithFlatFields() throws IOException { {"top":[{"file.name":"A","file.line":10},{"file.name":"B","file.line":20}]}""", syntheticSourceWithArray); } + public void testRegularObjectWithFlatFields() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("top").field("type", "object").field("synthetic_source_keep", "all").endObject(); + })).documentMapper(); + + CheckedConsumer document = b -> { + b.startObject("top"); + b.field("file.name", "A"); + b.field("file.line", 10); + b.endObject(); + }; + + var syntheticSource = syntheticSource(documentMapper, document); + assertEquals("{\"top\":{\"file.name\":\"A\",\"file.line\":10}}", syntheticSource); + + CheckedConsumer documentWithArray = b -> { + b.startArray("top"); + b.startObject(); + b.field("file.name", "A"); + b.field("file.line", 10); + b.endObject(); + b.startObject(); + b.field("file.name", "B"); + b.field("file.line", 20); + b.endObject(); + b.endArray(); + }; + + var syntheticSourceWithArray = syntheticSource(documentMapper, documentWithArray); + assertEquals(""" + {"top":[{"file.name":"A","file.line":10},{"file.name":"B","file.line":20}]}""", syntheticSourceWithArray); + } + + public void testRegularObjectWithFlatFieldsInsideAnArray() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("top"); + b.startObject("properties"); + { + b.startObject("inner").field("type", "object").field("synthetic_source_keep", "all").endObject(); + } + b.endObject(); + b.endObject(); + })).documentMapper(); + + CheckedConsumer document = b -> { + b.startArray("top"); + b.startObject(); + { + b.startObject("inner"); + b.field("file.name", "A"); + b.field("file.line", 10); + b.endObject(); + } + b.endObject(); + b.endArray(); + }; + + var syntheticSource = syntheticSource(documentMapper, document); + assertEquals("{\"top\":{\"inner\":{\"file.name\":\"A\",\"file.line\":10}}}", syntheticSource); + + CheckedConsumer documentWithArray = b -> { + b.startArray("top"); + b.startObject(); + { + b.startObject("inner"); + b.field("file.name", "A"); + b.field("file.line", 10); + b.endObject(); + } + b.endObject(); + b.startObject(); + { + b.startObject("inner"); + b.field("file.name", "B"); + b.field("file.line", 20); + b.endObject(); + } + b.endObject(); + b.endArray(); + }; + + var syntheticSourceWithArray = syntheticSource(documentMapper, documentWithArray); + assertEquals(""" + {"top":{"inner":[{"file.name":"A","file.line":10},{"file.name":"B","file.line":20}]}}""", syntheticSourceWithArray); + } + + public void testIgnoredDynamicObjectWithFlatFields() throws IOException { + var syntheticSource = getSyntheticSourceWithFieldLimit(b -> { + b.startObject("top"); + b.field("file.name", "A"); + b.field("file.line", 10); + b.endObject(); + }); + assertEquals("{\"top\":{\"file.name\":\"A\",\"file.line\":10}}", syntheticSource); + + var syntheticSourceWithArray = getSyntheticSourceWithFieldLimit(b -> { + b.startArray("top"); + b.startObject(); + b.field("file.name", "A"); + b.field("file.line", 10); + b.endObject(); + b.startObject(); + b.field("file.name", "B"); + b.field("file.line", 20); + b.endObject(); + b.endArray(); + }); + assertEquals(""" + {"top":[{"file.name":"A","file.line":10},{"file.name":"B","file.line":20}]}""", syntheticSourceWithArray); + } + + public void testStoredArrayWithFlatFields() throws IOException { + DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { + b.startObject("outer").startObject("properties"); + { + b.startObject("inner").field("type", "object").endObject(); + } + b.endObject().endObject(); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { + b.startObject("outer").startArray("inner"); + { + b.startObject().field("a.b", "a.b").field("a.c", "a.c").endObject(); + } + b.endArray().endObject(); + }); + assertEquals(""" + {"outer":{"inner":[{"a.b":"a.b","a.c":"a.c"}]}}""", syntheticSource); + } + protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader) throws IOException { // We exclude ignored source field since in some cases it contains an exact copy of a part of document source. From e5d5c17c99c476e9820ed141edd87af0c3adbef5 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 29 Oct 2024 13:02:28 -0700 Subject: [PATCH 14/18] Use directory name as project name for libs (#115720) The libs projects are configured to all begin with `elasticsearch-`. While this is desireable for the artifacts to contain this consistent prefix, it means the project names don't match up with their directories. Additionally, it creates complexities for subproject naming that must be manually adjusted. This commit adjusts the project names for those under libs to be their directory names. The resulting artifacts for these libs are kept the same, all beginning with `elasticsearch-`. --- benchmarks/build.gradle | 6 +-- .../internal/PublishPluginFuncTest.groovy | 2 +- .../src/main/groovy/elasticsearch.ide.gradle | 2 +- .../groovy/elasticsearch.stable-api.gradle | 6 +-- .../internal/ElasticsearchJavaBasePlugin.java | 2 +- .../InternalDistributionBwcSetupPlugin.java | 13 ++++-- .../precommit/JarHellPrecommitPlugin.java | 4 +- .../ThirdPartyAuditPrecommitPlugin.java | 2 +- .../fixtures/AbstractGradleFuncTest.groovy | 2 +- client/rest/build.gradle | 2 +- client/sniffer/build.gradle | 2 +- client/test/build.gradle | 2 +- distribution/build.gradle | 4 +- .../tools/entitlement-agent/build.gradle | 2 +- .../tools/entitlement-runtime/build.gradle | 4 +- distribution/tools/geoip-cli/build.gradle | 4 +- distribution/tools/keystore-cli/build.gradle | 2 +- distribution/tools/plugin-cli/build.gradle | 6 +-- distribution/tools/server-cli/build.gradle | 2 +- .../tools/windows-service-cli/build.gradle | 2 +- libs/build.gradle | 40 +++++++++++++++++-- libs/cli/build.gradle | 4 +- libs/core/build.gradle | 6 +-- libs/dissect/build.gradle | 2 +- libs/geo/build.gradle | 2 +- libs/grok/build.gradle | 2 +- libs/h3/build.gradle | 4 +- libs/logging/build.gradle | 4 +- libs/logstash-bridge/build.gradle | 6 +-- libs/lz4/build.gradle | 4 +- libs/native/build.gradle | 6 +-- libs/plugin-analysis-api/build.gradle | 4 +- libs/plugin-scanner/build.gradle | 8 ++-- libs/secure-sm/build.gradle | 2 +- libs/simdvec/build.gradle | 6 +-- libs/ssl-config/build.gradle | 4 +- libs/tdigest/build.gradle | 4 +- libs/x-content/build.gradle | 6 +-- libs/x-content/impl/build.gradle | 6 +-- modules/ingest-common/build.gradle | 4 +- modules/reindex/build.gradle | 2 +- modules/runtime-fields-common/build.gradle | 4 +- modules/systemd/build.gradle | 2 +- modules/transport-netty4/build.gradle | 4 +- qa/logging-config/build.gradle | 2 +- qa/packaging/build.gradle | 2 +- server/build.gradle | 26 ++++++------ settings.gradle | 12 +----- .../apm-integration/build.gradle | 2 +- test/fixtures/geoip-fixture/build.gradle | 4 +- test/framework/build.gradle | 4 +- test/x-content/build.gradle | 2 +- x-pack/plugin/blob-cache/build.gradle | 2 +- x-pack/plugin/core/build.gradle | 4 +- x-pack/plugin/esql/build.gradle | 4 +- .../plugin/esql/qa/testFixtures/build.gradle | 4 +- x-pack/plugin/inference/build.gradle | 2 +- x-pack/plugin/ml-package-loader/build.gradle | 2 +- x-pack/plugin/ml/build.gradle | 2 +- .../plugin/searchable-snapshots/build.gradle | 2 +- x-pack/plugin/spatial/build.gradle | 2 +- x-pack/plugin/sql/sql-action/build.gradle | 6 +-- x-pack/plugin/sql/sql-cli/build.gradle | 2 +- x-pack/plugin/sql/sql-proto/build.gradle | 4 +- x-pack/plugin/text-structure/build.gradle | 2 +- .../plugin/transform/qa/common/build.gradle | 2 +- 66 files changed, 165 insertions(+), 136 deletions(-) diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle index 3a03cbe2d934d..f3ced9f16d327 100644 --- a/benchmarks/build.gradle +++ b/benchmarks/build.gradle @@ -40,15 +40,15 @@ dependencies { // us to invoke the JMH uberjar as usual. exclude group: 'net.sf.jopt-simple', module: 'jopt-simple' } - api(project(':libs:elasticsearch-h3')) + api(project(':libs:h3')) api(project(':modules:aggregations')) api(project(':x-pack:plugin:esql-core')) api(project(':x-pack:plugin:esql')) api(project(':x-pack:plugin:esql:compute')) - implementation project(path: ':libs:elasticsearch-simdvec') + implementation project(path: ':libs:simdvec') expression(project(path: ':modules:lang-expression', configuration: 'zip')) painless(project(path: ':modules:lang-painless', configuration: 'zip')) - nativeLib(project(':libs:elasticsearch-native')) + nativeLib(project(':libs:native')) api "org.openjdk.jmh:jmh-core:$versions.jmh" annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh" // Dependencies of JMH diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy index 99d451116dbe7..6e403c85a23f4 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy @@ -18,7 +18,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { def setup() { // required for JarHell to work - subProject(":libs:elasticsearch-core") << "apply plugin:'java'" + subProject(":libs:core") << "apply plugin:'java'" configurationCacheCompatible = false } diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle index 67878181a005d..86b48f744e16e 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle @@ -161,7 +161,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') { ':server:generateModulesList', ':server:generatePluginsList', ':generateProviderImpls', - ':libs:elasticsearch-native:elasticsearch-native-libraries:extractLibs', + ':libs:native:native-libraries:extractLibs', ':x-pack:libs:es-opensaml-security-api:shadowJar'].collect { elasticsearchProject.right()?.task(it) ?: it }) } diff --git a/build-tools-internal/src/main/groovy/elasticsearch.stable-api.gradle b/build-tools-internal/src/main/groovy/elasticsearch.stable-api.gradle index 0148caf8983ef..1fab4d035177a 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.stable-api.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.stable-api.gradle @@ -33,12 +33,12 @@ BuildParams.bwcVersions.withIndexCompatible({ it.onOrAfter(Version.fromString(ex if (unreleasedVersion) { // For unreleased snapshot versions, build them from source "oldJar${baseName}"(files(project(unreleasedVersion.gradleProjectPath).tasks.named(buildBwcTaskName(project.name)))) - } else if(bwcVersion.onOrAfter('8.7.0') && project.name.endsWith("elasticsearch-logging")==false) { + } else if(bwcVersion.onOrAfter('8.7.0') && project.name.endsWith("logging")==false) { //there was a package rename in 8.7.0, except for es-logging - "oldJar${baseName}"("org.elasticsearch.plugin:${project.name}:${bwcVersion}") + "oldJar${baseName}"("org.elasticsearch.plugin:elasticsearch-${project.name}:${bwcVersion}") } else { // For released versions, download it - "oldJar${baseName}"("org.elasticsearch:${project.name}:${bwcVersion}") + "oldJar${baseName}"("org.elasticsearch:elasticsearch-${project.name}:${bwcVersion}") } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java index 5913339e32f47..05b7af83aa8e4 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java @@ -177,7 +177,7 @@ public static void configureInputNormalization(Project project) { } private static void configureNativeLibraryPath(Project project) { - String nativeProject = ":libs:elasticsearch-native:elasticsearch-native-libraries"; + String nativeProject = ":libs:native:native-libraries"; Configuration nativeConfig = project.getConfigurations().create("nativeLibs"); nativeConfig.defaultDependencies(deps -> { deps.add(project.getDependencies().project(Map.of("path", nativeProject, "configuration", "default"))); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java index 90b9c0d395f43..fcf286ed471dd 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java @@ -165,7 +165,12 @@ private static void configureBwcProject( DistributionProjectArtifact stableAnalysisPluginProjectArtifact = new DistributionProjectArtifact( new File( checkoutDir.get(), - relativeDir + "/build/distributions/" + stableApiProject.getName() + "-" + bwcVersion.get() + "-SNAPSHOT.jar" + relativeDir + + "/build/distributions/elasticsearch-" + + stableApiProject.getName() + + "-" + + bwcVersion.get() + + "-SNAPSHOT.jar" ), null ); @@ -275,7 +280,7 @@ private static List resolveArchiveProjects(File checkoutDir } private static List resolveStableProjects(Project project) { - Set stableProjectNames = Set.of("elasticsearch-logging", "elasticsearch-plugin-api", "elasticsearch-plugin-analysis-api"); + Set stableProjectNames = Set.of("logging", "plugin-api", "plugin-analysis-api"); return project.findProject(":libs") .getSubprojects() .stream() @@ -312,7 +317,9 @@ static void createBuildBwcTask( c.getOutputs().files(expectedOutputFile); } c.getOutputs().doNotCacheIf("BWC distribution caching is disabled for local builds", task -> BuildParams.isCi() == false); - c.getArgs().add(projectPath.replace('/', ':') + ":" + assembleTaskName); + c.getArgs().add("-p"); + c.getArgs().add(projectPath); + c.getArgs().add(assembleTaskName); if (project.getGradle().getStartParameter().isBuildCacheEnabled()) { c.getArgs().add("--build-cache"); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JarHellPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JarHellPrecommitPlugin.java index 56434cf1f4eda..0a22a2b61c953 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JarHellPrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JarHellPrecommitPlugin.java @@ -21,11 +21,11 @@ public class JarHellPrecommitPlugin extends PrecommitPlugin { public TaskProvider createTask(Project project) { project.getPluginManager().apply(JarHellPlugin.class); - if (project.getPath().equals(":libs:elasticsearch-core") == false) { + if (project.getPath().equals(":libs:core") == false) { // ideally we would configure this as a default dependency. But Default dependencies do not work correctly // with gradle project dependencies as they're resolved to late in the build and don't setup according task // dependencies properly - var elasticsearchCoreProject = project.findProject(":libs:elasticsearch-core"); + var elasticsearchCoreProject = project.findProject(":libs:core"); if (elasticsearchCoreProject != null) { project.getDependencies().add("jarHell", elasticsearchCoreProject); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java index f0eefe1f81a8c..80cece6074ab7 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java @@ -27,7 +27,7 @@ public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin { public static final String JDK_JAR_HELL_CONFIG_NAME = "jdkJarHell"; - public static final String LIBS_ELASTICSEARCH_CORE_PROJECT_PATH = ":libs:elasticsearch-core"; + public static final String LIBS_ELASTICSEARCH_CORE_PROJECT_PATH = ":libs:core"; @Override public TaskProvider createTask(Project project) { diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy index 567fb048fad54..d3d06b2de3575 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy @@ -56,7 +56,7 @@ abstract class AbstractGradleFuncTest extends Specification { propertiesFile << "org.gradle.java.installations.fromEnv=JAVA_HOME,RUNTIME_JAVA_HOME,JAVA15_HOME,JAVA14_HOME,JAVA13_HOME,JAVA12_HOME,JAVA11_HOME,JAVA8_HOME" - def nativeLibsProject = subProject(":libs:elasticsearch-native:elasticsearch-native-libraries") + def nativeLibsProject = subProject(":libs:native:native-libraries") nativeLibsProject << """ plugins { id 'base' diff --git a/client/rest/build.gradle b/client/rest/build.gradle index 6006fae1c2d84..003c251186510 100644 --- a/client/rest/build.gradle +++ b/client/rest/build.gradle @@ -79,7 +79,7 @@ tasks.named('forbiddenApisTest').configure { } // JarHell is part of es server, which we don't want to pull in -// TODO: Not anymore. Now in :libs:elasticsearch-core +// TODO: Not anymore. Now in :libs:core tasks.named("jarHell").configure { enabled = false } diff --git a/client/sniffer/build.gradle b/client/sniffer/build.gradle index 901917c7b25f8..f6f26c8f7c0d5 100644 --- a/client/sniffer/build.gradle +++ b/client/sniffer/build.gradle @@ -73,7 +73,7 @@ tasks.named("dependencyLicenses").configure { } // JarHell is part of es server, which we don't want to pull in -// TODO: Not anymore. Now in :libs:elasticsearch-core +// TODO: Not anymore. Now in :libs:core tasks.named("jarHell").configure { enabled = false } tasks.named("testTestingConventions").configure { diff --git a/client/test/build.gradle b/client/test/build.gradle index 3a3a9e3c03264..8de6b3dbf92be 100644 --- a/client/test/build.gradle +++ b/client/test/build.gradle @@ -54,7 +54,7 @@ tasks.named('forbiddenApisTest').configure { tasks.named("thirdPartyAudit").configure { enabled = false } // JarHell is part of es server, which we don't want to pull in -// TODO: Not anymore. Now in :libs:elasticsearch-core +// TODO: Not anymore. Now in :libs:core tasks.named("jarHell").configure { enabled = false } // TODO: should we have licenses for our test deps? diff --git a/distribution/build.gradle b/distribution/build.gradle index 72dea714fdcdb..f7b6f7bc1c7d0 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -275,7 +275,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { } all { resolutionStrategy.dependencySubstitution { - substitute module("org.apache.logging.log4j:log4j-core") using project(":libs:elasticsearch-log4j") because "patched to remove JndiLookup clas"} + substitute module("org.apache.logging.log4j:log4j-core") using project(":libs:log4j") because "patched to remove JndiLookup clas"} } } @@ -291,7 +291,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { libsKeystoreCli project(path: ':distribution:tools:keystore-cli') libsSecurityCli project(':x-pack:plugin:security:cli') libsGeoIpCli project(':distribution:tools:geoip-cli') - libsNative project(':libs:elasticsearch-native:elasticsearch-native-libraries') + libsNative project(':libs:native:native-libraries') } project.ext { diff --git a/distribution/tools/entitlement-agent/build.gradle b/distribution/tools/entitlement-agent/build.gradle index 3fa9d0f5ef83a..d3e7ae10dcc6d 100644 --- a/distribution/tools/entitlement-agent/build.gradle +++ b/distribution/tools/entitlement-agent/build.gradle @@ -22,7 +22,7 @@ configurations { dependencies { entitlementBridge project(":distribution:tools:entitlement-bridge") - compileOnly project(":libs:elasticsearch-core") + compileOnly project(":libs:core") compileOnly project(":distribution:tools:entitlement-runtime") testImplementation project(":test:framework") testImplementation project(":distribution:tools:entitlement-bridge") diff --git a/distribution/tools/entitlement-runtime/build.gradle b/distribution/tools/entitlement-runtime/build.gradle index 55471272c1b5f..aaeee76d8bc57 100644 --- a/distribution/tools/entitlement-runtime/build.gradle +++ b/distribution/tools/entitlement-runtime/build.gradle @@ -10,8 +10,8 @@ apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.publish' dependencies { - compileOnly project(':libs:elasticsearch-core') // For @SuppressForbidden - compileOnly project(":libs:elasticsearch-x-content") // for parsing policy files + compileOnly project(':libs:core') // For @SuppressForbidden + compileOnly project(":libs:x-content") // for parsing policy files compileOnly project(':server') // To access the main server module for special permission checks compileOnly project(':distribution:tools:entitlement-bridge') testImplementation project(":test:framework") diff --git a/distribution/tools/geoip-cli/build.gradle b/distribution/tools/geoip-cli/build.gradle index ee20d5e1bd88e..26af3bb4f9911 100644 --- a/distribution/tools/geoip-cli/build.gradle +++ b/distribution/tools/geoip-cli/build.gradle @@ -15,8 +15,8 @@ base { dependencies { compileOnly project(":server") - compileOnly project(":libs:elasticsearch-cli") - compileOnly project(":libs:elasticsearch-x-content") + compileOnly project(":libs:cli") + compileOnly project(":libs:x-content") testImplementation project(":test:framework") testImplementation "org.apache.commons:commons-compress:1.26.1" testImplementation "commons-io:commons-io:2.15.1" diff --git a/distribution/tools/keystore-cli/build.gradle b/distribution/tools/keystore-cli/build.gradle index 07aa92151171a..0140cd9d8eedf 100644 --- a/distribution/tools/keystore-cli/build.gradle +++ b/distribution/tools/keystore-cli/build.gradle @@ -11,7 +11,7 @@ apply plugin: 'elasticsearch.build' dependencies { compileOnly project(":server") - compileOnly project(":libs:elasticsearch-cli") + compileOnly project(":libs:cli") testImplementation project(":test:framework") testImplementation "com.google.jimfs:jimfs:${versions.jimfs}" testRuntimeOnly "com.google.guava:guava:${versions.jimfs_guava}" diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle index 16932df96e223..ac8ade89c9014 100644 --- a/distribution/tools/plugin-cli/build.gradle +++ b/distribution/tools/plugin-cli/build.gradle @@ -21,9 +21,9 @@ tasks.named("dependencyLicenses").configure { dependencies { compileOnly project(":server") - compileOnly project(":libs:elasticsearch-cli") - implementation project(":libs:elasticsearch-plugin-api") - implementation project(":libs:elasticsearch-plugin-scanner") + compileOnly project(":libs:cli") + implementation project(":libs:plugin-api") + implementation project(":libs:plugin-scanner") // TODO: asm is picked up from the plugin scanner, we should consolidate so it is not defined twice implementation 'org.ow2.asm:asm:9.7' implementation 'org.ow2.asm:asm-tree:9.7' diff --git a/distribution/tools/server-cli/build.gradle b/distribution/tools/server-cli/build.gradle index e8f70e9053d7c..299d511ba5dbe 100644 --- a/distribution/tools/server-cli/build.gradle +++ b/distribution/tools/server-cli/build.gradle @@ -12,7 +12,7 @@ apply plugin: 'elasticsearch.build' dependencies { compileOnly project(":server") - compileOnly project(":libs:elasticsearch-cli") + compileOnly project(":libs:cli") testImplementation project(":test:framework") } diff --git a/distribution/tools/windows-service-cli/build.gradle b/distribution/tools/windows-service-cli/build.gradle index 77da0d407a40d..dcfaf244b7eec 100644 --- a/distribution/tools/windows-service-cli/build.gradle +++ b/distribution/tools/windows-service-cli/build.gradle @@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.build' dependencies { compileOnly project(":server") - compileOnly project(":libs:elasticsearch-cli") + compileOnly project(":libs:cli") compileOnly project(":distribution:tools:server-cli") testImplementation project(":test:framework") diff --git a/libs/build.gradle b/libs/build.gradle index d0dfabd9c4fc5..efd2329ca2b5e 100644 --- a/libs/build.gradle +++ b/libs/build.gradle @@ -7,10 +7,42 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -configure(childProjects.values() - project('elasticsearch-log4j')) { +configure(childProjects.values()) { + + apply plugin: 'base' + /* - * All subprojects are java projects using Elasticsearch's standard build - * tools. + * Although these libs are local to Elasticsearch, they can conflict with other similarly + * named libraries when downloaded into a single directory via maven. Here we set the + * name of all libs to begin with the "elasticsearch-" prefix. Additionally, subprojects + * of libs begin with their parents artifactId. */ - apply plugin: 'elasticsearch.build' + def baseProject = project + def baseArtifactId = "elasticsearch-${it.name}" + base { + archivesName = baseArtifactId + } + subprojects { + apply plugin: 'base' + + def subArtifactId = baseArtifactId + def currentProject = project + while (currentProject != baseProject) { + subArtifactId += "-${currentProject.name}" + currentProject = currentProject.parent + } + base { + archivesName = subArtifactId + } + } + + // log4j is a hack, and not really a full elasticsearch built jar + if (project.name != 'log4j') { + + /* + * All subprojects are java projects using Elasticsearch's standard build + * tools. + */ + apply plugin: 'elasticsearch.build' + } } diff --git a/libs/cli/build.gradle b/libs/cli/build.gradle index b6ae962eaa603..d5842d4a2c59c 100644 --- a/libs/cli/build.gradle +++ b/libs/cli/build.gradle @@ -11,10 +11,10 @@ apply plugin: 'elasticsearch.publish' dependencies { api 'net.sf.jopt-simple:jopt-simple:5.0.2' - api project(':libs:elasticsearch-core') + api project(':libs:core') testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-cli' + exclude group: 'org.elasticsearch', module: 'cli' } } diff --git a/libs/core/build.gradle b/libs/core/build.gradle index ebbeac141e4bd..e24417e09a53d 100644 --- a/libs/core/build.gradle +++ b/libs/core/build.gradle @@ -13,19 +13,19 @@ apply plugin: 'elasticsearch.mrjar' dependencies { // This dependency is used only by :libs:core for null-checking interop with other tools compileOnly "com.google.code.findbugs:jsr305:3.0.2" - compileOnly project(':libs:elasticsearch-logging') + compileOnly project(':libs:logging') testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" testImplementation "junit:junit:${versions.junit}" testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}" testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-core' + exclude group: 'org.elasticsearch', module: 'core' } } tasks.named('forbiddenApisMain').configure { - // :libs:elasticsearch-core does not depend on server + // :libs:core does not depend on server // TODO: Need to decide how we want to handle for forbidden signatures with the changes to server replaceSignatureFiles 'jdk-signatures' } diff --git a/libs/dissect/build.gradle b/libs/dissect/build.gradle index be2691bfd332f..f1a09cc0ba0e6 100644 --- a/libs/dissect/build.gradle +++ b/libs/dissect/build.gradle @@ -9,7 +9,7 @@ dependencies { testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-dissect' + exclude group: 'org.elasticsearch', module: 'dissect' } testImplementation "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" testImplementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" diff --git a/libs/geo/build.gradle b/libs/geo/build.gradle index 37dd65cb19262..c753ba814a5f9 100644 --- a/libs/geo/build.gradle +++ b/libs/geo/build.gradle @@ -12,7 +12,7 @@ apply plugin: 'elasticsearch.publish' dependencies { testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-geo' + exclude group: 'org.elasticsearch', module: 'geo' } } diff --git a/libs/grok/build.gradle b/libs/grok/build.gradle index ce4be699953c7..2a74927fedd83 100644 --- a/libs/grok/build.gradle +++ b/libs/grok/build.gradle @@ -14,7 +14,7 @@ dependencies { api 'org.jruby.jcodings:jcodings:1.0.44' testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-grok' + exclude group: 'org.elasticsearch', module: 'grok' } } diff --git a/libs/h3/build.gradle b/libs/h3/build.gradle index 0eb1aea09d49c..81a0d56ed4606 100644 --- a/libs/h3/build.gradle +++ b/libs/h3/build.gradle @@ -23,7 +23,7 @@ apply plugin: 'elasticsearch.publish' dependencies { testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-geo' + exclude group: 'org.elasticsearch', module: 'geo' } // lucene topology library that uses spherical geometry testImplementation "org.apache.lucene:lucene-spatial3d:${versions.lucene}" @@ -40,4 +40,4 @@ licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt')) tasks.withType(LicenseHeadersTask.class).configureEach { approvedLicenses = ['Apache', 'Generated', 'Vendored'] -} \ No newline at end of file +} diff --git a/libs/logging/build.gradle b/libs/logging/build.gradle index 4222d89ebe2da..f52c2629176a7 100644 --- a/libs/logging/build.gradle +++ b/libs/logging/build.gradle @@ -14,12 +14,12 @@ tasks.named("loggerUsageCheck").configure {enabled = false } dependencies { testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-logging' + exclude group: 'org.elasticsearch', module: 'logging' } } tasks.named('forbiddenApisMain').configure { - // :libs:elasticsearch-logging does not depend on server + // :libs:logging does not depend on server replaceSignatureFiles 'jdk-signatures' } diff --git a/libs/logstash-bridge/build.gradle b/libs/logstash-bridge/build.gradle index e4b2728f693a0..117bed1e98105 100644 --- a/libs/logstash-bridge/build.gradle +++ b/libs/logstash-bridge/build.gradle @@ -10,9 +10,9 @@ apply plugin: 'elasticsearch.build' dependencies { compileOnly project(':server') - compileOnly project(':libs:elasticsearch-core') - compileOnly project(':libs:elasticsearch-plugin-api') - compileOnly project(':libs:elasticsearch-x-content') + compileOnly project(':libs:core') + compileOnly project(':libs:plugin-api') + compileOnly project(':libs:x-content') compileOnly project(':modules:lang-painless') compileOnly project(':modules:lang-painless:spi') compileOnly project(':modules:lang-mustache') diff --git a/libs/lz4/build.gradle b/libs/lz4/build.gradle index d9f1175248121..72e1bb50187a7 100644 --- a/libs/lz4/build.gradle +++ b/libs/lz4/build.gradle @@ -10,10 +10,10 @@ apply plugin: 'elasticsearch.publish' dependencies { api 'org.lz4:lz4-java:1.8.0' - api project(':libs:elasticsearch-core') + api project(':libs:core') testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-lz4' + exclude group: 'org.elasticsearch', module: 'lz4' } } diff --git a/libs/native/build.gradle b/libs/native/build.gradle index 0c889d47566fb..eff8f82434461 100644 --- a/libs/native/build.gradle +++ b/libs/native/build.gradle @@ -14,10 +14,10 @@ apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.mrjar' dependencies { - api project(':libs:elasticsearch-core') - api project(':libs:elasticsearch-logging') + api project(':libs:core') + api project(':libs:logging') testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-native' + exclude group: 'org.elasticsearch', module: 'native' } } diff --git a/libs/plugin-analysis-api/build.gradle b/libs/plugin-analysis-api/build.gradle index e240f18a88e0a..3f1670d76a0c1 100644 --- a/libs/plugin-analysis-api/build.gradle +++ b/libs/plugin-analysis-api/build.gradle @@ -18,12 +18,12 @@ tasks.named("loggerUsageCheck").configure {enabled = false } dependencies { api "org.apache.lucene:lucene-core:${versions.lucene}" - api project(':libs:elasticsearch-plugin-api') + api project(':libs:plugin-api') } tasks.named('forbiddenApisMain').configure { - // :libs:elasticsearch-logging does not depend on server + // :libs:logging does not depend on server replaceSignatureFiles 'jdk-signatures' } diff --git a/libs/plugin-scanner/build.gradle b/libs/plugin-scanner/build.gradle index b8cd224eba46a..d04af0624b3b1 100644 --- a/libs/plugin-scanner/build.gradle +++ b/libs/plugin-scanner/build.gradle @@ -16,16 +16,16 @@ tasks.named("dependencyLicenses").configure { } dependencies { - api project(':libs:elasticsearch-core') - api project(':libs:elasticsearch-plugin-api') - api project(":libs:elasticsearch-x-content") + api project(':libs:core') + api project(':libs:plugin-api') + api project(":libs:x-content") api 'org.ow2.asm:asm:9.7' api 'org.ow2.asm:asm-tree:9.7' testImplementation "junit:junit:${versions.junit}" testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-plugin-scanner' + exclude group: 'org.elasticsearch', module: 'plugin-scanner' } } tasks.named('forbiddenApisMain').configure { diff --git a/libs/secure-sm/build.gradle b/libs/secure-sm/build.gradle index 5e35f3ac7126f..473a86215e91e 100644 --- a/libs/secure-sm/build.gradle +++ b/libs/secure-sm/build.gradle @@ -16,7 +16,7 @@ dependencies { testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}" testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-secure-sm' + exclude group: 'org.elasticsearch', module: 'secure-sm' } } diff --git a/libs/simdvec/build.gradle b/libs/simdvec/build.gradle index eee56be72d0bf..02f960130e690 100644 --- a/libs/simdvec/build.gradle +++ b/libs/simdvec/build.gradle @@ -15,12 +15,12 @@ apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.mrjar' dependencies { - implementation project(':libs:elasticsearch-native') - implementation project(':libs:elasticsearch-logging') + implementation project(':libs:native') + implementation project(':libs:logging') implementation "org.apache.lucene:lucene-core:${versions.lucene}" testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-native' + exclude group: 'org.elasticsearch', module: 'native' } } diff --git a/libs/ssl-config/build.gradle b/libs/ssl-config/build.gradle index 3c0eb7c440510..d63df95003ab6 100644 --- a/libs/ssl-config/build.gradle +++ b/libs/ssl-config/build.gradle @@ -9,10 +9,10 @@ apply plugin: "elasticsearch.publish" dependencies { - api project(':libs:elasticsearch-core') + api project(':libs:core') testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-ssl-config' + exclude group: 'org.elasticsearch', module: 'ssl-config' } testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" diff --git a/libs/tdigest/build.gradle b/libs/tdigest/build.gradle index 231eb845339aa..2713df701fb44 100644 --- a/libs/tdigest/build.gradle +++ b/libs/tdigest/build.gradle @@ -22,11 +22,11 @@ apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.publish' dependencies { - api project(':libs:elasticsearch-core') + api project(':libs:core') api "org.apache.lucene:lucene-core:${versions.lucene}" testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-tdigest' + exclude group: 'org.elasticsearch', module: 'tdigest' } testImplementation 'org.junit.jupiter:junit-jupiter:5.8.1' } diff --git a/libs/x-content/build.gradle b/libs/x-content/build.gradle index 7540bd0fb68f0..1cf18d46e7610 100644 --- a/libs/x-content/build.gradle +++ b/libs/x-content/build.gradle @@ -12,14 +12,14 @@ apply plugin: 'elasticsearch.publish' apply plugin: 'elasticsearch.embedded-providers' embeddedProviders { - impl 'x-content', project(':libs:elasticsearch-x-content:impl') + impl 'x-content', project(':libs:x-content:impl') } dependencies { - api project(':libs:elasticsearch-core') + api project(':libs:core') testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-x-content' + exclude group: 'org.elasticsearch', module: 'x-content' } } diff --git a/libs/x-content/impl/build.gradle b/libs/x-content/impl/build.gradle index 753d2c3d5fe1e..35e122d336c68 100644 --- a/libs/x-content/impl/build.gradle +++ b/libs/x-content/impl/build.gradle @@ -16,8 +16,8 @@ base { String jacksonVersion = "2.17.2" dependencies { - compileOnly project(':libs:elasticsearch-core') - compileOnly project(':libs:elasticsearch-x-content') + compileOnly project(':libs:core') + compileOnly project(':libs:x-content') implementation "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${jacksonVersion}" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jacksonVersion}" @@ -25,7 +25,7 @@ dependencies { implementation "org.yaml:snakeyaml:${versions.snakeyaml}" testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-x-content' + exclude group: 'org.elasticsearch', module: 'x-content' } } diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 98dacce01fba4..7cfdba4d33744 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -20,8 +20,8 @@ esplugin { dependencies { compileOnly project(':modules:lang-painless:spi') - api project(':libs:elasticsearch-grok') - api project(':libs:elasticsearch-dissect') + api project(':libs:grok') + api project(':libs:dissect') implementation "org.apache.httpcomponents:httpclient:${versions.httpclient}" implementation "org.apache.httpcomponents:httpcore:${versions.httpcore}" } diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index ac68b565a0fbe..14a6b1e3f5b82 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -38,7 +38,7 @@ testClusters.configureEach { dependencies { api project(":client:rest") - api project(":libs:elasticsearch-ssl-config") + api project(":libs:ssl-config") // for parent/child testing testImplementation project(':modules:parent-join') testImplementation project(':modules:rest-root') diff --git a/modules/runtime-fields-common/build.gradle b/modules/runtime-fields-common/build.gradle index 00bb17df8665e..e743939cbf79e 100644 --- a/modules/runtime-fields-common/build.gradle +++ b/modules/runtime-fields-common/build.gradle @@ -19,8 +19,8 @@ esplugin { dependencies { compileOnly project(':modules:lang-painless:spi') - api project(':libs:elasticsearch-grok') - api project(':libs:elasticsearch-dissect') + api project(':libs:grok') + api project(':libs:dissect') } tasks.named("yamlRestCompatTestTransform").configure({ task -> diff --git a/modules/systemd/build.gradle b/modules/systemd/build.gradle index 28fd36160936a..8eb48e1d5f638 100644 --- a/modules/systemd/build.gradle +++ b/modules/systemd/build.gradle @@ -13,6 +13,6 @@ esplugin { } dependencies { - implementation project(':libs:elasticsearch-native') + implementation project(':libs:native') } diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle index d80b63bec53d8..8dc718a818cec 100644 --- a/modules/transport-netty4/build.gradle +++ b/modules/transport-netty4/build.gradle @@ -35,7 +35,7 @@ configurations { } dependencies { - api project(":libs:elasticsearch-ssl-config") + api project(":libs:ssl-config") // network stack api "io.netty:netty-buffer:${versions.netty}" @@ -244,4 +244,4 @@ tasks.named("thirdPartyAudit").configure { tasks.named('forbiddenApisMain').configure { signaturesFiles += files('forbidden/netty-signatures.txt') -} \ No newline at end of file +} diff --git a/qa/logging-config/build.gradle b/qa/logging-config/build.gradle index 78da8590660f7..4d65c4384afa1 100644 --- a/qa/logging-config/build.gradle +++ b/qa/logging-config/build.gradle @@ -10,7 +10,7 @@ apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.legacy-java-rest-test' dependencies { - testImplementation project(":libs:elasticsearch-x-content") + testImplementation project(":libs:x-content") testImplementation project(":test:framework") } diff --git a/qa/packaging/build.gradle b/qa/packaging/build.gradle index 73b6507490185..f9a903223c88a 100644 --- a/qa/packaging/build.gradle +++ b/qa/packaging/build.gradle @@ -13,7 +13,7 @@ plugins { dependencies { testImplementation project(':server') - testImplementation project(':libs:elasticsearch-core') + testImplementation project(':libs:core') testImplementation(testArtifact(project(':x-pack:plugin:core'))) testImplementation "junit:junit:${versions.junit}" testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}" diff --git a/server/build.gradle b/server/build.gradle index 963b3cfb2e747..e8493751cb327 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -28,17 +28,17 @@ base { dependencies { - api project(':libs:elasticsearch-core') - api project(':libs:elasticsearch-logging') - api project(':libs:elasticsearch-secure-sm') - api project(':libs:elasticsearch-x-content') - api project(":libs:elasticsearch-geo") - api project(":libs:elasticsearch-lz4") - api project(":libs:elasticsearch-plugin-api") - api project(":libs:elasticsearch-plugin-analysis-api") - api project(':libs:elasticsearch-grok') - api project(":libs:elasticsearch-tdigest") - implementation project(":libs:elasticsearch-simdvec") + api project(':libs:core') + api project(':libs:logging') + api project(':libs:secure-sm') + api project(':libs:x-content') + api project(":libs:geo") + api project(":libs:lz4") + api project(":libs:plugin-api") + api project(":libs:plugin-analysis-api") + api project(':libs:grok') + api project(":libs:tdigest") + implementation project(":libs:simdvec") // lucene api "org.apache.lucene:lucene-core:${versions.lucene}" @@ -56,7 +56,7 @@ dependencies { api "org.apache.lucene:lucene-suggest:${versions.lucene}" // utilities - api project(":libs:elasticsearch-cli") + api project(":libs:cli") implementation 'com.carrotsearch:hppc:0.8.1' // precentil ranks aggregation @@ -67,7 +67,7 @@ dependencies { api "org.apache.logging.log4j:log4j-core:${versions.log4j}" // access to native functions - implementation project(':libs:elasticsearch-native') + implementation project(':libs:native') api "co.elastic.logging:log4j2-ecs-layout:${versions.ecsLogging}" api "co.elastic.logging:ecs-logging-core:${versions.ecsLogging}" diff --git a/settings.gradle b/settings.gradle index 39453e8d0935a..25ed048d57253 100644 --- a/settings.gradle +++ b/settings.gradle @@ -155,17 +155,7 @@ addSubProjects('', new File(rootProject.projectDir, 'x-pack/libs')) include projects.toArray(new String[0]) -project(":libs").children.each { libsProject -> - libsProject.name = "elasticsearch-${libsProject.name}" - libsProject.children.each { lp -> - lp.name = lp.name // for :libs:elasticsearch-x-content:impl - } -} -project(":libs:elasticsearch-native:libraries").name = "elasticsearch-native-libraries" - -project(":qa:stable-api").children.each { libsProject -> - libsProject.name = "elasticsearch-${libsProject.name}" -} +project(":libs:native:libraries").name = "native-libraries" project(":test:external-modules").children.each { testProject -> testProject.name = "test-${testProject.name}" diff --git a/test/external-modules/apm-integration/build.gradle b/test/external-modules/apm-integration/build.gradle index 98090f33ee2c7..d0f5f889e9b30 100644 --- a/test/external-modules/apm-integration/build.gradle +++ b/test/external-modules/apm-integration/build.gradle @@ -22,5 +22,5 @@ tasks.named('javaRestTest').configure { dependencies { clusterModules project(':modules:apm') - implementation project(':libs:elasticsearch-logging') + implementation project(':libs:logging') } diff --git a/test/fixtures/geoip-fixture/build.gradle b/test/fixtures/geoip-fixture/build.gradle index f20db481814ea..13d2b6ae88e9c 100644 --- a/test/fixtures/geoip-fixture/build.gradle +++ b/test/fixtures/geoip-fixture/build.gradle @@ -13,8 +13,8 @@ description = 'Fixture for GeoIPv2 service' dependencies { api project(':server') api project(':distribution:tools:geoip-cli') - api project(":libs:elasticsearch-cli") - api project(":libs:elasticsearch-x-content") + api project(":libs:cli") + api project(":libs:x-content") api("junit:junit:${versions.junit}") { exclude module: 'hamcrest-core' } diff --git a/test/framework/build.gradle b/test/framework/build.gradle index 72a8eade3bce0..f130ecf131848 100644 --- a/test/framework/build.gradle +++ b/test/framework/build.gradle @@ -14,9 +14,9 @@ apply plugin: 'elasticsearch.publish' dependencies { api project(":client:rest") api project(':modules:transport-netty4') - api project(':libs:elasticsearch-ssl-config') + api project(':libs:ssl-config') api project(":server") - api project(":libs:elasticsearch-cli") + api project(":libs:cli") api "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" api "junit:junit:${versions.junit}" api "org.hamcrest:hamcrest:${versions.hamcrest}" diff --git a/test/x-content/build.gradle b/test/x-content/build.gradle index 48667eeb58735..281148a0fe819 100644 --- a/test/x-content/build.gradle +++ b/test/x-content/build.gradle @@ -12,7 +12,7 @@ apply plugin: 'elasticsearch.publish' dependencies { api project(":test:framework") - api project(":libs:elasticsearch-x-content") + api project(":libs:x-content") // json schema validation dependencies implementation "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" diff --git a/x-pack/plugin/blob-cache/build.gradle b/x-pack/plugin/blob-cache/build.gradle index ff21b64def768..c5c91a5ef87e3 100644 --- a/x-pack/plugin/blob-cache/build.gradle +++ b/x-pack/plugin/blob-cache/build.gradle @@ -15,5 +15,5 @@ esplugin { } dependencies { - compileOnly project(path: ':libs:elasticsearch-native') + compileOnly project(path: ':libs:native') } diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle index 1ed59d6fe3581..fb4acb0055a8c 100644 --- a/x-pack/plugin/core/build.gradle +++ b/x-pack/plugin/core/build.gradle @@ -36,8 +36,8 @@ configurations { dependencies { compileOnly project(":server") - api project(':libs:elasticsearch-grok') - api project(":libs:elasticsearch-ssl-config") + api project(':libs:grok') + api project(":libs:ssl-config") api "org.apache.httpcomponents:httpclient:${versions.httpclient}" api "org.apache.httpcomponents:httpcore:${versions.httpcore}" api "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}" diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle index 766d0c0f13892..150017ce9e955 100644 --- a/x-pack/plugin/esql/build.gradle +++ b/x-pack/plugin/esql/build.gradle @@ -26,8 +26,8 @@ dependencies { compileOnly project(xpackModule('ml')) implementation project('compute') implementation project('compute:ann') - implementation project(':libs:elasticsearch-dissect') - implementation project(':libs:elasticsearch-grok') + implementation project(':libs:dissect') + implementation project(':libs:grok') implementation project('arrow') // Also contains a dummy processor to allow compilation with unused annotations. diff --git a/x-pack/plugin/esql/qa/testFixtures/build.gradle b/x-pack/plugin/esql/qa/testFixtures/build.gradle index b6ed610406631..903986466b77f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/build.gradle +++ b/x-pack/plugin/esql/qa/testFixtures/build.gradle @@ -5,9 +5,9 @@ dependencies { implementation project(':x-pack:plugin:esql:compute') implementation project(':x-pack:plugin:esql') compileOnly project(path: xpackModule('core')) - implementation project(":libs:elasticsearch-x-content") + implementation project(":libs:x-content") implementation project(':client:rest') - implementation project(':libs:elasticsearch-logging') + implementation project(':libs:logging') implementation project(':test:framework') api(testArtifact(project(xpackModule('esql-core')))) implementation project(':server') diff --git a/x-pack/plugin/inference/build.gradle b/x-pack/plugin/inference/build.gradle index 28e1405cf7b97..6791aad6619d3 100644 --- a/x-pack/plugin/inference/build.gradle +++ b/x-pack/plugin/inference/build.gradle @@ -32,7 +32,7 @@ versions << [ ] dependencies { - implementation project(path: ':libs:elasticsearch-logging') + implementation project(path: ':libs:logging') compileOnly project(":server") compileOnly project(path: xpackModule('core')) testImplementation(testArtifact(project(xpackModule('core')))) diff --git a/x-pack/plugin/ml-package-loader/build.gradle b/x-pack/plugin/ml-package-loader/build.gradle index bdd1e54f20c86..122ad396b507d 100644 --- a/x-pack/plugin/ml-package-loader/build.gradle +++ b/x-pack/plugin/ml-package-loader/build.gradle @@ -18,7 +18,7 @@ esplugin { } dependencies { - implementation project(path: ':libs:elasticsearch-logging') + implementation project(path: ':libs:logging') compileOnly project(":server") compileOnly project(path: xpackModule('core')) testImplementation(testArtifact(project(xpackModule('core')))) diff --git a/x-pack/plugin/ml/build.gradle b/x-pack/plugin/ml/build.gradle index 706d7ea73aea9..e79a771293392 100644 --- a/x-pack/plugin/ml/build.gradle +++ b/x-pack/plugin/ml/build.gradle @@ -76,7 +76,7 @@ dependencies { testImplementation(testArtifact(project(xpackModule('security')))) testImplementation project(path: xpackModule('wildcard')) // ml deps - api project(':libs:elasticsearch-grok') + api project(':libs:grok') api project(':modules:lang-mustache') api "org.apache.commons:commons-math3:3.6.1" api "com.ibm.icu:icu4j:${versions.icu4j}" diff --git a/x-pack/plugin/searchable-snapshots/build.gradle b/x-pack/plugin/searchable-snapshots/build.gradle index 4e309499445e6..747e94b0e8d8d 100644 --- a/x-pack/plugin/searchable-snapshots/build.gradle +++ b/x-pack/plugin/searchable-snapshots/build.gradle @@ -15,7 +15,7 @@ base { dependencies { compileOnly project(path: xpackModule('core')) compileOnly project(path: xpackModule('blob-cache')) - compileOnly project(path: ':libs:elasticsearch-native') + compileOnly project(path: ':libs:native') testImplementation(testArtifact(project(xpackModule('blob-cache')))) internalClusterTestImplementation(testArtifact(project(xpackModule('core')))) internalClusterTestImplementation(project(path: xpackModule('shutdown'))) diff --git a/x-pack/plugin/spatial/build.gradle b/x-pack/plugin/spatial/build.gradle index e111949724844..5bcec68c227ce 100644 --- a/x-pack/plugin/spatial/build.gradle +++ b/x-pack/plugin/spatial/build.gradle @@ -15,7 +15,7 @@ dependencies { compileOnly project(':modules:lang-painless:spi') compileOnly project(path: xpackModule('core')) api "org.apache.lucene:lucene-spatial3d:${versions.lucene}" - api project(":libs:elasticsearch-h3") + api project(":libs:h3") testImplementation(testArtifact(project(xpackModule('core')))) testImplementation project(path: ':modules:percolator') testImplementation project(path: xpackModule('vector-tile')) diff --git a/x-pack/plugin/sql/sql-action/build.gradle b/x-pack/plugin/sql/sql-action/build.gradle index 9a0aefac4e434..60e809df00ae0 100644 --- a/x-pack/plugin/sql/sql-action/build.gradle +++ b/x-pack/plugin/sql/sql-action/build.gradle @@ -11,10 +11,10 @@ dependencies { api(project(':server')) { transitive = false } - api(project(':libs:elasticsearch-core')) { + api(project(':libs:core')) { transitive = false } - api(project(':libs:elasticsearch-x-content')) { + api(project(':libs:x-content')) { transitive = false } api project(':x-pack:plugin:core') @@ -33,4 +33,4 @@ tasks.named('forbiddenApisMain').configure { tasks.named("dependencyLicenses").configure { mapping from: /jackson-.*/, to: 'jackson' mapping from: /lucene-.*/, to: 'lucene' -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/sql-cli/build.gradle b/x-pack/plugin/sql/sql-cli/build.gradle index 1d3a63ec13c98..b9713bcb8e7a3 100644 --- a/x-pack/plugin/sql/sql-cli/build.gradle +++ b/x-pack/plugin/sql/sql-cli/build.gradle @@ -29,7 +29,7 @@ dependencies { api "org.jline:jline-style:${jlineVersion}" api project(':x-pack:plugin:sql:sql-client') - api project(":libs:elasticsearch-cli") + api project(":libs:cli") implementation "net.java.dev.jna:jna:${versions.jna}" testImplementation project(":test:framework") } diff --git a/x-pack/plugin/sql/sql-proto/build.gradle b/x-pack/plugin/sql/sql-proto/build.gradle index de3f3462da85e..2cb1cfa89f033 100644 --- a/x-pack/plugin/sql/sql-proto/build.gradle +++ b/x-pack/plugin/sql/sql-proto/build.gradle @@ -10,9 +10,9 @@ dependencies { api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" api "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" - testImplementation project(":libs:elasticsearch-x-content") + testImplementation project(":libs:x-content") testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-x-content' + exclude group: 'org.elasticsearch', module: 'x-content' } } diff --git a/x-pack/plugin/text-structure/build.gradle b/x-pack/plugin/text-structure/build.gradle index cab7f3ceeaa13..5bb6d8ef50274 100644 --- a/x-pack/plugin/text-structure/build.gradle +++ b/x-pack/plugin/text-structure/build.gradle @@ -12,7 +12,7 @@ base { dependencies { compileOnly project(path: xpackModule('core')) testImplementation(testArtifact(project(xpackModule('core')))) - api project(':libs:elasticsearch-grok') + api project(':libs:grok') api "com.ibm.icu:icu4j:${versions.icu4j}" api "net.sf.supercsv:super-csv:${versions.supercsv}" } diff --git a/x-pack/plugin/transform/qa/common/build.gradle b/x-pack/plugin/transform/qa/common/build.gradle index 9e7abfa2f977e..28e4068d31c6b 100644 --- a/x-pack/plugin/transform/qa/common/build.gradle +++ b/x-pack/plugin/transform/qa/common/build.gradle @@ -1,7 +1,7 @@ apply plugin: 'elasticsearch.internal-java-rest-test' dependencies { - api project(':libs:elasticsearch-x-content') + api project(':libs:x-content') api project(':test:framework') api project(xpackModule('core')) } From b6d2d4bc10233649cc9af7e7d7598c1b8b7355f7 Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Tue, 29 Oct 2024 16:18:42 -0400 Subject: [PATCH 15/18] ES|QL CCS uses skip_unavailable setting for handling disconnected remote clusters (#115266) As part of ES|QL planning of a cross-cluster search, a field-caps call is done to each cluster and, if an ENRICH command is present, the enrich policy-resolve API is called on each remote. If a remote cluster cannot be connected to in these calls, the outcome depends on the skip_unavailable setting. For skip_unavailable=false clusters, the error is fatal and the error will immediately be propagated back to the client with a top level error message with a 500 HTTP status response code. For skip_unavailable=true clusters, the error is not fatal. The error will be trapped, recorded in the EsqlExecutionInfo object for the query, marking the cluster as SKIPPED. If the user requested CCS metadata to be included, the cluster status and connection failure will be present in the _clusters/details section of the response. If no clusters can be contacted, if they are all marked as skip_unavailable=true, no error will be returned. Instead a 200 HTTP status will be returned with no column and no values. If the include_ccs_metadata: true setting was included on the query, the errors will listed in the _clusters metadata section. (Note: this is also how the _search endpoint works for CCS.) Partially addresses https://github.com/elastic/elasticsearch/issues/114531 --- docs/changelog/115266.yaml | 6 + .../org/elasticsearch/TransportVersions.java | 1 + ...ossClusterEnrichUnavailableClustersIT.java | 690 ++++++++++++++++++ ...CrossClusterQueryUnavailableRemotesIT.java | 525 +++++++++++++ .../esql/action/CrossClustersQueryIT.java | 14 +- .../xpack/esql/action/EsqlExecutionInfo.java | 49 +- .../xpack/esql/analysis/EnrichResolution.java | 9 + .../esql/enrich/EnrichPolicyResolver.java | 74 +- .../xpack/esql/index/IndexResolution.java | 19 +- .../xpack/esql/session/EsqlSession.java | 216 +++++- .../xpack/esql/session/IndexResolver.java | 11 +- .../session/NoClustersToSearchException.java | 15 + .../esql/action/EsqlQueryResponseTests.java | 3 + .../esql/plugin/ComputeListenerTests.java | 1 + .../xpack/esql/session/EsqlSessionTests.java | 227 ++++-- .../esql/session/IndexResolverTests.java | 21 +- .../RemoteClusterSecurityEsqlIT.java | 82 ++- 17 files changed, 1819 insertions(+), 144 deletions(-) create mode 100644 docs/changelog/115266.yaml create mode 100644 x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java create mode 100644 x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/NoClustersToSearchException.java diff --git a/docs/changelog/115266.yaml b/docs/changelog/115266.yaml new file mode 100644 index 0000000000000..1d7fb1368c0e8 --- /dev/null +++ b/docs/changelog/115266.yaml @@ -0,0 +1,6 @@ +pr: 115266 +summary: ES|QL CCS uses `skip_unavailable` setting for handling disconnected remote + clusters +area: ES|QL +type: enhancement +issues: [ 114531 ] diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 7bf3204b7e1a6..ea3e649de9ef8 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -185,6 +185,7 @@ static TransportVersion def(int id) { public static final TransportVersion INDEX_REQUEST_REMOVE_METERING = def(8_780_00_0); public static final TransportVersion CPU_STAT_STRING_PARSING = def(8_781_00_0); public static final TransportVersion QUERY_RULES_RETRIEVER = def(8_782_00_0); + public static final TransportVersion ESQL_CCS_EXEC_INFO_WITH_FAILURES = def(8_783_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java new file mode 100644 index 0000000000000..d142752d0c408 --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java @@ -0,0 +1,690 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.action; + +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.ingest.common.IngestCommonPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.reindex.ReindexPlugin; +import org.elasticsearch.test.AbstractMultiClustersTestCase; +import org.elasticsearch.transport.RemoteClusterAware; +import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyAction; +import org.elasticsearch.xpack.core.enrich.action.PutEnrichPolicyAction; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.plan.logical.Enrich; +import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList; +import static org.elasticsearch.xpack.esql.action.CrossClustersEnrichIT.enrichHosts; +import static org.elasticsearch.xpack.esql.action.CrossClustersEnrichIT.enrichVendors; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +/** + * This IT test is the dual of CrossClustersEnrichIT, which tests "happy path" + * and this one tests unavailable cluster scenarios using (most of) the same tests. + */ +public class CrossClusterEnrichUnavailableClustersIT extends AbstractMultiClustersTestCase { + + public static String REMOTE_CLUSTER_1 = "c1"; + public static String REMOTE_CLUSTER_2 = "c2"; + + @Override + protected Collection remoteClusterAlias() { + return List.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2); + } + + @Override + protected boolean reuseClusters() { + return false; + } + + private Collection allClusters() { + return CollectionUtils.appendToCopy(remoteClusterAlias(), LOCAL_CLUSTER); + } + + @Override + protected Collection> nodePlugins(String clusterAlias) { + List> plugins = new ArrayList<>(super.nodePlugins(clusterAlias)); + plugins.add(EsqlPlugin.class); + plugins.add(CrossClustersEnrichIT.LocalStateEnrich.class); + plugins.add(IngestCommonPlugin.class); + plugins.add(ReindexPlugin.class); + return plugins; + } + + @Override + protected Settings nodeSettings() { + return Settings.builder().put(super.nodeSettings()).put(XPackSettings.SECURITY_ENABLED.getKey(), false).build(); + } + + @Before + public void setupHostsEnrich() { + // the hosts policy are identical on every node + Map allHosts = Map.of( + "192.168.1.2", + "Windows", + "192.168.1.3", + "MacOS", + "192.168.1.4", + "Linux", + "192.168.1.5", + "Android", + "192.168.1.6", + "iOS", + "192.168.1.7", + "Windows", + "192.168.1.8", + "MacOS", + "192.168.1.9", + "Linux", + "192.168.1.10", + "Linux", + "192.168.1.11", + "Windows" + ); + for (String cluster : allClusters()) { + Client client = client(cluster); + client.admin().indices().prepareCreate("hosts").setMapping("ip", "type=ip", "os", "type=keyword").get(); + for (Map.Entry h : allHosts.entrySet()) { + client.prepareIndex("hosts").setSource("ip", h.getKey(), "os", h.getValue()).get(); + } + client.admin().indices().prepareRefresh("hosts").get(); + client.execute( + PutEnrichPolicyAction.INSTANCE, + new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "hosts", CrossClustersEnrichIT.hostPolicy) + ).actionGet(); + client.execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "hosts")) + .actionGet(); + assertAcked(client.admin().indices().prepareDelete("hosts")); + } + } + + @Before + public void setupVendorPolicy() { + var localVendors = Map.of("Windows", "Microsoft", "MacOS", "Apple", "iOS", "Apple", "Android", "Samsung", "Linux", "Redhat"); + var c1Vendors = Map.of("Windows", "Microsoft", "MacOS", "Apple", "iOS", "Apple", "Android", "Google", "Linux", "Suse"); + var c2Vendors = Map.of("Windows", "Microsoft", "MacOS", "Apple", "iOS", "Apple", "Android", "Sony", "Linux", "Ubuntu"); + var vendors = Map.of(LOCAL_CLUSTER, localVendors, "c1", c1Vendors, "c2", c2Vendors); + for (Map.Entry> e : vendors.entrySet()) { + Client client = client(e.getKey()); + client.admin().indices().prepareCreate("vendors").setMapping("os", "type=keyword", "vendor", "type=keyword").get(); + for (Map.Entry v : e.getValue().entrySet()) { + client.prepareIndex("vendors").setSource("os", v.getKey(), "vendor", v.getValue()).get(); + } + client.admin().indices().prepareRefresh("vendors").get(); + client.execute( + PutEnrichPolicyAction.INSTANCE, + new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "vendors", CrossClustersEnrichIT.vendorPolicy) + ).actionGet(); + client.execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "vendors")) + .actionGet(); + assertAcked(client.admin().indices().prepareDelete("vendors")); + } + } + + @Before + public void setupEventsIndices() { + record Event(long timestamp, String user, String host) {} + + List e0 = List.of( + new Event(1, "matthew", "192.168.1.3"), + new Event(2, "simon", "192.168.1.5"), + new Event(3, "park", "192.168.1.2"), + new Event(4, "andrew", "192.168.1.7"), + new Event(5, "simon", "192.168.1.20"), + new Event(6, "kevin", "192.168.1.2"), + new Event(7, "akio", "192.168.1.5"), + new Event(8, "luke", "192.168.1.2"), + new Event(9, "jack", "192.168.1.4") + ); + List e1 = List.of( + new Event(1, "andres", "192.168.1.2"), + new Event(2, "sergio", "192.168.1.6"), + new Event(3, "kylian", "192.168.1.8"), + new Event(4, "andrew", "192.168.1.9"), + new Event(5, "jack", "192.168.1.3"), + new Event(6, "kevin", "192.168.1.4"), + new Event(7, "akio", "192.168.1.7"), + new Event(8, "kevin", "192.168.1.21"), + new Event(9, "andres", "192.168.1.8") + ); + List e2 = List.of( + new Event(1, "park", "192.168.1.25"), + new Event(2, "akio", "192.168.1.5"), + new Event(3, "park", "192.168.1.2"), + new Event(4, "kevin", "192.168.1.3") + ); + for (var c : Map.of(LOCAL_CLUSTER, e0, "c1", e1, "c2", e2).entrySet()) { + Client client = client(c.getKey()); + client.admin() + .indices() + .prepareCreate("events") + .setMapping("timestamp", "type=long", "user", "type=keyword", "host", "type=ip") + .get(); + for (var e : c.getValue()) { + client.prepareIndex("events").setSource("timestamp", e.timestamp, "user", e.user, "host", e.host).get(); + } + client.admin().indices().prepareRefresh("events").get(); + } + } + + public void testEnrichWithHostsPolicyAndDisconnectedRemotesWithSkipUnavailableTrue() throws IOException { + setSkipUnavailable(REMOTE_CLUSTER_1, true); + setSkipUnavailable(REMOTE_CLUSTER_2, true); + + try { + // close remote-cluster-1 so that it is unavailable + cluster(REMOTE_CLUSTER_1).close(); + + Tuple includeCCSMetadata = CrossClustersEnrichIT.randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); + + { + Enrich.Mode mode = randomFrom(Enrich.Mode.values()); + String query = "FROM *:events | eval ip= TO_STR(host) | " + enrichHosts(mode) + " | stats c = COUNT(*) by os | SORT os"; + try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) { + List> rows = getValuesList(resp); + assertThat(rows.size(), greaterThanOrEqualTo(1)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertCCSExecutionInfoDetails(executionInfo); + + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2))); + + EsqlExecutionInfo.Cluster cluster1 = executionInfo.getCluster(REMOTE_CLUSTER_1); + assertThat(cluster1.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(cluster1.getTotalShards(), equalTo(0)); + assertThat(cluster1.getSuccessfulShards(), equalTo(0)); + assertThat(cluster1.getSkippedShards(), equalTo(0)); + assertThat(cluster1.getFailedShards(), equalTo(0)); + + EsqlExecutionInfo.Cluster cluster2 = executionInfo.getCluster(REMOTE_CLUSTER_2); + assertThat(cluster2.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(cluster2.getTotalShards(), greaterThanOrEqualTo(0)); + assertThat(cluster2.getSuccessfulShards(), equalTo(cluster2.getSuccessfulShards())); + assertThat(cluster2.getSkippedShards(), equalTo(0)); + assertThat(cluster2.getFailedShards(), equalTo(0)); + } + } + + // close remote-cluster-2 so that it is also unavailable + cluster(REMOTE_CLUSTER_2).close(); + + { + Enrich.Mode mode = randomFrom(Enrich.Mode.values()); + String query = "FROM *:events | eval ip= TO_STR(host) | " + enrichHosts(mode) + " | stats c = COUNT(*) by os | SORT os"; + try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) { + List columns = resp.columns(); + assertThat(columns.size(), equalTo(1)); + // column from an empty result should be {"name":"","type":"null"} + assertThat(columns.get(0).name(), equalTo("")); + assertThat(columns.get(0).type(), equalTo(DataType.NULL)); + + List> rows = getValuesList(resp); + assertThat(rows.size(), equalTo(0)); + + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertCCSExecutionInfoDetails(executionInfo); + + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2))); + + EsqlExecutionInfo.Cluster cluster1 = executionInfo.getCluster(REMOTE_CLUSTER_1); + assertThat(cluster1.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(cluster1.getTotalShards(), equalTo(0)); + assertThat(cluster1.getSuccessfulShards(), equalTo(0)); + assertThat(cluster1.getSkippedShards(), equalTo(0)); + assertThat(cluster1.getFailedShards(), equalTo(0)); + + EsqlExecutionInfo.Cluster cluster2 = executionInfo.getCluster(REMOTE_CLUSTER_2); + assertThat(cluster2.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(cluster2.getTotalShards(), equalTo(0)); + assertThat(cluster2.getSuccessfulShards(), equalTo(0)); + assertThat(cluster2.getSkippedShards(), equalTo(0)); + assertThat(cluster2.getFailedShards(), equalTo(0)); + } + } + } finally { + clearSkipUnavailable(); + } + } + + public void testEnrichWithHostsPolicyAndDisconnectedRemotesWithSkipUnavailableFalse() throws IOException { + setSkipUnavailable(REMOTE_CLUSTER_1, true); + setSkipUnavailable(REMOTE_CLUSTER_2, false); + + try { + // close remote-cluster-1 so that it is unavailable + cluster(REMOTE_CLUSTER_1).close(); + + Tuple includeCCSMetadata = CrossClustersEnrichIT.randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); + + { + Enrich.Mode mode = randomFrom(Enrich.Mode.values()); + String query = "FROM *:events | EVAL ip= TO_STR(host) | " + enrichHosts(mode) + " | STATS c = COUNT(*) by os | SORT os"; + try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) { + List> rows = getValuesList(resp); + assertThat(rows.size(), greaterThanOrEqualTo(1)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertCCSExecutionInfoDetails(executionInfo); + + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2))); + + EsqlExecutionInfo.Cluster cluster1 = executionInfo.getCluster(REMOTE_CLUSTER_1); + assertThat(cluster1.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(cluster1.getTotalShards(), equalTo(0)); + assertThat(cluster1.getSuccessfulShards(), equalTo(0)); + assertThat(cluster1.getSkippedShards(), equalTo(0)); + assertThat(cluster1.getFailedShards(), equalTo(0)); + + EsqlExecutionInfo.Cluster cluster2 = executionInfo.getCluster(REMOTE_CLUSTER_2); + assertThat(cluster2.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(cluster2.getTotalShards(), greaterThanOrEqualTo(0)); + assertThat(cluster2.getSuccessfulShards(), equalTo(cluster2.getSuccessfulShards())); + assertThat(cluster2.getSkippedShards(), equalTo(0)); + assertThat(cluster2.getFailedShards(), equalTo(0)); + } + } + + // close remote-cluster-2 so that it is also unavailable + cluster(REMOTE_CLUSTER_2).close(); + { + Enrich.Mode mode = randomFrom(Enrich.Mode.values()); + String query = "FROM *:events | eval ip= TO_STR(host) | " + enrichHosts(mode) + " | stats c = COUNT(*) by os | SORT os"; + Exception exception = expectThrows(Exception.class, () -> runQuery(query, requestIncludeMeta)); + assertTrue(ExceptionsHelper.isRemoteUnavailableException(exception)); + } + } finally { + clearSkipUnavailable(); + } + } + + public void testEnrichTwiceThenAggsWithUnavailableRemotes() throws IOException { + Tuple includeCCSMetadata = CrossClustersEnrichIT.randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); + + boolean skipUnavailableRemote1 = randomBoolean(); + setSkipUnavailable(REMOTE_CLUSTER_1, skipUnavailableRemote1); + setSkipUnavailable(REMOTE_CLUSTER_2, true); + + try { + // close remote-cluster-2 so that it is unavailable + cluster(REMOTE_CLUSTER_2).close(); + + for (var hostMode : Enrich.Mode.values()) { + String query = String.format(Locale.ROOT, """ + FROM *:events,events + | eval ip= TO_STR(host) + | %s + | %s + | stats c = COUNT(*) by vendor + | sort vendor + """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.COORDINATOR)); + try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) { + assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", REMOTE_CLUSTER_1, REMOTE_CLUSTER_2))); + assertCCSExecutionInfoDetails(executionInfo); + + EsqlExecutionInfo.Cluster cluster1 = executionInfo.getCluster(REMOTE_CLUSTER_1); + assertThat(cluster1.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(cluster1.getTotalShards(), greaterThanOrEqualTo(0)); + assertThat(cluster1.getSuccessfulShards(), equalTo(cluster1.getSuccessfulShards())); + assertThat(cluster1.getSkippedShards(), equalTo(0)); + assertThat(cluster1.getFailedShards(), equalTo(0)); + + EsqlExecutionInfo.Cluster cluster2 = executionInfo.getCluster(REMOTE_CLUSTER_2); + assertThat(cluster2.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(cluster2.getTotalShards(), equalTo(0)); + assertThat(cluster2.getSuccessfulShards(), equalTo(0)); + assertThat(cluster2.getSkippedShards(), equalTo(0)); + assertThat(cluster2.getFailedShards(), equalTo(0)); + + EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); + assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(localCluster.getTotalShards(), greaterThan(0)); + assertThat(localCluster.getSuccessfulShards(), equalTo(localCluster.getTotalShards())); + assertThat(localCluster.getSkippedShards(), equalTo(0)); + assertThat(localCluster.getFailedShards(), equalTo(0)); + } + } + + // close remote-cluster-1 so that it is also unavailable + cluster(REMOTE_CLUSTER_1).close(); + + for (var hostMode : Enrich.Mode.values()) { + String query = String.format(Locale.ROOT, """ + FROM *:events,events + | eval ip= TO_STR(host) + | %s + | %s + | stats c = COUNT(*) by vendor + | sort vendor + """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.COORDINATOR)); + if (skipUnavailableRemote1 == false) { + Exception exception = expectThrows(Exception.class, () -> runQuery(query, requestIncludeMeta)); + assertTrue(ExceptionsHelper.isRemoteUnavailableException(exception)); + } else { + try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) { + assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", REMOTE_CLUSTER_1, REMOTE_CLUSTER_2))); + assertCCSExecutionInfoDetails(executionInfo); + + EsqlExecutionInfo.Cluster cluster1 = executionInfo.getCluster(REMOTE_CLUSTER_1); + assertThat(cluster1.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(cluster1.getTotalShards(), equalTo(0)); + assertThat(cluster1.getSuccessfulShards(), equalTo(0)); + assertThat(cluster1.getSkippedShards(), equalTo(0)); + assertThat(cluster1.getFailedShards(), equalTo(0)); + assertThat(cluster1.getTook().millis(), greaterThanOrEqualTo(0L)); + + EsqlExecutionInfo.Cluster cluster2 = executionInfo.getCluster(REMOTE_CLUSTER_2); + assertThat(cluster2.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(cluster2.getTotalShards(), equalTo(0)); + assertThat(cluster2.getSuccessfulShards(), equalTo(0)); + assertThat(cluster2.getSkippedShards(), equalTo(0)); + assertThat(cluster2.getFailedShards(), equalTo(0)); + + EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); + assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(localCluster.getTotalShards(), greaterThan(0)); + assertThat(localCluster.getSuccessfulShards(), equalTo(localCluster.getTotalShards())); + assertThat(localCluster.getSkippedShards(), equalTo(0)); + assertThat(localCluster.getFailedShards(), equalTo(0)); + } + } + } + } finally { + clearSkipUnavailable(); + } + } + + public void testEnrichCoordinatorThenAnyWithSingleUnavailableRemoteAndLocal() throws IOException { + Tuple includeCCSMetadata = CrossClustersEnrichIT.randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); + + boolean skipUnavailableRemote1 = randomBoolean(); + setSkipUnavailable(REMOTE_CLUSTER_1, skipUnavailableRemote1); + + try { + // close remote-cluster-1 so that it is unavailable + cluster(REMOTE_CLUSTER_1).close(); + String query = String.format(Locale.ROOT, """ + FROM %s:events,events + | eval ip= TO_STR(host) + | %s + | %s + | stats c = COUNT(*) by vendor + | sort vendor + """, REMOTE_CLUSTER_1, enrichHosts(Enrich.Mode.COORDINATOR), enrichVendors(Enrich.Mode.ANY)); + if (skipUnavailableRemote1 == false) { + Exception exception = expectThrows(Exception.class, () -> runQuery(query, requestIncludeMeta)); + assertTrue(ExceptionsHelper.isRemoteUnavailableException(exception)); + } else { + try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) { + assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertThat( + executionInfo.clusterAliases(), + equalTo(Set.of(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, REMOTE_CLUSTER_1)) + ); + assertCCSExecutionInfoDetails(executionInfo); + + EsqlExecutionInfo.Cluster cluster1 = executionInfo.getCluster(REMOTE_CLUSTER_1); + assertThat(cluster1.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(cluster1.getTotalShards(), equalTo(0)); + assertThat(cluster1.getSuccessfulShards(), equalTo(0)); + assertThat(cluster1.getSkippedShards(), equalTo(0)); + assertThat(cluster1.getFailedShards(), equalTo(0)); + + EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); + assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(localCluster.getTotalShards(), greaterThan(0)); + assertThat(localCluster.getSuccessfulShards(), equalTo(localCluster.getTotalShards())); + assertThat(localCluster.getSkippedShards(), equalTo(0)); + assertThat(localCluster.getFailedShards(), equalTo(0)); + } + } + } finally { + clearSkipUnavailable(); + } + } + + public void testEnrichCoordinatorThenAnyWithSingleUnavailableRemoteAndNotLocal() throws IOException { + Tuple includeCCSMetadata = CrossClustersEnrichIT.randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); + + boolean skipUnavailableRemote1 = randomBoolean(); + setSkipUnavailable(REMOTE_CLUSTER_1, skipUnavailableRemote1); + + try { + // close remote-cluster-1 so that it is unavailable + cluster(REMOTE_CLUSTER_1).close(); + String query = String.format(Locale.ROOT, """ + FROM %s:events + | eval ip= TO_STR(host) + | %s + | %s + | stats c = COUNT(*) by vendor + | sort vendor + """, REMOTE_CLUSTER_1, enrichHosts(Enrich.Mode.COORDINATOR), enrichVendors(Enrich.Mode.ANY)); + if (skipUnavailableRemote1 == false) { + Exception exception = expectThrows(Exception.class, () -> runQuery(query, requestIncludeMeta)); + assertTrue(ExceptionsHelper.isRemoteUnavailableException(exception)); + } else { + try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) { + List columns = resp.columns(); + assertThat(columns.size(), equalTo(1)); + // column from an empty result should be {"name":"","type":"null"} + assertThat(columns.get(0).name(), equalTo("")); + assertThat(columns.get(0).type(), equalTo(DataType.NULL)); + + assertThat(getValuesList(resp).size(), equalTo(0)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1))); + assertCCSExecutionInfoDetails(executionInfo); + + EsqlExecutionInfo.Cluster cluster1 = executionInfo.getCluster(REMOTE_CLUSTER_1); + assertThat(cluster1.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(cluster1.getTotalShards(), equalTo(0)); + assertThat(cluster1.getSuccessfulShards(), equalTo(0)); + assertThat(cluster1.getSkippedShards(), equalTo(0)); + assertThat(cluster1.getFailedShards(), equalTo(0)); + } + } + } finally { + clearSkipUnavailable(); + } + } + + public void testEnrichRemoteWithVendor() throws IOException { + Tuple includeCCSMetadata = CrossClustersEnrichIT.randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); + + boolean skipUnavailableRemote2 = randomBoolean(); + setSkipUnavailable(REMOTE_CLUSTER_1, true); + setSkipUnavailable(REMOTE_CLUSTER_2, skipUnavailableRemote2); + + try { + // close remote-cluster-1 so that it is unavailable + cluster(REMOTE_CLUSTER_1).close(); + + for (Enrich.Mode hostMode : List.of(Enrich.Mode.ANY, Enrich.Mode.REMOTE)) { + var query = String.format(Locale.ROOT, """ + FROM *:events,events + | eval ip= TO_STR(host) + | %s + | %s + | stats c = COUNT(*) by vendor + | sort vendor + """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.REMOTE)); + try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) { + assertThat(getValuesList(resp).size(), greaterThan(0)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertThat( + executionInfo.clusterAliases(), + equalTo(Set.of(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, REMOTE_CLUSTER_1, REMOTE_CLUSTER_2)) + ); + assertCCSExecutionInfoDetails(executionInfo); + + EsqlExecutionInfo.Cluster cluster1 = executionInfo.getCluster(REMOTE_CLUSTER_1); + assertThat(cluster1.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(cluster1.getTotalShards(), equalTo(0)); + assertThat(cluster1.getSuccessfulShards(), equalTo(0)); + assertThat(cluster1.getSkippedShards(), equalTo(0)); + assertThat(cluster1.getFailedShards(), equalTo(0)); + assertThat(cluster1.getTook().millis(), greaterThanOrEqualTo(0L)); + + EsqlExecutionInfo.Cluster cluster2 = executionInfo.getCluster(REMOTE_CLUSTER_2); + assertThat(cluster2.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(cluster2.getTotalShards(), greaterThan(0)); + assertThat(cluster2.getSuccessfulShards(), equalTo(cluster2.getSuccessfulShards())); + assertThat(cluster2.getSkippedShards(), equalTo(0)); + assertThat(cluster2.getFailedShards(), equalTo(0)); + + EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); + assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(localCluster.getTotalShards(), greaterThan(0)); + assertThat(localCluster.getSuccessfulShards(), equalTo(localCluster.getTotalShards())); + assertThat(localCluster.getSkippedShards(), equalTo(0)); + assertThat(localCluster.getFailedShards(), equalTo(0)); + } + } + + // close remote-cluster-2 so that it is also unavailable + cluster(REMOTE_CLUSTER_2).close(); + + for (Enrich.Mode hostMode : List.of(Enrich.Mode.ANY, Enrich.Mode.REMOTE)) { + var query = String.format(Locale.ROOT, """ + FROM *:events,events + | eval ip= TO_STR(host) + | %s + | %s + | stats c = COUNT(*) by vendor + | sort vendor + """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.REMOTE)); + if (skipUnavailableRemote2 == false) { + Exception exception = expectThrows(Exception.class, () -> runQuery(query, requestIncludeMeta)); + assertTrue(ExceptionsHelper.isRemoteUnavailableException(exception)); + } else { + + try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) { + assertThat(getValuesList(resp).size(), greaterThan(0)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertThat( + executionInfo.clusterAliases(), + equalTo(Set.of(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, REMOTE_CLUSTER_1, REMOTE_CLUSTER_2)) + ); + assertCCSExecutionInfoDetails(executionInfo); + + EsqlExecutionInfo.Cluster cluster1 = executionInfo.getCluster(REMOTE_CLUSTER_1); + assertThat(cluster1.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(cluster1.getTotalShards(), equalTo(0)); + assertThat(cluster1.getSuccessfulShards(), equalTo(0)); + assertThat(cluster1.getSkippedShards(), equalTo(0)); + assertThat(cluster1.getFailedShards(), equalTo(0)); + assertThat(cluster1.getTook().millis(), greaterThanOrEqualTo(0L)); + + EsqlExecutionInfo.Cluster cluster2 = executionInfo.getCluster(REMOTE_CLUSTER_2); + assertThat(cluster2.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(cluster2.getTotalShards(), equalTo(0)); + assertThat(cluster2.getSuccessfulShards(), equalTo(0)); + assertThat(cluster2.getSkippedShards(), equalTo(0)); + assertThat(cluster2.getFailedShards(), equalTo(0)); + + EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); + assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(localCluster.getTotalShards(), greaterThan(0)); + assertThat(localCluster.getSuccessfulShards(), equalTo(localCluster.getTotalShards())); + assertThat(localCluster.getSkippedShards(), equalTo(0)); + assertThat(localCluster.getFailedShards(), equalTo(0)); + } + } + } + } finally { + clearSkipUnavailable(); + } + } + + protected EsqlQueryResponse runQuery(String query, Boolean ccsMetadataInResponse) { + EsqlQueryRequest request = EsqlQueryRequest.syncEsqlQueryRequest(); + request.query(query); + request.pragmas(AbstractEsqlIntegTestCase.randomPragmas()); + if (randomBoolean()) { + request.profile(true); + } + if (ccsMetadataInResponse != null) { + request.includeCCSMetadata(ccsMetadataInResponse); + } + return client(LOCAL_CLUSTER).execute(EsqlQueryAction.INSTANCE, request).actionGet(30, TimeUnit.SECONDS); + } + + private static void assertCCSExecutionInfoDetails(EsqlExecutionInfo executionInfo) { + assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); + assertTrue(executionInfo.isCrossClusterSearch()); + + for (String clusterAlias : executionInfo.clusterAliases()) { + EsqlExecutionInfo.Cluster cluster = executionInfo.getCluster(clusterAlias); + assertThat(cluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(cluster.getTook().millis(), lessThanOrEqualTo(executionInfo.overallTook().millis())); + } + } + + private void setSkipUnavailable(String clusterAlias, boolean skip) { + client(LOCAL_CLUSTER).admin() + .cluster() + .prepareUpdateSettings(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT) + .setPersistentSettings(Settings.builder().put("cluster.remote." + clusterAlias + ".skip_unavailable", skip).build()) + .get(); + } + + private void clearSkipUnavailable() { + Settings.Builder settingsBuilder = Settings.builder() + .putNull("cluster.remote." + REMOTE_CLUSTER_1 + ".skip_unavailable") + .putNull("cluster.remote." + REMOTE_CLUSTER_2 + ".skip_unavailable"); + client(LOCAL_CLUSTER).admin() + .cluster() + .prepareUpdateSettings(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT) + .setPersistentSettings(settingsBuilder.build()) + .get(); + } +} diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java new file mode 100644 index 0000000000000..0f1aa8541fdd9 --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java @@ -0,0 +1,525 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.action; + +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.compute.operator.exchange.ExchangeService; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.AbstractMultiClustersTestCase; +import org.elasticsearch.test.XContentTestUtils; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +public class CrossClusterQueryUnavailableRemotesIT extends AbstractMultiClustersTestCase { + private static final String REMOTE_CLUSTER_1 = "cluster-a"; + private static final String REMOTE_CLUSTER_2 = "cluster-b"; + + @Override + protected Collection remoteClusterAlias() { + return List.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2); + } + + @Override + protected boolean reuseClusters() { + return false; + } + + @Override + protected Collection> nodePlugins(String clusterAlias) { + List> plugins = new ArrayList<>(super.nodePlugins(clusterAlias)); + plugins.add(EsqlPlugin.class); + plugins.add(org.elasticsearch.xpack.esql.action.CrossClustersQueryIT.InternalExchangePlugin.class); + return plugins; + } + + public static class InternalExchangePlugin extends Plugin { + @Override + public List> getSettings() { + return List.of( + Setting.timeSetting( + ExchangeService.INACTIVE_SINKS_INTERVAL_SETTING, + TimeValue.timeValueSeconds(30), + Setting.Property.NodeScope + ) + ); + } + } + + public void testCCSAgainstDisconnectedRemoteWithSkipUnavailableTrue() throws Exception { + int numClusters = 3; + Map testClusterInfo = setupClusters(numClusters); + int localNumShards = (Integer) testClusterInfo.get("local.num_shards"); + int remote2NumShards = (Integer) testClusterInfo.get("remote2.num_shards"); + setSkipUnavailable(REMOTE_CLUSTER_1, true); + setSkipUnavailable(REMOTE_CLUSTER_2, true); + + Tuple includeCCSMetadata = randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); + + try { + // close remote-cluster-1 so that it is unavailable + cluster(REMOTE_CLUSTER_1).close(); + + try (EsqlQueryResponse resp = runQuery("FROM logs-*,*:logs-* | STATS sum (v)", requestIncludeMeta)) { + List> values = getValuesList(resp); + assertThat(values, hasSize(1)); + assertThat(values.get(0), equalTo(List.of(330L))); + + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertNotNull(executionInfo); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + long overallTookMillis = executionInfo.overallTook().millis(); + assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + + assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2, LOCAL_CLUSTER))); + + EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(REMOTE_CLUSTER_1); + assertThat(remote1Cluster.getIndexExpression(), equalTo("logs-*")); + assertThat(remote1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(remote1Cluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(remote1Cluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertThat(remote1Cluster.getTotalShards(), equalTo(0)); + assertThat(remote1Cluster.getSuccessfulShards(), equalTo(0)); + assertThat(remote1Cluster.getSkippedShards(), equalTo(0)); + assertThat(remote1Cluster.getFailedShards(), equalTo(0)); + + EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(REMOTE_CLUSTER_2); + assertThat(remote2Cluster.getIndexExpression(), equalTo("logs-*")); + assertThat(remote2Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(remote2Cluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(remote2Cluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertThat(remote2Cluster.getTotalShards(), equalTo(remote2NumShards)); + assertThat(remote2Cluster.getSuccessfulShards(), equalTo(remote2NumShards)); + assertThat(remote2Cluster.getSkippedShards(), equalTo(0)); + assertThat(remote2Cluster.getFailedShards(), equalTo(0)); + + EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER); + assertThat(localCluster.getIndexExpression(), equalTo("logs-*")); + assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertThat(localCluster.getTotalShards(), equalTo(localNumShards)); + assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards)); + assertThat(localCluster.getSkippedShards(), equalTo(0)); + assertThat(localCluster.getFailedShards(), equalTo(0)); + + // ensure that the _clusters metadata is present only if requested + assertClusterMetadataInResponse(resp, responseExpectMeta); + } + + // scenario where there are no indices to match because + // 1) the local cluster indexExpression and REMOTE_CLUSTER_2 indexExpression match no indices + // 2) the REMOTE_CLUSTER_1 is unavailable + // 3) both remotes are marked as skip_un=true + String query = "FROM nomatch*," + REMOTE_CLUSTER_1 + ":logs-*," + REMOTE_CLUSTER_2 + ":nomatch* | STATS sum (v)"; + try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) { + List> values = getValuesList(resp); + assertThat(values, hasSize(0)); + + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertNotNull(executionInfo); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + long overallTookMillis = executionInfo.overallTook().millis(); + assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + + assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2, LOCAL_CLUSTER))); + + EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(REMOTE_CLUSTER_1); + assertThat(remote1Cluster.getIndexExpression(), equalTo("logs-*")); + assertThat(remote1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(remote1Cluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(remote1Cluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertThat(remote1Cluster.getTotalShards(), equalTo(0)); + assertThat(remote1Cluster.getSuccessfulShards(), equalTo(0)); + assertThat(remote1Cluster.getSkippedShards(), equalTo(0)); + assertThat(remote1Cluster.getFailedShards(), equalTo(0)); + + EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(REMOTE_CLUSTER_2); + assertThat(remote2Cluster.getIndexExpression(), equalTo("nomatch*")); + assertThat(remote2Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(remote2Cluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(remote2Cluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertThat(remote2Cluster.getTotalShards(), equalTo(0)); + assertThat(remote2Cluster.getSuccessfulShards(), equalTo(0)); + assertThat(remote2Cluster.getSkippedShards(), equalTo(0)); + assertThat(remote2Cluster.getFailedShards(), equalTo(0)); + + EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER); + assertThat(localCluster.getIndexExpression(), equalTo("nomatch*")); + // local cluster should never be marked as SKIPPED + assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertThat(localCluster.getTotalShards(), equalTo(0)); + assertThat(localCluster.getSuccessfulShards(), equalTo(0)); + assertThat(localCluster.getSkippedShards(), equalTo(0)); + assertThat(localCluster.getFailedShards(), equalTo(0)); + + // ensure that the _clusters metadata is present only if requested + assertClusterMetadataInResponse(resp, responseExpectMeta); + } + + // close remote-cluster-2 so that it is also unavailable + cluster(REMOTE_CLUSTER_2).close(); + + try (EsqlQueryResponse resp = runQuery("FROM logs-*,*:logs-* | STATS sum (v)", requestIncludeMeta)) { + List> values = getValuesList(resp); + assertThat(values, hasSize(1)); + assertThat(values.get(0), equalTo(List.of(45L))); + + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertNotNull(executionInfo); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + long overallTookMillis = executionInfo.overallTook().millis(); + assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + + assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2, LOCAL_CLUSTER))); + + EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(REMOTE_CLUSTER_1); + assertThat(remote1Cluster.getIndexExpression(), equalTo("logs-*")); + assertThat(remote1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(remote1Cluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(remote1Cluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertThat(remote1Cluster.getTotalShards(), equalTo(0)); + assertThat(remote1Cluster.getSuccessfulShards(), equalTo(0)); + assertThat(remote1Cluster.getSkippedShards(), equalTo(0)); + assertThat(remote1Cluster.getFailedShards(), equalTo(0)); + + EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(REMOTE_CLUSTER_2); + assertThat(remote2Cluster.getIndexExpression(), equalTo("logs-*")); + assertThat(remote2Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(remote2Cluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(remote2Cluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertThat(remote2Cluster.getTotalShards(), equalTo(0)); + assertThat(remote2Cluster.getSuccessfulShards(), equalTo(0)); + assertThat(remote2Cluster.getSkippedShards(), equalTo(0)); + assertThat(remote2Cluster.getFailedShards(), equalTo(0)); + + EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER); + assertThat(localCluster.getIndexExpression(), equalTo("logs-*")); + assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertThat(localCluster.getTotalShards(), equalTo(localNumShards)); + assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards)); + assertThat(localCluster.getSkippedShards(), equalTo(0)); + assertThat(localCluster.getFailedShards(), equalTo(0)); + + // ensure that the _clusters metadata is present only if requested + assertClusterMetadataInResponse(resp, responseExpectMeta); + } + } finally { + clearSkipUnavailable(numClusters); + } + } + + public void testRemoteOnlyCCSAgainstDisconnectedRemoteWithSkipUnavailableTrue() throws Exception { + int numClusters = 3; + setupClusters(numClusters); + setSkipUnavailable(REMOTE_CLUSTER_1, true); + setSkipUnavailable(REMOTE_CLUSTER_2, true); + + try { + // close remote cluster 1 so that it is unavailable + cluster(REMOTE_CLUSTER_1).close(); + + Tuple includeCCSMetadata = randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); + + // query only the REMOTE_CLUSTER_1 + try (EsqlQueryResponse resp = runQuery("FROM " + REMOTE_CLUSTER_1 + ":logs-* | STATS sum (v)", requestIncludeMeta)) { + List columns = resp.columns(); + assertThat(columns.size(), equalTo(1)); + // column from an empty result should be {"name":"","type":"null"} + assertThat(columns.get(0).name(), equalTo("")); + assertThat(columns.get(0).type(), equalTo(DataType.NULL)); + + List> values = getValuesList(resp); + assertThat(values, hasSize(0)); + + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertNotNull(executionInfo); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + long overallTookMillis = executionInfo.overallTook().millis(); + assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + + assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1))); + + EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER_1); + assertThat(remoteCluster.getIndexExpression(), equalTo("logs-*")); + assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertThat(remoteCluster.getTotalShards(), equalTo(0)); + assertThat(remoteCluster.getSuccessfulShards(), equalTo(0)); + assertThat(remoteCluster.getSkippedShards(), equalTo(0)); + assertThat(remoteCluster.getFailedShards(), equalTo(0)); + + // ensure that the _clusters metadata is present only if requested + assertClusterMetadataInResponse(resp, responseExpectMeta); + } + + // close remote cluster 2 so that it is also unavailable + cluster(REMOTE_CLUSTER_2).close(); + + // query only the both remote clusters + try ( + EsqlQueryResponse resp = runQuery( + "FROM " + REMOTE_CLUSTER_1 + ":logs-*," + REMOTE_CLUSTER_2 + ":logs-* | STATS sum (v)", + requestIncludeMeta + ) + ) { + List columns = resp.columns(); + assertThat(columns.size(), equalTo(1)); + // column from an empty result should be {"name":"","type":"null"} + assertThat(columns.get(0).name(), equalTo("")); + assertThat(columns.get(0).type(), equalTo(DataType.NULL)); + + List> values = getValuesList(resp); + assertThat(values, hasSize(0)); + + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertNotNull(executionInfo); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + long overallTookMillis = executionInfo.overallTook().millis(); + assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + + assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2))); + + EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(REMOTE_CLUSTER_1); + assertThat(remote1Cluster.getIndexExpression(), equalTo("logs-*")); + assertThat(remote1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(remote1Cluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(remote1Cluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertThat(remote1Cluster.getTotalShards(), equalTo(0)); + assertThat(remote1Cluster.getSuccessfulShards(), equalTo(0)); + assertThat(remote1Cluster.getSkippedShards(), equalTo(0)); + assertThat(remote1Cluster.getFailedShards(), equalTo(0)); + + EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(REMOTE_CLUSTER_2); + assertThat(remote2Cluster.getIndexExpression(), equalTo("logs-*")); + assertThat(remote2Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(remote2Cluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(remote2Cluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertThat(remote2Cluster.getTotalShards(), equalTo(0)); + assertThat(remote2Cluster.getSuccessfulShards(), equalTo(0)); + assertThat(remote2Cluster.getSkippedShards(), equalTo(0)); + assertThat(remote2Cluster.getFailedShards(), equalTo(0)); + + // ensure that the _clusters metadata is present only if requested + assertClusterMetadataInResponse(resp, responseExpectMeta); + } + + } finally { + clearSkipUnavailable(numClusters); + } + } + + public void testCCSAgainstDisconnectedRemoteWithSkipUnavailableFalse() throws Exception { + int numClusters = 2; + setupClusters(numClusters); + setSkipUnavailable(REMOTE_CLUSTER_1, false); + + try { + // close the remote cluster so that it is unavailable + cluster(REMOTE_CLUSTER_1).close(); + + Tuple includeCCSMetadata = randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + + final Exception exception = expectThrows( + Exception.class, + () -> runQuery("FROM logs-*,*:logs-* | STATS sum (v)", requestIncludeMeta) + ); + assertThat(ExceptionsHelper.isRemoteUnavailableException(exception), is(true)); + } finally { + clearSkipUnavailable(numClusters); + } + } + + public void testRemoteOnlyCCSAgainstDisconnectedRemoteWithSkipUnavailableFalse() throws Exception { + int numClusters = 3; + setupClusters(numClusters); + setSkipUnavailable(REMOTE_CLUSTER_1, false); + setSkipUnavailable(REMOTE_CLUSTER_2, randomBoolean()); + + try { + Tuple includeCCSMetadata = randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + { + // close the remote cluster so that it is unavailable + cluster(REMOTE_CLUSTER_1).close(); + Exception exception = expectThrows(Exception.class, () -> runQuery("FROM *:logs-* | STATS sum (v)", requestIncludeMeta)); + assertThat(ExceptionsHelper.isRemoteUnavailableException(exception), is(true)); + } + { + // close remote cluster 2 so that it is unavailable + cluster(REMOTE_CLUSTER_2).close(); + Exception exception = expectThrows(Exception.class, () -> runQuery("FROM *:logs-* | STATS sum (v)", requestIncludeMeta)); + assertThat(ExceptionsHelper.isRemoteUnavailableException(exception), is(true)); + } + } finally { + clearSkipUnavailable(numClusters); + } + } + + private void setSkipUnavailable(String clusterAlias, boolean skip) { + client(LOCAL_CLUSTER).admin() + .cluster() + .prepareUpdateSettings(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT) + .setPersistentSettings(Settings.builder().put("cluster.remote." + clusterAlias + ".skip_unavailable", skip).build()) + .get(); + } + + private void clearSkipUnavailable(int numClusters) { + assert numClusters == 2 || numClusters == 3 : "Only 2 or 3 clusters supported"; + Settings.Builder settingsBuilder = Settings.builder().putNull("cluster.remote." + REMOTE_CLUSTER_1 + ".skip_unavailable"); + if (numClusters == 3) { + settingsBuilder.putNull("cluster.remote." + REMOTE_CLUSTER_2 + ".skip_unavailable"); + } + client(LOCAL_CLUSTER).admin() + .cluster() + .prepareUpdateSettings(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT) + .setPersistentSettings(settingsBuilder.build()) + .get(); + } + + private static void assertClusterMetadataInResponse(EsqlQueryResponse resp, boolean responseExpectMeta) { + try { + final Map esqlResponseAsMap = XContentTestUtils.convertToMap(resp); + final Object clusters = esqlResponseAsMap.get("_clusters"); + if (responseExpectMeta) { + assertNotNull(clusters); + // test a few entries to ensure it looks correct (other tests do a full analysis of the metadata in the response) + @SuppressWarnings("unchecked") + Map inner = (Map) clusters; + assertTrue(inner.containsKey("total")); + assertTrue(inner.containsKey("details")); + } else { + assertNull(clusters); + } + } catch (IOException e) { + fail("Could not convert ESQL response to Map: " + e); + } + } + + protected EsqlQueryResponse runQuery(String query, Boolean ccsMetadataInResponse) { + EsqlQueryRequest request = EsqlQueryRequest.syncEsqlQueryRequest(); + request.query(query); + request.pragmas(AbstractEsqlIntegTestCase.randomPragmas()); + request.profile(randomInt(5) == 2); + request.columnar(randomBoolean()); + if (ccsMetadataInResponse != null) { + request.includeCCSMetadata(ccsMetadataInResponse); + } + return runQuery(request); + } + + protected EsqlQueryResponse runQuery(EsqlQueryRequest request) { + return client(LOCAL_CLUSTER).execute(EsqlQueryAction.INSTANCE, request).actionGet(30, TimeUnit.SECONDS); + } + + /** + * v1: value to send to runQuery (can be null; null means use default value) + * v2: whether to expect CCS Metadata in the response (cannot be null) + * @return + */ + public static Tuple randomIncludeCCSMetadata() { + return switch (randomIntBetween(1, 3)) { + case 1 -> new Tuple<>(Boolean.TRUE, Boolean.TRUE); + case 2 -> new Tuple<>(Boolean.FALSE, Boolean.FALSE); + case 3 -> new Tuple<>(null, Boolean.FALSE); + default -> throw new AssertionError("should not get here"); + }; + } + + Map setupClusters(int numClusters) { + assert numClusters == 2 || numClusters == 3 : "2 or 3 clusters supported not: " + numClusters; + String localIndex = "logs-1"; + int numShardsLocal = randomIntBetween(1, 5); + populateLocalIndices(localIndex, numShardsLocal); + + String remoteIndex = "logs-2"; + int numShardsRemote = randomIntBetween(1, 5); + populateRemoteIndices(REMOTE_CLUSTER_1, remoteIndex, numShardsRemote); + + Map clusterInfo = new HashMap<>(); + clusterInfo.put("local.num_shards", numShardsLocal); + clusterInfo.put("local.index", localIndex); + clusterInfo.put("remote.num_shards", numShardsRemote); + clusterInfo.put("remote.index", remoteIndex); + + if (numClusters == 3) { + int numShardsRemote2 = randomIntBetween(1, 5); + populateRemoteIndices(REMOTE_CLUSTER_2, remoteIndex, numShardsRemote2); + clusterInfo.put("remote2.index", remoteIndex); + clusterInfo.put("remote2.num_shards", numShardsRemote2); + } + + return clusterInfo; + } + + void populateLocalIndices(String indexName, int numShards) { + Client localClient = client(LOCAL_CLUSTER); + assertAcked( + localClient.admin() + .indices() + .prepareCreate(indexName) + .setSettings(Settings.builder().put("index.number_of_shards", numShards)) + .setMapping("id", "type=keyword", "tag", "type=keyword", "v", "type=long") + ); + for (int i = 0; i < 10; i++) { + localClient.prepareIndex(indexName).setSource("id", "local-" + i, "tag", "local", "v", i).get(); + } + localClient.admin().indices().prepareRefresh(indexName).get(); + } + + void populateRemoteIndices(String clusterAlias, String indexName, int numShards) { + Client remoteClient = client(clusterAlias); + assertAcked( + remoteClient.admin() + .indices() + .prepareCreate(indexName) + .setSettings(Settings.builder().put("index.number_of_shards", numShards)) + .setMapping("id", "type=keyword", "tag", "type=keyword", "v", "type=long") + ); + for (int i = 0; i < 10; i++) { + remoteClient.prepareIndex(indexName).setSource("id", "remote-" + i, "tag", "remote", "v", i * i).get(); + } + remoteClient.admin().indices().prepareRefresh(indexName).get(); + } +} diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java index ddd5cff014ed2..ba44adb5a85e0 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.test.AbstractMultiClustersTestCase; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.XContentTestUtils; +import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import org.elasticsearch.xpack.esql.plugin.QueryPragmas; @@ -246,7 +247,8 @@ public void testSearchesWhereMissingIndicesAreSpecified() { EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER); assertThat(localCluster.getIndexExpression(), equalTo("no_such_index")); - assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + // TODO: a follow on PR will change this to throw an Exception when the local cluster requests a concrete index that is missing + assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L)); assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); assertThat(localCluster.getTotalShards(), equalTo(0)); @@ -499,7 +501,7 @@ public void testCCSExecutionOnSearchesWithLimit0() { EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER); assertThat(localCluster.getIndexExpression(), equalTo("nomatch*")); - assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L)); assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); assertThat(remoteCluster.getTotalShards(), equalTo(0)); @@ -803,6 +805,14 @@ Map setupTwoClusters() { clusterInfo.put("local.index", localIndex); clusterInfo.put("remote.num_shards", numShardsRemote); clusterInfo.put("remote.index", remoteIndex); + + String skipUnavailableKey = Strings.format("cluster.remote.%s.skip_unavailable", REMOTE_CLUSTER); + Setting skipUnavailableSetting = cluster(REMOTE_CLUSTER).clusterService().getClusterSettings().get(skipUnavailableKey); + boolean skipUnavailable = (boolean) cluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY).clusterService() + .getClusterSettings() + .get(skipUnavailableSetting); + clusterInfo.put("remote.skip_unavailable", skipUnavailable); + return clusterInfo; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java index aeac14091f378..f2ab0355304b3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.action; import org.elasticsearch.TransportVersions; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -281,6 +282,7 @@ public static class Cluster implements ToXContentFragment, Writeable { private final Integer successfulShards; private final Integer skippedShards; private final Integer failedShards; + private final List failures; private final TimeValue took; // search latency for this cluster sub-search /** @@ -300,7 +302,7 @@ public String toString() { } public Cluster(String clusterAlias, String indexExpression) { - this(clusterAlias, indexExpression, true, Cluster.Status.RUNNING, null, null, null, null, null); + this(clusterAlias, indexExpression, true, Cluster.Status.RUNNING, null, null, null, null, null, null); } /** @@ -312,7 +314,7 @@ public Cluster(String clusterAlias, String indexExpression) { * @param skipUnavailable whether this Cluster is marked as skip_unavailable in remote cluster settings */ public Cluster(String clusterAlias, String indexExpression, boolean skipUnavailable) { - this(clusterAlias, indexExpression, skipUnavailable, Cluster.Status.RUNNING, null, null, null, null, null); + this(clusterAlias, indexExpression, skipUnavailable, Cluster.Status.RUNNING, null, null, null, null, null, null); } /** @@ -324,7 +326,7 @@ public Cluster(String clusterAlias, String indexExpression, boolean skipUnavaila * @param status current status of the search on this Cluster */ public Cluster(String clusterAlias, String indexExpression, boolean skipUnavailable, Cluster.Status status) { - this(clusterAlias, indexExpression, skipUnavailable, status, null, null, null, null, null); + this(clusterAlias, indexExpression, skipUnavailable, status, null, null, null, null, null, null); } public Cluster( @@ -336,6 +338,7 @@ public Cluster( Integer successfulShards, Integer skippedShards, Integer failedShards, + List failures, TimeValue took ) { assert clusterAlias != null : "clusterAlias cannot be null"; @@ -349,6 +352,11 @@ public Cluster( this.successfulShards = successfulShards; this.skippedShards = skippedShards; this.failedShards = failedShards; + if (failures == null) { + this.failures = List.of(); + } else { + this.failures = failures; + } this.took = took; } @@ -362,6 +370,11 @@ public Cluster(StreamInput in) throws IOException { this.failedShards = in.readOptionalInt(); this.took = in.readOptionalTimeValue(); this.skipUnavailable = in.readBoolean(); + if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_EXEC_INFO_WITH_FAILURES)) { + this.failures = Collections.unmodifiableList(in.readCollectionAsList(ShardSearchFailure::readShardSearchFailure)); + } else { + this.failures = List.of(); + } } @Override @@ -375,6 +388,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalInt(failedShards); out.writeOptionalTimeValue(took); out.writeBoolean(skipUnavailable); + if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_EXEC_INFO_WITH_FAILURES)) { + out.writeCollection(failures); + } } /** @@ -387,12 +403,12 @@ public void writeTo(StreamOutput out) throws IOException { * All other fields can be set and override the value in the "copyFrom" Cluster. */ public static class Builder { - private String indexExpression; private Cluster.Status status; private Integer totalShards; private Integer successfulShards; private Integer skippedShards; private Integer failedShards; + private List failures; private TimeValue took; private final Cluster original; @@ -408,22 +424,18 @@ public Builder(Cluster copyFrom) { public Cluster build() { return new Cluster( original.getClusterAlias(), - indexExpression == null ? original.getIndexExpression() : indexExpression, + original.getIndexExpression(), original.isSkipUnavailable(), status != null ? status : original.getStatus(), totalShards != null ? totalShards : original.getTotalShards(), successfulShards != null ? successfulShards : original.getSuccessfulShards(), skippedShards != null ? skippedShards : original.getSkippedShards(), failedShards != null ? failedShards : original.getFailedShards(), + failures != null ? failures : original.getFailures(), took != null ? took : original.getTook() ); } - public Cluster.Builder setIndexExpression(String indexExpression) { - this.indexExpression = indexExpression; - return this; - } - public Cluster.Builder setStatus(Cluster.Status status) { this.status = status; return this; @@ -449,6 +461,11 @@ public Cluster.Builder setFailedShards(int failedShards) { return this; } + public Cluster.Builder setFailures(List failures) { + this.failures = failures; + return this; + } + public Cluster.Builder setTook(TimeValue took) { this.took = took; return this; @@ -466,7 +483,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(STATUS_FIELD.getPreferredName(), getStatus().toString()); builder.field(INDICES_FIELD.getPreferredName(), indexExpression); if (took != null) { - // TODO: change this to took_nanos and call took.nanos? builder.field(TOOK.getPreferredName(), took.millis()); } if (totalShards != null) { @@ -483,6 +499,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.endObject(); } + if (failures != null && failures.size() > 0) { + builder.startArray(RestActions.FAILURES_FIELD.getPreferredName()); + for (ShardSearchFailure failure : failures) { + failure.toXContent(builder, params); + } + builder.endArray(); + } } builder.endObject(); return builder; @@ -529,6 +552,10 @@ public Integer getFailedShards() { return failedShards; } + public List getFailures() { + return failures; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/EnrichResolution.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/EnrichResolution.java index 7fb279f18b1dc..4f6886edc5fbc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/EnrichResolution.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/EnrichResolution.java @@ -23,6 +23,7 @@ public final class EnrichResolution { private final Map resolvedPolicies = ConcurrentCollections.newConcurrentMap(); private final Map errors = ConcurrentCollections.newConcurrentMap(); + private final Map unavailableClusters = ConcurrentCollections.newConcurrentMap(); public ResolvedEnrichPolicy getResolvedPolicy(String policyName, Enrich.Mode mode) { return resolvedPolicies.get(new Key(policyName, mode)); @@ -51,6 +52,14 @@ public void addError(String policyName, Enrich.Mode mode, String reason) { errors.putIfAbsent(new Key(policyName, mode), reason); } + public void addUnavailableCluster(String clusterAlias, Exception e) { + unavailableClusters.put(clusterAlias, e); + } + + public Map getUnavailableClusters() { + return unavailableClusters; + } + private record Key(String policyName, Enrich.Mode mode) { } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java index e67c406e26929..77ef5ef597bb5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.enrich; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.search.SearchRequest; @@ -50,6 +51,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -113,12 +115,27 @@ public void resolvePolicies( final boolean includeLocal = remoteClusters.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); lookupPolicies(remoteClusters, includeLocal, unresolvedPolicies, listener.map(lookupResponses -> { final EnrichResolution enrichResolution = new EnrichResolution(); + + Map lookupResponsesToProcess = new HashMap<>(); + + for (Map.Entry entry : lookupResponses.entrySet()) { + String clusterAlias = entry.getKey(); + if (entry.getValue().connectionError != null) { + enrichResolution.addUnavailableCluster(clusterAlias, entry.getValue().connectionError); + // remove unavailable cluster from the list of clusters which is used below to create the ResolvedEnrichPolicy + remoteClusters.remove(clusterAlias); + } else { + lookupResponsesToProcess.put(clusterAlias, entry.getValue()); + } + } + for (UnresolvedPolicy unresolved : unresolvedPolicies) { Tuple resolved = mergeLookupResults( unresolved, calculateTargetClusters(unresolved.mode, includeLocal, remoteClusters), - lookupResponses + lookupResponsesToProcess ); + if (resolved.v1() != null) { enrichResolution.addResolvedPolicy(unresolved.name, unresolved.mode, resolved.v1()); } else { @@ -149,13 +166,16 @@ private Tuple mergeLookupResults( Collection targetClusters, Map lookupResults ) { - assert targetClusters.isEmpty() == false; String policyName = unresolved.name; + if (targetClusters.isEmpty()) { + return Tuple.tuple(null, "enrich policy [" + policyName + "] cannot be resolved since remote clusters are unavailable"); + } final Map policies = new HashMap<>(); final List failures = new ArrayList<>(); for (String cluster : targetClusters) { LookupResponse lookupResult = lookupResults.get(cluster); if (lookupResult != null) { + assert lookupResult.connectionError == null : "Should never have a non-null connectionError here"; ResolvedEnrichPolicy policy = lookupResult.policies.get(policyName); if (policy != null) { policies.put(cluster, policy); @@ -261,22 +281,34 @@ private void lookupPolicies( if (remotePolicies.isEmpty() == false) { for (String cluster : remoteClusters) { ActionListener lookupListener = refs.acquire(resp -> lookupResponses.put(cluster, resp)); - getRemoteConnection( - cluster, - lookupListener.delegateFailureAndWrap( - (delegate, connection) -> transportService.sendRequest( + getRemoteConnection(cluster, new ActionListener() { + @Override + public void onResponse(Transport.Connection connection) { + transportService.sendRequest( connection, RESOLVE_ACTION_NAME, new LookupRequest(cluster, remotePolicies), TransportRequestOptions.EMPTY, - new ActionListenerResponseHandler<>( - delegate, - LookupResponse::new, - threadPool.executor(ThreadPool.Names.SEARCH) - ) - ) - ) - ); + new ActionListenerResponseHandler<>(lookupListener.delegateResponse((l, e) -> { + if (ExceptionsHelper.isRemoteUnavailableException(e) + && remoteClusterService.isSkipUnavailable(cluster)) { + l.onResponse(new LookupResponse(e)); + } else { + l.onFailure(e); + } + }), LookupResponse::new, threadPool.executor(ThreadPool.Names.SEARCH)) + ); + } + + @Override + public void onFailure(Exception e) { + if (ExceptionsHelper.isRemoteUnavailableException(e) && remoteClusterService.isSkipUnavailable(cluster)) { + lookupListener.onResponse(new LookupResponse(e)); + } else { + lookupListener.onFailure(e); + } + } + }); } } // local cluster @@ -323,16 +355,30 @@ public void writeTo(StreamOutput out) throws IOException { private static class LookupResponse extends TransportResponse { final Map policies; final Map failures; + // does not need to be Writable since this indicates a failure to contact a remote cluster, so only set on querying cluster + final transient Exception connectionError; LookupResponse(Map policies, Map failures) { this.policies = policies; this.failures = failures; + this.connectionError = null; + } + + /** + * Use this constructor when the remote cluster is unavailable to indicate inability to do the enrich policy lookup + * @param connectionError Exception received when trying to connect to a remote cluster + */ + LookupResponse(Exception connectionError) { + this.policies = Collections.emptyMap(); + this.failures = Collections.emptyMap(); + this.connectionError = connectionError; } LookupResponse(StreamInput in) throws IOException { PlanStreamInput planIn = new PlanStreamInput(in, in.namedWriteableRegistry(), null); this.policies = planIn.readMap(StreamInput::readString, ResolvedEnrichPolicy::new); this.failures = planIn.readMap(StreamInput::readString, StreamInput::readString); + this.connectionError = null; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java index 371aa1b632309..b2eaefcf09d65 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java @@ -6,27 +6,28 @@ */ package org.elasticsearch.xpack.esql.index; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure; import org.elasticsearch.core.Nullable; import java.util.Collections; +import java.util.Map; import java.util.Objects; -import java.util.Set; public final class IndexResolution { - public static IndexResolution valid(EsIndex index, Set unavailableClusters) { + public static IndexResolution valid(EsIndex index, Map unavailableClusters) { Objects.requireNonNull(index, "index must not be null if it was found"); Objects.requireNonNull(unavailableClusters, "unavailableClusters must not be null"); return new IndexResolution(index, null, unavailableClusters); } public static IndexResolution valid(EsIndex index) { - return valid(index, Collections.emptySet()); + return valid(index, Collections.emptyMap()); } public static IndexResolution invalid(String invalid) { Objects.requireNonNull(invalid, "invalid must not be null to signal that the index is invalid"); - return new IndexResolution(null, invalid, Collections.emptySet()); + return new IndexResolution(null, invalid, Collections.emptyMap()); } public static IndexResolution notFound(String name) { @@ -39,9 +40,9 @@ public static IndexResolution notFound(String name) { private final String invalid; // remote clusters included in the user's index expression that could not be connected to - private final Set unavailableClusters; + private final Map unavailableClusters; - private IndexResolution(EsIndex index, @Nullable String invalid, Set unavailableClusters) { + private IndexResolution(EsIndex index, @Nullable String invalid, Map unavailableClusters) { this.index = index; this.invalid = invalid; this.unavailableClusters = unavailableClusters; @@ -70,7 +71,11 @@ public boolean isValid() { return invalid == null; } - public Set getUnavailableClusters() { + /** + * @return Map of unavailable clusters (could not be connected to during field-caps query). Key of map is cluster alias, + * value is the {@link FieldCapabilitiesFailure} describing the issue. + */ + public Map getUnavailableClusters() { return unavailableClusters; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index ccd167942340c..1e78f454b7531 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -7,8 +7,11 @@ package org.elasticsearch.xpack.esql.session; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.OriginalIndices; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Iterators; @@ -22,7 +25,9 @@ import org.elasticsearch.indices.IndicesExpressionGrouper; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; +import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.RemoteClusterAware; +import org.elasticsearch.transport.RemoteTransportException; import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo; import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; import org.elasticsearch.xpack.esql.analysis.Analyzer; @@ -44,6 +49,7 @@ import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy; import org.elasticsearch.xpack.esql.expression.UnresolvedNamePattern; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.index.IndexResolution; import org.elasticsearch.xpack.esql.index.MappingException; import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; @@ -68,6 +74,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -143,12 +150,105 @@ public void execute( analyzedPlan( parse(request.query(), request.params()), executionInfo, - listener.delegateFailureAndWrap( - (next, analyzedPlan) -> executeOptimizedPlan(request, executionInfo, runPhase, optimizedPlan(analyzedPlan), next) - ) + new LogicalPlanActionListener(request, executionInfo, runPhase, listener) ); } + /** + * ActionListener that receives LogicalPlan or error from logical planning. + * Any Exception sent to onFailure stops processing, but not all are fatal (return a 4xx or 5xx), so + * the onFailure handler determines whether to return an empty successful result or a 4xx/5xx error. + */ + class LogicalPlanActionListener implements ActionListener { + private final EsqlQueryRequest request; + private final EsqlExecutionInfo executionInfo; + private final BiConsumer> runPhase; + private final ActionListener listener; + + LogicalPlanActionListener( + EsqlQueryRequest request, + EsqlExecutionInfo executionInfo, + BiConsumer> runPhase, + ActionListener listener + ) { + this.request = request; + this.executionInfo = executionInfo; + this.runPhase = runPhase; + this.listener = listener; + } + + @Override + public void onResponse(LogicalPlan analyzedPlan) { + executeOptimizedPlan(request, executionInfo, runPhase, optimizedPlan(analyzedPlan), listener); + } + + /** + * Whether to return an empty result (HTTP status 200) for a CCS rather than a top level 4xx/5xx error. + * + * For cases where field-caps had no indices to search and the remotes were unavailable, we + * return an empty successful response (200) if all remotes are marked with skip_unavailable=true. + * + * Note: a follow-on PR will expand this logic to handle cases where no indices could be found to match + * on any of the requested clusters. + */ + private boolean returnSuccessWithEmptyResult(Exception e) { + if (executionInfo.isCrossClusterSearch() == false) { + return false; + } + + if (e instanceof NoClustersToSearchException || ExceptionsHelper.isRemoteUnavailableException(e)) { + for (String clusterAlias : executionInfo.clusterAliases()) { + if (executionInfo.isSkipUnavailable(clusterAlias) == false + && clusterAlias.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) == false) { + return false; + } + } + return true; + } + return false; + } + + @Override + public void onFailure(Exception e) { + if (returnSuccessWithEmptyResult(e)) { + executionInfo.markEndQuery(); + Exception exceptionForResponse; + if (e instanceof ConnectTransportException) { + // when field-caps has no field info (since no clusters could be connected to or had matching indices) + // it just throws the first exception in its list, so this odd special handling is here is to avoid + // having one specific remote alias name in all failure lists in the metadata response + exceptionForResponse = new RemoteTransportException( + "connect_transport_exception - unable to connect to remote cluster", + null + ); + } else { + exceptionForResponse = e; + } + for (String clusterAlias : executionInfo.clusterAliases()) { + executionInfo.swapCluster(clusterAlias, (k, v) -> { + EsqlExecutionInfo.Cluster.Builder builder = new EsqlExecutionInfo.Cluster.Builder(v).setTook( + executionInfo.overallTook() + ).setTotalShards(0).setSuccessfulShards(0).setSkippedShards(0).setFailedShards(0); + if (RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(clusterAlias)) { + // never mark local cluster as skipped + builder.setStatus(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL); + } else { + builder.setStatus(EsqlExecutionInfo.Cluster.Status.SKIPPED); + // add this exception to the failures list only if there is no failure already recorded there + if (v.getFailures() == null || v.getFailures().size() == 0) { + builder.setFailures(List.of(new ShardSearchFailure(exceptionForResponse))); + } + } + return builder.build(); + }); + } + listener.onResponse(new Result(Analyzer.NO_FIELDS, Collections.emptyList(), Collections.emptyList(), executionInfo)); + } else { + listener.onFailure(e); + } + } + } + /** * Execute an analyzed plan. Most code should prefer calling {@link #execute} but * this is public for testing. See {@link Phased} for the sequence of operations. @@ -161,8 +261,8 @@ public void executeOptimizedPlan( ActionListener listener ) { LogicalPlan firstPhase = Phased.extractFirstPhase(optimizedPlan); + updateExecutionInfoAtEndOfPlanning(executionInfo); if (firstPhase == null) { - updateExecutionInfoAtEndOfPlanning(executionInfo); runPhase.accept(logicalPlanToPhysicalPlan(optimizedPlan, request), listener); } else { executePhased(new ArrayList<>(), optimizedPlan, request, executionInfo, firstPhase, runPhase, listener); @@ -242,17 +342,30 @@ private void preAnalyze( .stream() .map(ResolvedEnrichPolicy::matchField) .collect(Collectors.toSet()); - preAnalyzeIndices(parsed, executionInfo, l.delegateFailureAndWrap((ll, indexResolution) -> { + Map unavailableClusters = enrichResolution.getUnavailableClusters(); + preAnalyzeIndices(parsed, executionInfo, unavailableClusters, l.delegateFailureAndWrap((ll, indexResolution) -> { + // TODO in follow-PR (for skip_unavailble handling of missing concrete indexes) add some tests for invalid index + // resolution to updateExecutionInfo if (indexResolution.isValid()) { updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution); updateExecutionInfoWithUnavailableClusters(executionInfo, indexResolution.getUnavailableClusters()); + if (executionInfo.isCrossClusterSearch() + && executionInfo.getClusterStateCount(EsqlExecutionInfo.Cluster.Status.RUNNING) == 0) { + // for a CCS, if all clusters have been marked as SKIPPED, nothing to search so send a sentinel + // Exception to let the LogicalPlanActionListener decide how to proceed + ll.onFailure(new NoClustersToSearchException()); + return; + } + Set newClusters = enrichPolicyResolver.groupIndicesPerCluster( indexResolution.get().concreteIndices().toArray(String[]::new) ).keySet(); // If new clusters appear when resolving the main indices, we need to resolve the enrich policies again // or exclude main concrete indices. Since this is rare, it's simpler to resolve the enrich policies again. // TODO: add a test for this - if (targetClusters.containsAll(newClusters) == false) { + if (targetClusters.containsAll(newClusters) == false + // do not bother with a re-resolution if only remotes were requested and all were offline + && executionInfo.getClusterStateCount(EsqlExecutionInfo.Cluster.Status.RUNNING) > 0) { enrichPolicyResolver.resolvePolicies( newClusters, unresolvedPolicies, @@ -269,6 +382,7 @@ private void preAnalyze( private void preAnalyzeIndices( LogicalPlan parsed, EsqlExecutionInfo executionInfo, + Map unavailableClusters, // known to be unavailable from the enrich policy API call ActionListener listener, Set enrichPolicyMatchFields ) { @@ -288,10 +402,34 @@ private void preAnalyzeIndices( String indexExpr = Strings.arrayToCommaDelimitedString(entry.getValue().indices()); executionInfo.swapCluster(clusterAlias, (k, v) -> { assert v == null : "No cluster for " + clusterAlias + " should have been added to ExecutionInfo yet"; - return new EsqlExecutionInfo.Cluster(clusterAlias, indexExpr, executionInfo.isSkipUnavailable(clusterAlias)); + if (unavailableClusters.containsKey(k)) { + return new EsqlExecutionInfo.Cluster( + clusterAlias, + indexExpr, + executionInfo.isSkipUnavailable(clusterAlias), + EsqlExecutionInfo.Cluster.Status.SKIPPED, + 0, + 0, + 0, + 0, + List.of(new ShardSearchFailure(unavailableClusters.get(k))), + new TimeValue(0) + ); + } else { + return new EsqlExecutionInfo.Cluster(clusterAlias, indexExpr, executionInfo.isSkipUnavailable(clusterAlias)); + } }); } - indexResolver.resolveAsMergedMapping(table.index(), fieldNames, listener); + // if the preceding call to the enrich policy API found unavailable clusters, recreate the index expression to search + // based only on available clusters (which could now be an empty list) + String indexExpressionToResolve = createIndexExpressionFromAvailableClusters(executionInfo); + if (indexExpressionToResolve.isEmpty()) { + // if this was a pure remote CCS request (no local indices) and all remotes are offline, return an empty IndexResolution + listener.onResponse(IndexResolution.valid(new EsIndex(table.index(), Map.of(), Map.of()))); + } else { + // call the EsqlResolveFieldsAction (field-caps) to resolve indices and get field types + indexResolver.resolveAsMergedMapping(indexExpressionToResolve, fieldNames, listener); + } } else { try { // occurs when dealing with local relations (row a = 1) @@ -302,6 +440,30 @@ private void preAnalyzeIndices( } } + // visible for testing + static String createIndexExpressionFromAvailableClusters(EsqlExecutionInfo executionInfo) { + StringBuilder sb = new StringBuilder(); + for (String clusterAlias : executionInfo.clusterAliases()) { + EsqlExecutionInfo.Cluster cluster = executionInfo.getCluster(clusterAlias); + if (cluster.getStatus() != EsqlExecutionInfo.Cluster.Status.SKIPPED) { + if (cluster.getClusterAlias().equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)) { + sb.append(executionInfo.getCluster(clusterAlias).getIndexExpression()).append(','); + } else { + String indexExpression = executionInfo.getCluster(clusterAlias).getIndexExpression(); + for (String index : indexExpression.split(",")) { + sb.append(clusterAlias).append(':').append(index).append(','); + } + } + } + } + + if (sb.length() > 0) { + return sb.substring(0, sb.length() - 1); + } else { + return ""; + } + } + static Set fieldNames(LogicalPlan parsed, Set enrichPolicyMatchFields) { if (false == parsed.anyMatch(plan -> plan instanceof Aggregate || plan instanceof Project)) { // no explicit columns selection, for example "from employees" @@ -446,14 +608,28 @@ public PhysicalPlan optimizedPhysicalPlan(LogicalPlan optimizedPlan) { return plan; } - // visible for testing - static void updateExecutionInfoWithUnavailableClusters(EsqlExecutionInfo executionInfo, Set unavailableClusters) { - for (String clusterAlias : unavailableClusters) { - executionInfo.swapCluster( - clusterAlias, - (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setStatus(EsqlExecutionInfo.Cluster.Status.SKIPPED).build() + static void updateExecutionInfoWithUnavailableClusters(EsqlExecutionInfo execInfo, Map unavailable) { + for (Map.Entry entry : unavailable.entrySet()) { + String clusterAlias = entry.getKey(); + boolean skipUnavailable = execInfo.getCluster(clusterAlias).isSkipUnavailable(); + RemoteTransportException e = new RemoteTransportException( + Strings.format("Remote cluster [%s] (with setting skip_unavailable=%s) is not available", clusterAlias, skipUnavailable), + entry.getValue().getException() ); - // TODO: follow-on PR will set SKIPPED status when skip_unavailable=true and throw an exception when skip_un=false + if (skipUnavailable) { + execInfo.swapCluster( + clusterAlias, + (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setStatus(EsqlExecutionInfo.Cluster.Status.SKIPPED) + .setTotalShards(0) + .setSuccessfulShards(0) + .setSkippedShards(0) + .setFailedShards(0) + .setFailures(List.of(new ShardSearchFailure(e))) + .build() + ); + } else { + throw e; + } } } @@ -466,16 +642,22 @@ static void updateExecutionInfoWithClustersWithNoMatchingIndices(EsqlExecutionIn } Set clustersRequested = executionInfo.clusterAliases(); Set clustersWithNoMatchingIndices = Sets.difference(clustersRequested, clustersWithResolvedIndices); - clustersWithNoMatchingIndices.removeAll(indexResolution.getUnavailableClusters()); + clustersWithNoMatchingIndices.removeAll(indexResolution.getUnavailableClusters().keySet()); /* * These are clusters in the original request that are not present in the field-caps response. They were * specified with an index or indices that do not exist, so the search on that cluster is done. * Mark it as SKIPPED with 0 shards searched and took=0. */ for (String c : clustersWithNoMatchingIndices) { + // TODO: in a follow-on PR, throw a Verification(400 status code) for local and remotes with skip_unavailable=false if + // they were requested with one or more concrete indices + // for now we never mark the local cluster as SKIPPED + final var status = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(c) + ? EsqlExecutionInfo.Cluster.Status.SUCCESSFUL + : EsqlExecutionInfo.Cluster.Status.SKIPPED; executionInfo.swapCluster( c, - (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setStatus(EsqlExecutionInfo.Cluster.Status.SKIPPED) + (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setStatus(status) .setTook(new TimeValue(0)) .setTotalShards(0) .setSuccessfulShards(0) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index c0f94bccc50a4..f76f7798dece8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -90,6 +90,7 @@ public void resolveAsMergedMapping(String indexWildcard, Set fieldNames, public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResponse fieldCapsResponse) { assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION); // too expensive to run this on a transport worker if (fieldCapsResponse.getIndexResponses().isEmpty()) { + // TODO in follow-on PR, handle the case where remotes were specified with non-existent indices, according to skip_unavailable return IndexResolution.notFound(indexPattern); } @@ -158,18 +159,18 @@ public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResp for (FieldCapabilitiesIndexResponse ir : fieldCapsResponse.getIndexResponses()) { concreteIndices.put(ir.getIndexName(), ir.getIndexMode()); } - Set unavailableRemoteClusters = determineUnavailableRemoteClusters(fieldCapsResponse.getFailures()); - return IndexResolution.valid(new EsIndex(indexPattern, rootFields, concreteIndices), unavailableRemoteClusters); + Map unavailableRemotes = determineUnavailableRemoteClusters(fieldCapsResponse.getFailures()); + return IndexResolution.valid(new EsIndex(indexPattern, rootFields, concreteIndices), unavailableRemotes); } // visible for testing - static Set determineUnavailableRemoteClusters(List failures) { - Set unavailableRemotes = new HashSet<>(); + static Map determineUnavailableRemoteClusters(List failures) { + Map unavailableRemotes = new HashMap<>(); for (FieldCapabilitiesFailure failure : failures) { if (ExceptionsHelper.isRemoteUnavailableException(failure.getException())) { for (String indexExpression : failure.getIndices()) { if (indexExpression.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR) > 0) { - unavailableRemotes.add(RemoteClusterAware.parseClusterAlias(indexExpression)); + unavailableRemotes.put(RemoteClusterAware.parseClusterAlias(indexExpression), failure); } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/NoClustersToSearchException.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/NoClustersToSearchException.java new file mode 100644 index 0000000000000..f7ae78a521933 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/NoClustersToSearchException.java @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.session; + +/** + * Sentinel exception indicating that logical planning could not find any clusters to search + * when, for a remote-only cross-cluster search, all clusters have been marked as SKIPPED. + * Intended for use only on the querying coordinating during ES|QL logical planning. + */ +public class NoClustersToSearchException extends RuntimeException {} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java index 27343bf7ce205..4aaf4f6cccf0f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java @@ -147,6 +147,7 @@ EsqlExecutionInfo createExecutionInfo() { 10, 3, 0, + null, new TimeValue(4444L) ) ); @@ -161,6 +162,7 @@ EsqlExecutionInfo createExecutionInfo() { 12, 5, 0, + null, new TimeValue(4999L) ) ); @@ -498,6 +500,7 @@ private static EsqlExecutionInfo.Cluster parseCluster(String clusterAlias, XCont successfulShardsFinal, skippedShardsFinal, failedShardsFinal, + null, tookTimeValue ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java index 5fbd5dd28050f..625cb5628d039 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java @@ -313,6 +313,7 @@ public void testAcquireComputeRunningOnRemoteClusterFillsInTookTime() { 10, 3, 0, + null, null // to be filled in the acquireCompute listener ) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java index 32b31cf78650b..dddfa67338419 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java @@ -7,117 +7,200 @@ package org.elasticsearch.xpack.esql.session; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure; +import org.elasticsearch.common.Strings; import org.elasticsearch.index.IndexMode; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.transport.NoSeedNodeLeftException; import org.elasticsearch.transport.RemoteClusterAware; +import org.elasticsearch.transport.RemoteTransportException; import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.index.IndexResolution; import org.elasticsearch.xpack.esql.type.EsFieldTests; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; public class EsqlSessionTests extends ESTestCase { - public void testUpdateExecutionInfoWithUnavailableClusters() { - // skip_unavailable=true clusters are unavailable, both marked as SKIPPED + public void testCreateIndexExpressionFromAvailableClusters() { + final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; + final String remote1Alias = "remote1"; + final String remote2Alias = "remote2"; + + // no clusters marked as skipped { - final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; - final String remote1Alias = "remote1"; - final String remote2Alias = "remote2"; EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true)); executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", true)); - EsqlSession.updateExecutionInfoWithUnavailableClusters(executionInfo, Set.of(remote1Alias, remote2Alias)); + String indexExpr = EsqlSession.createIndexExpressionFromAvailableClusters(executionInfo); + List list = Arrays.stream(Strings.splitStringByCommaToArray(indexExpr)).toList(); + assertThat(list.size(), equalTo(5)); + assertThat( + new HashSet<>(list), + equalTo(Strings.commaDelimitedListToSet("logs*,remote1:*,remote2:mylogs1,remote2:mylogs2,remote2:logs*")) + ); + } - assertThat(executionInfo.clusterAliases(), equalTo(Set.of(localClusterAlias, remote1Alias, remote2Alias))); - assertNull(executionInfo.overallTook()); + // one cluster marked as skipped, so not present in revised index expression + { + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); + executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); + executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*,foo", true)); + executionInfo.swapCluster( + remote2Alias, + (k, v) -> new EsqlExecutionInfo.Cluster( + remote2Alias, + "mylogs1,mylogs2,logs*", + true, + EsqlExecutionInfo.Cluster.Status.SKIPPED + ) + ); - EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(localClusterAlias); - assertThat(localCluster.getIndexExpression(), equalTo("logs*")); - assertClusterStatusAndHasNullCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING); + String indexExpr = EsqlSession.createIndexExpressionFromAvailableClusters(executionInfo); + List list = Arrays.stream(Strings.splitStringByCommaToArray(indexExpr)).toList(); + assertThat(list.size(), equalTo(3)); + assertThat(new HashSet<>(list), equalTo(Strings.commaDelimitedListToSet("logs*,remote1:*,remote1:foo"))); + } - EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias); - assertThat(remote1Cluster.getIndexExpression(), equalTo("*")); - assertClusterStatusAndHasNullCounts(remote1Cluster, EsqlExecutionInfo.Cluster.Status.SKIPPED); + // two clusters marked as skipped, so only local cluster present in revised index expression + { + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); + executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); + executionInfo.swapCluster( + remote1Alias, + (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*,foo", true, EsqlExecutionInfo.Cluster.Status.SKIPPED) + ); + executionInfo.swapCluster( + remote2Alias, + (k, v) -> new EsqlExecutionInfo.Cluster( + remote2Alias, + "mylogs1,mylogs2,logs*", + true, + EsqlExecutionInfo.Cluster.Status.SKIPPED + ) + ); - EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias); - assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*")); - assertClusterStatusAndHasNullCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.SKIPPED); + assertThat(EsqlSession.createIndexExpressionFromAvailableClusters(executionInfo), equalTo("logs*")); + } + + // only remotes present and all marked as skipped, so in revised index expression should be empty string + { + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); + executionInfo.swapCluster( + remote1Alias, + (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*,foo", true, EsqlExecutionInfo.Cluster.Status.SKIPPED) + ); + executionInfo.swapCluster( + remote2Alias, + (k, v) -> new EsqlExecutionInfo.Cluster( + remote2Alias, + "mylogs1,mylogs2,logs*", + true, + EsqlExecutionInfo.Cluster.Status.SKIPPED + ) + ); + + assertThat(EsqlSession.createIndexExpressionFromAvailableClusters(executionInfo), equalTo("")); } + } + + public void testUpdateExecutionInfoWithUnavailableClusters() { + final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; + final String remote1Alias = "remote1"; + final String remote2Alias = "remote2"; - // skip_unavailable=false cluster is unavailable, marked as SKIPPED // TODO: in follow on PR this will change to throwing an - // Exception + // skip_unavailable=true clusters are unavailable, both marked as SKIPPED { - final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; - final String remote1Alias = "remote1"; - final String remote2Alias = "remote2"; EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true)); - executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false)); + executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", true)); - EsqlSession.updateExecutionInfoWithUnavailableClusters(executionInfo, Set.of(remote2Alias)); + var failure = new FieldCapabilitiesFailure(new String[] { "logs-a" }, new NoSeedNodeLeftException("unable to connect")); + var unvailableClusters = Map.of(remote1Alias, failure, remote2Alias, failure); + EsqlSession.updateExecutionInfoWithUnavailableClusters(executionInfo, unvailableClusters); assertThat(executionInfo.clusterAliases(), equalTo(Set.of(localClusterAlias, remote1Alias, remote2Alias))); assertNull(executionInfo.overallTook()); EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(localClusterAlias); assertThat(localCluster.getIndexExpression(), equalTo("logs*")); - assertClusterStatusAndHasNullCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING); + assertClusterStatusAndShardCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING); EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias); assertThat(remote1Cluster.getIndexExpression(), equalTo("*")); - assertClusterStatusAndHasNullCounts(remote1Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING); + assertClusterStatusAndShardCounts(remote1Cluster, EsqlExecutionInfo.Cluster.Status.SKIPPED); EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias); assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*")); - assertClusterStatusAndHasNullCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.SKIPPED); + assertClusterStatusAndShardCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.SKIPPED); + } + + // skip_unavailable=false cluster is unavailable, throws Exception + { + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); + executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); + executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true)); + executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false)); + + var failure = new FieldCapabilitiesFailure(new String[] { "logs-a" }, new NoSeedNodeLeftException("unable to connect")); + RemoteTransportException e = expectThrows( + RemoteTransportException.class, + () -> EsqlSession.updateExecutionInfoWithUnavailableClusters(executionInfo, Map.of(remote2Alias, failure)) + ); + assertThat(e.status().getStatus(), equalTo(500)); + assertThat( + e.getDetailedMessage(), + containsString("Remote cluster [remote2] (with setting skip_unavailable=false) is not available") + ); + assertThat(e.getCause().getMessage(), containsString("unable to connect")); } // all clusters available, no Clusters in ExecutionInfo should be modified { - final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; - final String remote1Alias = "remote1"; - final String remote2Alias = "remote2"; EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true)); executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false)); - EsqlSession.updateExecutionInfoWithUnavailableClusters(executionInfo, Set.of()); + EsqlSession.updateExecutionInfoWithUnavailableClusters(executionInfo, Map.of()); assertThat(executionInfo.clusterAliases(), equalTo(Set.of(localClusterAlias, remote1Alias, remote2Alias))); assertNull(executionInfo.overallTook()); EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(localClusterAlias); assertThat(localCluster.getIndexExpression(), equalTo("logs*")); - assertClusterStatusAndHasNullCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING); + assertClusterStatusAndShardCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING); EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias); assertThat(remote1Cluster.getIndexExpression(), equalTo("*")); - assertClusterStatusAndHasNullCounts(remote1Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING); + assertClusterStatusAndShardCounts(remote1Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING); EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias); assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*")); - assertClusterStatusAndHasNullCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING); + assertClusterStatusAndShardCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING); } } public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { + final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; + final String remote1Alias = "remote1"; + final String remote2Alias = "remote2"; // all clusters present in EsIndex, so no updates to EsqlExecutionInfo should happen { - final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; - final String remote1Alias = "remote1"; - final String remote2Alias = "remote2"; EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true)); @@ -139,28 +222,25 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { IndexMode.STANDARD ) ); - IndexResolution indexResolution = IndexResolution.valid(esIndex, Set.of()); + IndexResolution indexResolution = IndexResolution.valid(esIndex, Map.of()); EsqlSession.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution); EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(localClusterAlias); assertThat(localCluster.getIndexExpression(), equalTo("logs*")); - assertClusterStatusAndHasNullCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING); + assertClusterStatusAndShardCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING); EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias); assertThat(remote1Cluster.getIndexExpression(), equalTo("*")); - assertClusterStatusAndHasNullCounts(remote1Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING); + assertClusterStatusAndShardCounts(remote1Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING); EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias); assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*")); - assertClusterStatusAndHasNullCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING); + assertClusterStatusAndShardCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING); } // remote1 is missing from EsIndex info, so it should be updated and marked as SKIPPED with 0 total shards, 0 took time, etc. { - final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; - final String remote1Alias = "remote1"; - final String remote2Alias = "remote2"; EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true)); @@ -180,13 +260,13 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { IndexMode.STANDARD ) ); - IndexResolution indexResolution = IndexResolution.valid(esIndex, Set.of()); + IndexResolution indexResolution = IndexResolution.valid(esIndex, Map.of()); EsqlSession.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution); EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(localClusterAlias); assertThat(localCluster.getIndexExpression(), equalTo("logs*")); - assertClusterStatusAndHasNullCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING); + assertClusterStatusAndShardCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING); EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias); assertThat(remote1Cluster.getIndexExpression(), equalTo("*")); @@ -199,14 +279,11 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias); assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*")); - assertClusterStatusAndHasNullCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING); + assertClusterStatusAndShardCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING); } // all remotes are missing from EsIndex info, so they should be updated and marked as SKIPPED with 0 total shards, 0 took time, etc. { - final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; - final String remote1Alias = "remote1"; - final String remote2Alias = "remote2"; EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true)); @@ -217,21 +294,21 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { randomMapping(), Map.of("logs-a", IndexMode.STANDARD) ); - // mark remote1 as unavailable - IndexResolution indexResolution = IndexResolution.valid(esIndex, Set.of(remote1Alias)); + // remote1 is unavailable + var failure = new FieldCapabilitiesFailure(new String[] { "logs-a" }, new NoSeedNodeLeftException("unable to connect")); + IndexResolution indexResolution = IndexResolution.valid(esIndex, Map.of(remote1Alias, failure)); EsqlSession.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution); EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(localClusterAlias); assertThat(localCluster.getIndexExpression(), equalTo("logs*")); - assertClusterStatusAndHasNullCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING); + assertClusterStatusAndShardCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING); EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias); assertThat(remote1Cluster.getIndexExpression(), equalTo("*")); - // remote1 is left as RUNNING, since another method (updateExecutionInfoWithUnavailableClusters) not under test changes status + // since remote1 is in the unavailable Map (passed to IndexResolution.valid), it's status will not be changed + // by updateExecutionInfoWithClustersWithNoMatchingIndices (it is handled in updateExecutionInfoWithUnavailableClusters) assertThat(remote1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.RUNNING)); - assertNull(remote1Cluster.getTook()); - assertNull(remote1Cluster.getTotalShards()); EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias); assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*")); @@ -242,6 +319,25 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { assertThat(remote2Cluster.getSkippedShards(), equalTo(0)); assertThat(remote2Cluster.getFailedShards(), equalTo(0)); } + + // all remotes are missing from EsIndex info. Since one is configured with skip_unavailable=false, + // an exception should be thrown + { + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); + executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*")); + executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true)); + executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false)); + + EsIndex esIndex = new EsIndex( + "logs*,remote2:mylogs1,remote2:mylogs2,remote2:logs*", + randomMapping(), + Map.of("logs-a", IndexMode.STANDARD) + ); + + var failure = new FieldCapabilitiesFailure(new String[] { "logs-a" }, new NoSeedNodeLeftException("unable to connect")); + IndexResolution indexResolution = IndexResolution.valid(esIndex, Map.of(remote1Alias, failure)); + EsqlSession.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution); + } } public void testUpdateExecutionInfoAtEndOfPlanning() { @@ -288,13 +384,22 @@ public void testUpdateExecutionInfoAtEndOfPlanning() { assertNull(remote2Cluster.getTook()); } - private void assertClusterStatusAndHasNullCounts(EsqlExecutionInfo.Cluster cluster, EsqlExecutionInfo.Cluster.Status status) { + private void assertClusterStatusAndShardCounts(EsqlExecutionInfo.Cluster cluster, EsqlExecutionInfo.Cluster.Status status) { assertThat(cluster.getStatus(), equalTo(status)); assertNull(cluster.getTook()); - assertNull(cluster.getTotalShards()); - assertNull(cluster.getSuccessfulShards()); - assertNull(cluster.getSkippedShards()); - assertNull(cluster.getFailedShards()); + if (status == EsqlExecutionInfo.Cluster.Status.RUNNING) { + assertNull(cluster.getTotalShards()); + assertNull(cluster.getSuccessfulShards()); + assertNull(cluster.getSkippedShards()); + assertNull(cluster.getFailedShards()); + } else if (status == EsqlExecutionInfo.Cluster.Status.SKIPPED) { + assertThat(cluster.getTotalShards(), equalTo(0)); + assertThat(cluster.getSuccessfulShards(), equalTo(0)); + assertThat(cluster.getSkippedShards(), equalTo(0)); + assertThat(cluster.getFailedShards(), equalTo(0)); + } else { + fail("Unexpected status: " + status); + } } private static Map randomMapping() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverTests.java index 51497b5ca5093..d6e410305afaa 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverTests.java @@ -15,6 +15,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; import static org.hamcrest.Matchers.equalTo; @@ -33,8 +34,8 @@ public void testDetermineUnavailableRemoteClusters() { ) ); - Set unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures); - assertThat(unavailableClusters, equalTo(Set.of("remote1", "remote2"))); + Map unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures); + assertThat(unavailableClusters.keySet(), equalTo(Set.of("remote1", "remote2"))); } // one cluster with "remote unavailable" with two failures @@ -43,8 +44,8 @@ public void testDetermineUnavailableRemoteClusters() { failures.add(new FieldCapabilitiesFailure(new String[] { "remote2:mylogs1" }, new NoSuchRemoteClusterException("remote2"))); failures.add(new FieldCapabilitiesFailure(new String[] { "remote2:mylogs1" }, new NoSeedNodeLeftException("no seed node"))); - Set unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures); - assertThat(unavailableClusters, equalTo(Set.of("remote2"))); + Map unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures); + assertThat(unavailableClusters.keySet(), equalTo(Set.of("remote2"))); } // two clusters, one "remote unavailable" type exceptions and one with another type @@ -57,23 +58,23 @@ public void testDetermineUnavailableRemoteClusters() { new IllegalStateException("Unable to open any connections") ) ); - Set unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures); - assertThat(unavailableClusters, equalTo(Set.of("remote2"))); + Map unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures); + assertThat(unavailableClusters.keySet(), equalTo(Set.of("remote2"))); } // one cluster1 with exception not known to indicate "remote unavailable" { List failures = new ArrayList<>(); failures.add(new FieldCapabilitiesFailure(new String[] { "remote1:mylogs1" }, new RuntimeException("foo"))); - Set unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures); - assertThat(unavailableClusters, equalTo(Set.of())); + Map unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures); + assertThat(unavailableClusters.keySet(), equalTo(Set.of())); } // empty failures list { List failures = new ArrayList<>(); - Set unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures); - assertThat(unavailableClusters, equalTo(Set.of())); + Map unavailableClusters = IndexResolver.determineUnavailableRemoteClusters(failures); + assertThat(unavailableClusters.keySet(), equalTo(Set.of())); } } } diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java index 1a236ccb6aa06..d5b3141b539eb 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java @@ -34,9 +34,12 @@ import java.util.Base64; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -491,6 +494,10 @@ public void testCrossClusterQueryWithRemoteDLSAndFLS() throws Exception { assertThat(flatList, containsInAnyOrder("engineering")); } + /** + * Note: invalid_remote is "invalid" because it has a bogus API key and the cluster does not exist (cannot be connected to) + */ + @SuppressWarnings("unchecked") public void testCrossClusterQueryAgainstInvalidRemote() throws Exception { configureRemoteCluster(); populateData(); @@ -512,22 +519,53 @@ public void testCrossClusterQueryAgainstInvalidRemote() throws Exception { ); // invalid remote with local index should return local results - var q = "FROM invalid_remote:employees,employees | SORT emp_id DESC | LIMIT 10"; - Response response = performRequestWithRemoteSearchUser(esqlRequest(q)); - assertLocalOnlyResults(response); - - // only calling an invalid remote should error - ResponseException error = expectThrows(ResponseException.class, () -> { - var q2 = "FROM invalid_remote:employees | SORT emp_id DESC | LIMIT 10"; - performRequestWithRemoteSearchUser(esqlRequest(q2)); - }); - - if (skipUnavailable == false) { - assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(401)); - assertThat(error.getMessage(), containsString("unable to find apikey")); - } else { - assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(500)); - assertThat(error.getMessage(), containsString("Unable to connect to [invalid_remote]")); + { + var q = "FROM invalid_remote:employees,employees | SORT emp_id DESC | LIMIT 10"; + Response response = performRequestWithRemoteSearchUser(esqlRequest(q)); + // TODO: when skip_unavailable=false for invalid_remote, a fatal exception should be thrown + // this does not yet happen because field-caps returns nothing for this cluster, rather + // than an error, so the current code cannot detect that error. Follow on PR will handle this. + assertLocalOnlyResults(response); + } + + { + var q = "FROM invalid_remote:employees | SORT emp_id DESC | LIMIT 10"; + // errors from invalid remote should be ignored if the cluster is marked with skip_unavailable=true + if (skipUnavailable) { + // expected response: + // {"took":1,"columns":[],"values":[],"_clusters":{"total":1,"successful":0,"running":0,"skipped":1,"partial":0, + // "failed":0,"details":{"invalid_remote":{"status":"skipped","indices":"employees","took":1,"_shards": + // {"total":0,"successful":0,"skipped":0,"failed":0},"failures":[{"shard":-1,"index":null,"reason": + // {"type":"remote_transport_exception", + // "reason":"[connect_transport_exception - unable to connect to remote cluster]"}}]}}}} + Response response = performRequestWithRemoteSearchUser(esqlRequest(q)); + assertOK(response); + Map responseAsMap = entityAsMap(response); + List columns = (List) responseAsMap.get("columns"); + List values = (List) responseAsMap.get("values"); + assertThat(columns.size(), equalTo(1)); + Map column1 = (Map) columns.get(0); + assertThat(column1.get("name").toString(), equalTo("")); + assertThat(values.size(), equalTo(0)); + Map clusters = (Map) responseAsMap.get("_clusters"); + Map details = (Map) clusters.get("details"); + Map invalidRemoteEntry = (Map) details.get("invalid_remote"); + assertThat(invalidRemoteEntry.get("status").toString(), equalTo("skipped")); + List failures = (List) invalidRemoteEntry.get("failures"); + assertThat(failures.size(), equalTo(1)); + Map failuresMap = (Map) failures.get(0); + Map reason = (Map) failuresMap.get("reason"); + assertThat(reason.get("type").toString(), equalTo("remote_transport_exception")); + assertThat(reason.get("reason").toString(), containsString("unable to connect to remote cluster")); + + } else { + // errors from invalid remote should throw an exception if the cluster is marked with skip_unavailable=false + ResponseException error = expectThrows(ResponseException.class, () -> { + final Response response1 = performRequestWithRemoteSearchUser(esqlRequest(q)); + }); + assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(401)); + assertThat(error.getMessage(), containsString("unable to find apikey")); + } } } @@ -887,7 +925,16 @@ public void testAlias() throws Exception { Request request = esqlRequest("FROM " + index + " | KEEP emp_id | SORT emp_id | LIMIT 100"); ResponseException error = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUser(request)); assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(400)); - assertThat(error.getMessage(), containsString(" Unknown index [" + index + "]")); + String expectedIndexExpressionInError = index.replace("*", "my_remote_cluster"); + Pattern p = Pattern.compile("Unknown index \\[([^\\]]+)\\]"); + Matcher m = p.matcher(error.getMessage()); + assertTrue("Pattern matcher to parse error message did not find matching string: " + error.getMessage(), m.find()); + String unknownIndexExpressionInErrorMessage = m.group(1); + Set actualUnknownIndexes = org.elasticsearch.common.Strings.commaDelimitedListToSet( + unknownIndexExpressionInErrorMessage + ); + Set expectedUnknownIndexes = org.elasticsearch.common.Strings.commaDelimitedListToSet(expectedIndexExpressionInError); + assertThat(actualUnknownIndexes, equalTo(expectedUnknownIndexes)); } for (var index : List.of( @@ -920,6 +967,7 @@ protected Request esqlRequest(String command) throws IOException { XContentBuilder body = JsonXContent.contentBuilder(); body.startObject(); body.field("query", command); + body.field("include_ccs_metadata", true); if (Build.current().isSnapshot() && randomBoolean()) { Settings.Builder settings = Settings.builder(); if (randomBoolean()) { From d6d11d8788030c618982f19f8d74f5c569c9352b Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 08:03:49 +0100 Subject: [PATCH 16/18] Update docker.elastic.co/wolfi/chainguard-base:latest Docker digest to 9734313 (main) (#115350) * Update docker.elastic.co/wolfi/chainguard-base:latest Docker digest to 9734313 * Only allow renovate PRs once per week --------- Co-authored-by: elastic-renovate-prod[bot] <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Co-authored-by: Rene Groeschke --- .../java/org/elasticsearch/gradle/internal/DockerBase.java | 2 +- renovate.json | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java index 0535f0bdc3cc8..3e0a47a8f453c 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java @@ -24,7 +24,7 @@ public enum DockerBase { // Chainguard based wolfi image with latest jdk // This is usually updated via renovatebot // spotless:off - WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:bf163e1977002301f7b9fd28fe6837a8cb2dd5c83e4cd45fb67fb28d15d5d40f", + WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:973431347ad45f40e01afbbd010bf9de929c088a63382239b90dd84f39618bc8", "-wolfi", "apk" ), diff --git a/renovate.json b/renovate.json index 293a2bb262375..c1637ae651c1c 100644 --- a/renovate.json +++ b/renovate.json @@ -4,6 +4,9 @@ "github>elastic/renovate-config:only-chainguard", ":disableDependencyDashboard" ], + "schedule": [ + "after 1pm on tuesday" + ], "labels": [">non-issue", ":Delivery/Packaging", "Team:Delivery"], "baseBranches": ["main", "8.x"], "packageRules": [ From 75ab4eb93bb17b89458042be8924a9c34fae626c Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Wed, 30 Oct 2024 08:36:27 +0100 Subject: [PATCH 17/18] fix: _ignored_source is a multi-value field (#115853) Co-authored-by: Elastic Machine --- muted-tests.yml | 6 ------ .../main/java/org/elasticsearch/index/get/GetResult.java | 5 +++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 22e57a524f0bc..ddb50c5a829f9 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -251,18 +251,12 @@ tests: - class: org.elasticsearch.oldrepos.OldRepositoryAccessIT method: testOldRepoAccess issue: https://github.com/elastic/elasticsearch/issues/115631 -- class: org.elasticsearch.index.get.GetResultTests - method: testToAndFromXContent - issue: https://github.com/elastic/elasticsearch/issues/115688 - class: org.elasticsearch.action.update.UpdateResponseTests method: testToAndFromXContent issue: https://github.com/elastic/elasticsearch/issues/115689 - class: org.elasticsearch.xpack.shutdown.NodeShutdownIT method: testStalledShardMigrationProperlyDetected issue: https://github.com/elastic/elasticsearch/issues/115697 -- class: org.elasticsearch.index.get.GetResultTests - method: testToAndFromXContentEmbedded - issue: https://github.com/elastic/elasticsearch/issues/115657 - class: org.elasticsearch.xpack.spatial.search.GeoGridAggAndQueryConsistencyIT method: testGeoShapeGeoHash issue: https://github.com/elastic/elasticsearch/issues/115664 diff --git a/server/src/main/java/org/elasticsearch/index/get/GetResult.java b/server/src/main/java/org/elasticsearch/index/get/GetResult.java index 109f645f24caf..3c504d400c7c6 100644 --- a/server/src/main/java/org/elasticsearch/index/get/GetResult.java +++ b/server/src/main/java/org/elasticsearch/index/get/GetResult.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.mapper.IgnoredFieldMapper; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.search.lookup.Source; @@ -247,7 +248,7 @@ public XContentBuilder toXContentEmbedded(XContentBuilder builder, Params params for (DocumentField field : metaFields.values()) { // TODO: can we avoid having an exception here? - if (field.getName().equals(IgnoredFieldMapper.NAME)) { + if (field.getName().equals(IgnoredFieldMapper.NAME) || field.getName().equals(IgnoredSourceFieldMapper.NAME)) { builder.field(field.getName(), field.getValues()); } else { builder.field(field.getName(), field.getValue()); @@ -341,7 +342,7 @@ public static GetResult fromXContentEmbedded(XContentParser parser, String index parser.skipChildren(); // skip potential inner objects for forward compatibility } } else if (token == XContentParser.Token.START_ARRAY) { - if (IgnoredFieldMapper.NAME.equals(currentFieldName)) { + if (IgnoredFieldMapper.NAME.equals(currentFieldName) || IgnoredSourceFieldMapper.NAME.equals(currentFieldName)) { metaFields.put(currentFieldName, new DocumentField(currentFieldName, parser.list())); } else { parser.skipChildren(); // skip potential inner arrays for forward compatibility From feea0a09b87509a201e7dc5a9a79c0e2bd29d620 Mon Sep 17 00:00:00 2001 From: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Date: Wed, 30 Oct 2024 10:14:22 +0100 Subject: [PATCH 18/18] [DOCS] Update connectors link on landing page (#115904) --- docs/reference/landing-page.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/landing-page.asciidoc b/docs/reference/landing-page.asciidoc index f1b5ce8210996..1f2145a3aae82 100644 --- a/docs/reference/landing-page.asciidoc +++ b/docs/reference/landing-page.asciidoc @@ -128,7 +128,7 @@ Adding data to Elasticsearch
  • - Connectors + Connectors
  • Web crawler