From 9a11f139deb5929808ad5e2cf7ff0de66eaea627 Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Fri, 10 Jan 2025 12:24:09 +0200 Subject: [PATCH 01/13] [9.0] Deprecate ILM Freeze Action & remove min_age check (#119456) --- .../xpack/core/ilm/FreezeAction.java | 51 ++++------------ .../xpack/core/ilm/FreezeStep.java | 35 ----------- .../xpack/core/ilm/LifecyclePolicy.java | 12 ++++ .../elasticsearch/xpack/core/ilm/Phase.java | 43 +++++++------ .../xpack/core/ilm/FreezeActionTests.java | 16 ++--- .../xpack/core/ilm/FreezeStepTests.java | 43 ------------- .../xpack/core/ilm/LifecyclePolicyTests.java | 1 + .../xpack/ilm/TimeSeriesDataStreamsIT.java | 37 ++++++------ .../ilm/TimeSeriesLifecycleActionsIT.java | 22 +++++-- .../actions/SearchableSnapshotActionIT.java | 60 ++++++++++++++----- .../TransportExplainLifecycleAction.java | 1 + .../action/TransportPutLifecycleAction.java | 2 +- 12 files changed, 140 insertions(+), 183 deletions(-) delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeStep.java delete mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeStepTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java index 09e625b96135c..fceeae07ce75b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeAction.java @@ -6,12 +6,10 @@ */ package org.elasticsearch.xpack.core.ilm; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -21,10 +19,14 @@ import java.util.List; /** - * A {@link LifecycleAction} which freezes the index. + * A noop {@link LifecycleAction} that replaces the removed freeze action. We keep it for backwards compatibility purposes in case we + * encounter a policy or an index that refers to this action and its steps in the lifecycle state. + * At 10.x we would like to sanitize the freeze action from input and expunge from the lifecycle execution + * state of indices. */ +@Deprecated +@UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) public class FreezeAction implements LifecycleAction { - private static final Logger logger = LogManager.getLogger(FreezeAction.class); public static final String NAME = "freeze"; public static final String CONDITIONAL_SKIP_FREEZE_STEP = BranchingStep.NAME + "-freeze-check-prerequisites"; @@ -34,6 +36,7 @@ public class FreezeAction implements LifecycleAction { private static final ObjectParser PARSER = new ObjectParser<>(NAME, () -> INSTANCE); public static FreezeAction parse(XContentParser parser) { + return PARSER.apply(parser, null); } @@ -63,40 +66,10 @@ public boolean isSafeAction() { public List toSteps(Client client, String phase, StepKey nextStepKey) { StepKey preFreezeMergeBranchingKey = new StepKey(phase, NAME, CONDITIONAL_SKIP_FREEZE_STEP); StepKey checkNotWriteIndex = new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME); - StepKey freezeStepKey = new StepKey(phase, NAME, FreezeStep.NAME); - - BranchingStep conditionalSkipFreezeStep = new BranchingStep( - preFreezeMergeBranchingKey, - checkNotWriteIndex, - nextStepKey, - (index, clusterState) -> { - IndexMetadata indexMetadata = clusterState.getMetadata().index(index); - assert indexMetadata != null : "index " + index.getName() + " must exist in the cluster state"; - String policyName = indexMetadata.getLifecyclePolicyName(); - if (indexMetadata.getSettings().get(LifecycleSettings.SNAPSHOT_INDEX_NAME) != null) { - logger.warn( - "[{}] action is configured for index [{}] in policy [{}] which is mounted as searchable snapshot. " - + "Skipping this action", - FreezeAction.NAME, - index.getName(), - policyName - ); - return true; - } - if (indexMetadata.getSettings().getAsBoolean("index.frozen", false)) { - logger.debug( - "skipping [{}] action for index [{}] in policy [{}] as the index is already frozen", - FreezeAction.NAME, - index.getName(), - policyName - ); - return true; - } - return false; - } - ); - CheckNotDataStreamWriteIndexStep checkNoWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNotWriteIndex, freezeStepKey); - FreezeStep freezeStep = new FreezeStep(freezeStepKey, nextStepKey, client); + StepKey freezeStepKey = new StepKey(phase, NAME, NAME); + NoopStep conditionalSkipFreezeStep = new NoopStep(preFreezeMergeBranchingKey, nextStepKey); + NoopStep checkNoWriteIndexStep = new NoopStep(checkNotWriteIndex, nextStepKey); + NoopStep freezeStep = new NoopStep(freezeStepKey, nextStepKey); return List.of(conditionalSkipFreezeStep, checkNoWriteIndexStep, freezeStep); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeStep.java deleted file mode 100644 index e36548035f939..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/FreezeStep.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.core.ilm; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetadata; - -/** - * Freezes an index. - */ -@Deprecated // To be removed in 9.0 -public class FreezeStep extends AsyncRetryDuringSnapshotActionStep { - public static final String NAME = "freeze"; - - public FreezeStep(StepKey key, StepKey nextStepKey, Client client) { - super(key, nextStepKey, client); - } - - @Override - public void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentState, ActionListener listener) { - // Deprecated in 7.x, the freeze action is a noop in 8.x, so immediately return here - listener.onResponse(null); - } - - @Override - public boolean isRetryable() { - return true; - } -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicy.java index e7021d22de47e..9129e4f029dd6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicy.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicy.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; @@ -33,6 +34,8 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.COLD_PHASE; + /** * Represents the lifecycle of an index from creation to deletion. A * {@link LifecyclePolicy} is made up of a set of {@link Phase}s which it will @@ -365,4 +368,13 @@ public boolean equals(Object obj) { public String toString() { return Strings.toString(this, true, true); } + + @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) + public boolean maybeAddDeprecationWarningForFreezeAction(String policyName) { + Phase coldPhase = phases.get(COLD_PHASE); + if (coldPhase != null) { + return coldPhase.maybeAddDeprecationWarningForFreezeAction(policyName); + } + return false; + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/Phase.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/Phase.java index b2ace46e3708b..8608a85ddc9af 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/Phase.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/Phase.java @@ -6,15 +6,15 @@ */ package org.elasticsearch.xpack.core.ilm; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.UpdateForV9; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ContextParser; import org.elasticsearch.xcontent.ObjectParser.ValueType; @@ -34,7 +34,7 @@ * particular point in the lifecycle of an index. */ public class Phase implements ToXContentObject, Writeable { - private static final Logger logger = LogManager.getLogger(Phase.class); + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(Phase.class); public static final ParseField MIN_AGE = new ParseField("min_age"); public static final ParseField ACTIONS_FIELD = new ParseField("actions"); @@ -51,19 +51,12 @@ public class Phase implements ToXContentObject, Writeable { return new Phase(name, (TimeValue) a[0], map); }); static { - PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), (ContextParser) (p, c) -> { - // In earlier versions it was possible to create a Phase with a negative `min_age` which would then cause errors - // when the phase is read from the cluster state during startup (even before negative timevalues were strictly - // disallowed) so this is a hack to treat negative `min_age`s as 0 to prevent those errors. - // They will be saved as `0` so this hack can be removed once we no longer have to read cluster states from 7.x. - @UpdateForV9(owner = UpdateForV9.Owner.DATA_MANAGEMENT) // remove this hack now that we don't have to read 7.x cluster states - final String timeValueString = p.text(); - if (timeValueString.startsWith("-")) { - logger.warn("phase has negative min_age value of [{}] - this will be treated as a min_age of 0", timeValueString); - return TimeValue.ZERO; - } - return TimeValue.parseTimeValue(timeValueString, MIN_AGE.getPreferredName()); - }, MIN_AGE, ValueType.VALUE); + PARSER.declareField( + ConstructingObjectParser.optionalConstructorArg(), + (ContextParser) (p, c) -> TimeValue.parseTimeValue(p.text(), MIN_AGE.getPreferredName()), + MIN_AGE, + ValueType.VALUE + ); PARSER.declareNamedObjects( ConstructingObjectParser.constructorArg(), (p, c, n) -> p.namedObject(LifecycleAction.class, n, null), @@ -183,4 +176,20 @@ public String toString() { return Strings.toString(this, true, true); } + @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) + public boolean maybeAddDeprecationWarningForFreezeAction(String policyName) { + if (getActions().containsKey(FreezeAction.NAME)) { + deprecationLogger.warn( + DeprecationCategory.OTHER, + "ilm_freeze_action_deprecation", + "The freeze action in ILM is deprecated and will be removed in a future version;" + + " this action is already a noop so it can be safely removed. Please remove the freeze action from the '" + + policyName + + "' policy." + ); + return true; + } + return false; + } + } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeActionTests.java index 858ba31d27a76..efbd15ac2065f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeActionTests.java @@ -29,7 +29,8 @@ protected FreezeAction createTestInstance() { @Override protected FreezeAction mutateInstance(FreezeAction instance) { - return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929 + // This class is a singleton + return null; } @Override @@ -50,16 +51,15 @@ public void testToSteps() { assertEquals(3, steps.size()); StepKey expectedFirstStepKey = new StepKey(phase, FreezeAction.NAME, FreezeAction.CONDITIONAL_SKIP_FREEZE_STEP); StepKey expectedSecondStepKey = new StepKey(phase, FreezeAction.NAME, CheckNotDataStreamWriteIndexStep.NAME); - StepKey expectedThirdStepKey = new StepKey(phase, FreezeAction.NAME, FreezeStep.NAME); - - BranchingStep firstStep = (BranchingStep) steps.get(0); - CheckNotDataStreamWriteIndexStep secondStep = (CheckNotDataStreamWriteIndexStep) steps.get(1); - FreezeStep thirdStep = (FreezeStep) steps.get(2); + StepKey expectedThirdStepKey = new StepKey(phase, FreezeAction.NAME, FreezeAction.NAME); + NoopStep firstStep = (NoopStep) steps.get(0); assertThat(firstStep.getKey(), equalTo(expectedFirstStepKey)); - + assertThat(firstStep.getNextStepKey(), equalTo(nextStepKey)); + NoopStep secondStep = (NoopStep) steps.get(1); assertEquals(expectedSecondStepKey, secondStep.getKey()); - assertEquals(expectedThirdStepKey, secondStep.getNextStepKey()); + assertEquals(nextStepKey, secondStep.getNextStepKey()); + NoopStep thirdStep = (NoopStep) steps.get(2); assertEquals(expectedThirdStepKey, thirdStep.getKey()); assertEquals(nextStepKey, thirdStep.getNextStepKey()); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeStepTests.java deleted file mode 100644 index 32386ceb2b882..0000000000000 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeStepTests.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.core.ilm; - -import org.elasticsearch.xpack.core.ilm.Step.StepKey; - -public class FreezeStepTests extends AbstractStepTestCase { - - @Override - public FreezeStep createRandomInstance() { - StepKey stepKey = randomStepKey(); - StepKey nextStepKey = randomStepKey(); - - return new FreezeStep(stepKey, nextStepKey, client); - } - - @Override - public FreezeStep mutateInstance(FreezeStep instance) { - StepKey key = instance.getKey(); - StepKey nextKey = instance.getNextStepKey(); - - switch (between(0, 1)) { - case 0 -> key = new StepKey(key.phase(), key.action(), key.name() + randomAlphaOfLength(5)); - case 1 -> nextKey = new StepKey(nextKey.phase(), nextKey.action(), nextKey.name() + randomAlphaOfLength(5)); - default -> throw new AssertionError("Illegal randomisation branch"); - } - - return new FreezeStep(key, nextKey, instance.getClient()); - } - - @Override - public FreezeStep copyInstance(FreezeStep instance) { - return new FreezeStep(instance.getKey(), instance.getNextStepKey(), instance.getClient()); - } - - public void testIndexSurvives() { - assertTrue(createRandomInstance().indexSurvives()); - } -} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java index 1bea0ac6d192c..4d34115919710 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java @@ -35,6 +35,7 @@ public class LifecyclePolicyTests extends AbstractXContentSerializingTestCase { + // Excluding the deprecated freeze action and test it separately private String lifecycleName; @Override diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java index 28f97adec8814..91afa4da560b5 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.client.WarningFailureException; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; @@ -216,29 +217,27 @@ public void testReadOnlyAction() throws Exception { } public void testFreezeAction() throws Exception { - createNewSingletonPolicy(client(), policyName, "cold", FreezeAction.INSTANCE); + try { + createNewSingletonPolicy(client(), policyName, "cold", FreezeAction.INSTANCE); + } catch (WarningFailureException e) { + assertThat(e.getMessage(), containsString("The freeze action in ILM is deprecated and will be removed in a future version")); + } createComposableTemplate(client(), template, dataStream + "*", getTemplate(policyName)); indexDocument(client(), dataStream, true); + // The freeze action is a noop action with only noop steps and should pass through to complete the phase asap. String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1); - assertBusy( - () -> assertThat( - "index must wait in the " + CheckNotDataStreamWriteIndexStep.NAME + " until it is not the write index anymore", - explainIndex(client(), backingIndexName).get("step"), - is(CheckNotDataStreamWriteIndexStep.NAME) - ), - 30, - TimeUnit.SECONDS - ); - - // Manual rollover the original index such that it's not the write index in the data stream anymore - rolloverMaxOneDocCondition(client(), dataStream); - - assertBusy( - () -> assertThat(explainIndex(client(), backingIndexName).get("step"), is(PhaseCompleteStep.NAME)), - 30, - TimeUnit.SECONDS - ); + assertBusy(() -> { + try { + assertThat(explainIndex(client(), backingIndexName).get("step"), is(PhaseCompleteStep.NAME)); + fail("expected a deprecation warning"); + } catch (WarningFailureException e) { + assertThat( + e.getMessage(), + containsString("The freeze action in ILM is deprecated and will be removed in a future version") + ); + } + }, 30, TimeUnit.SECONDS); Map settings = getOnlyIndexSettings(client(), backingIndexName); assertNull(settings.get("index.frozen")); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java index 4c53d711ffdef..23a8dbd220118 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java @@ -14,6 +14,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.WarningFailureException; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; @@ -190,7 +191,11 @@ public void testUpdatePolicyToNotContainFailedStep() throws Exception { } public void testFreezeNoop() throws Exception { - createNewSingletonPolicy(client(), policy, "cold", FreezeAction.INSTANCE); + try { + createNewSingletonPolicy(client(), policy, "cold", FreezeAction.INSTANCE); + } catch (WarningFailureException e) { + assertThat(e.getMessage(), containsString("The freeze action in ILM is deprecated and will be removed in a future version")); + } createIndexWithSettings( client(), @@ -202,11 +207,16 @@ public void testFreezeNoop() throws Exception { .put("index.lifecycle.name", policy) ); - assertBusy( - () -> assertThat(getStepKeyForIndex(client(), index), equalTo(PhaseCompleteStep.finalStep("cold").getKey())), - 30, - TimeUnit.SECONDS - ); + assertBusy(() -> { + try { + assertThat(getStepKeyForIndex(client(), index), equalTo(PhaseCompleteStep.finalStep("cold").getKey())); + } catch (WarningFailureException e) { + assertThat( + e.getMessage(), + containsString("The freeze action in ILM is deprecated and will be removed in a future version") + ); + } + }, 30, TimeUnit.SECONDS); assertFalse(getOnlyIndexSettings(client(), index).containsKey("index.frozen")); } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java index 15e59a1593337..bae3064971a5e 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.WarningFailureException; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; @@ -361,9 +362,10 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex null ); + String template = randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT); createComposableTemplate( client(), - randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT), + template, dataStream, new Template( Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 5).put(LifecycleSettings.LIFECYCLE_NAME, policy).build(), @@ -407,19 +409,24 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex // snapshot) assertOK(client().performRequest(new Request("DELETE", "/_data_stream/" + dataStream))); - createPolicy( - client(), - policy, - new Phase("hot", TimeValue.ZERO, Map.of()), - new Phase( - "warm", - TimeValue.ZERO, - Map.of(ShrinkAction.NAME, new ShrinkAction(1, null, false), ForceMergeAction.NAME, new ForceMergeAction(1, null)) - ), - new Phase("cold", TimeValue.ZERO, Map.of(FreezeAction.NAME, FreezeAction.INSTANCE)), - null, - null - ); + try { + createPolicy( + client(), + policy, + new Phase("hot", TimeValue.ZERO, Map.of()), + new Phase( + "warm", + TimeValue.ZERO, + Map.of(ShrinkAction.NAME, new ShrinkAction(1, null, false), ForceMergeAction.NAME, new ForceMergeAction(1, null)) + ), + new Phase("cold", TimeValue.ZERO, Map.of(FreezeAction.NAME, FreezeAction.INSTANCE)), + null, + null + ); + fail("Expected a deprecation warning."); + } catch (WarningFailureException e) { + assertThat(e.getMessage(), containsString("The freeze action in ILM is deprecated and will be removed in a future version")); + } // restore the datastream Request restoreSnapshot = new Request("POST", "/_snapshot/" + snapshotRepo + "/" + dsSnapshotName + "/_restore"); @@ -432,7 +439,16 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex // the restored index is now managed by the now updated ILM policy and needs to go through the warm and cold phase assertBusy(() -> { - Step.StepKey stepKeyForIndex = getStepKeyForIndex(client(), searchableSnapMountedIndexName); + Step.StepKey stepKeyForIndex; + try { + stepKeyForIndex = getStepKeyForIndex(client(), searchableSnapMountedIndexName); + } catch (WarningFailureException e) { + assertThat( + e.getMessage(), + containsString("The freeze action in ILM is deprecated and will be removed in a future version") + ); + stepKeyForIndex = getKeyForIndex(e.getResponse(), searchableSnapMountedIndexName); + } assertThat(stepKeyForIndex.phase(), is("cold")); assertThat(stepKeyForIndex.name(), is(PhaseCompleteStep.NAME)); }, 30, TimeUnit.SECONDS); @@ -984,4 +1000,18 @@ private void triggerStateChange() throws IOException { Request rerouteRequest = new Request("POST", "/_cluster/reroute"); client().performRequest(rerouteRequest); } + + private Step.StepKey getKeyForIndex(Response response, String indexName) throws IOException { + Map responseMap; + try (InputStream is = response.getEntity().getContent()) { + responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true); + } + + @SuppressWarnings("unchecked") + Map indexResponse = ((Map>) responseMap.get("indices")).get(indexName); + String phase = (String) indexResponse.get("phase"); + String action = (String) indexResponse.get("action"); + String step = (String) indexResponse.get("step"); + return new Step.StepKey(phase, action, step); + } } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java index c50ea682ca9a2..85acc3fa8da6f 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java @@ -153,6 +153,7 @@ static IndexLifecycleExplainResponse getIndexLifecycleExplainResponse( // Try to add default rollover conditions to the response. var phase = phaseExecutionInfo.getPhase(); if (phase != null) { + phase.maybeAddDeprecationWarningForFreezeAction(policyName); var rolloverAction = (RolloverAction) phase.getActions().get(RolloverAction.NAME); if (rolloverAction != null) { var conditions = applyDefaultConditions(rolloverAction.getConditions(), rolloverOnlyIfHasDocuments); diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java index 92d182ea6d44a..7bdb95ff14673 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java @@ -109,7 +109,7 @@ protected void masterOperation( Map filteredHeaders = ClientHelper.getPersistableSafeSecurityHeaders(threadPool.getThreadContext(), state); LifecyclePolicy.validatePolicyName(request.getPolicy().getName()); - + request.getPolicy().maybeAddDeprecationWarningForFreezeAction(request.getPolicy().getName()); { IndexLifecycleMetadata lifecycleMetadata = state.metadata().custom(IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata.EMPTY); LifecyclePolicyMetadata existingPolicy = lifecycleMetadata.getPolicyMetadatas().get(request.getPolicy().getName()); From 79713f5e990bb9d95b0fed8aac6c966ac97a2f27 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Fri, 10 Jan 2025 11:30:44 +0100 Subject: [PATCH 02/13] Restrict Connector APIs to manage/monitor_connector privileges (#119863) * Reapply "Restrict Connector APIs to manage/monitor_connector privileges (#119389)" (#119833) This reverts commit e0cefb8ff0662943f91d2fd0245a6c57309a984a. * Update docs/changelog/119863.yaml * Update docs/changelog/119863.yaml * Update changelog * Fix changelog --------- Co-authored-by: Elastic Machine --- docs/changelog/119863.yaml | 11 + x-pack/plugin/ent-search/qa/rest/roles.yml | 8 +- .../connector/ConnectorIndexService.java | 221 ++++++++++-------- .../action/ConnectorActionRequest.java | 18 +- .../action/DeleteConnectorAction.java | 11 +- .../connector/action/GetConnectorAction.java | 2 +- .../connector/action/ListConnectorAction.java | 2 +- .../connector/action/PostConnectorAction.java | 2 +- .../connector/action/PutConnectorAction.java | 5 +- .../UpdateConnectorActiveFilteringAction.java | 2 +- .../action/UpdateConnectorApiKeyIdAction.java | 2 +- .../UpdateConnectorConfigurationAction.java | 2 +- .../action/UpdateConnectorErrorAction.java | 2 +- .../action/UpdateConnectorFeaturesAction.java | 2 +- .../UpdateConnectorFilteringAction.java | 2 +- ...ateConnectorFilteringValidationAction.java | 2 +- .../UpdateConnectorIndexNameAction.java | 2 +- .../action/UpdateConnectorLastSeenAction.java | 2 +- .../UpdateConnectorLastSyncStatsAction.java | 2 +- .../action/UpdateConnectorNameAction.java | 2 +- .../action/UpdateConnectorNativeAction.java | 2 +- .../action/UpdateConnectorPipelineAction.java | 2 +- .../UpdateConnectorSchedulingAction.java | 2 +- .../UpdateConnectorServiceTypeAction.java | 2 +- .../action/UpdateConnectorStatusAction.java | 2 +- .../syncjob/ConnectorSyncJobIndexService.java | 29 +-- .../action/CancelConnectorSyncJobAction.java | 2 +- .../action/CheckInConnectorSyncJobAction.java | 2 +- .../action/ClaimConnectorSyncJobAction.java | 2 +- .../action/ConnectorSyncJobActionRequest.java | 17 +- .../action/DeleteConnectorSyncJobAction.java | 2 +- .../action/GetConnectorSyncJobAction.java | 2 +- .../action/ListConnectorSyncJobsAction.java | 2 +- .../action/PostConnectorSyncJobAction.java | 11 +- .../UpdateConnectorSyncJobErrorAction.java | 2 +- ...eConnectorSyncJobIngestionStatsAction.java | 2 +- .../xpack/security/operator/Constants.java | 60 ++--- 37 files changed, 224 insertions(+), 221 deletions(-) create mode 100644 docs/changelog/119863.yaml diff --git a/docs/changelog/119863.yaml b/docs/changelog/119863.yaml new file mode 100644 index 0000000000000..63cbf1ba07851 --- /dev/null +++ b/docs/changelog/119863.yaml @@ -0,0 +1,11 @@ +pr: 119863 +summary: Restrict Connector APIs to manage/monitor_connector privileges +area: Extract&Transform +type: breaking +issues: [] +breaking: + title: Restrict Connector APIs to manage/monitor_connector privileges + area: REST API + details: Connector APIs now enforce the manage_connector and monitor_connector privileges (introduced in 8.15), replacing the previous reliance on index-level permissions for .elastic-connectors and .elastic-connectors-sync-jobs in API calls. + impact: Connector APIs now require manage_connector and monitor_connector privileges + notable: false diff --git a/x-pack/plugin/ent-search/qa/rest/roles.yml b/x-pack/plugin/ent-search/qa/rest/roles.yml index d32f05b7b749e..661ba482c6367 100644 --- a/x-pack/plugin/ent-search/qa/rest/roles.yml +++ b/x-pack/plugin/ent-search/qa/rest/roles.yml @@ -4,13 +4,12 @@ admin: - manage_behavioral_analytics - manage - monitor + - manage_connector indices: - names: [ # indices and search applications "test-*", "another-test-search-application", - ".elastic-connectors-v1", - ".elastic-connectors-sync-jobs-v1" ] privileges: [ "manage", "write", "read" ] @@ -20,6 +19,7 @@ user: - manage_api_key - read_connector_secrets - write_connector_secrets + - monitor_connector indices: - names: [ "test-index1", @@ -27,9 +27,7 @@ user: "test-search-application-1", "test-search-application-with-aggs", "test-search-application-with-list", - "test-search-application-with-list-invalid", - ".elastic-connectors-v1", - ".elastic-connectors-sync-jobs-v1" + "test-search-application-with-list-invalid" ] privileges: [ "read" ] diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index 4c0b4cf814a07..bb80f5fee4ec9 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.common.Strings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -74,13 +75,15 @@ import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.fromXContentBytesConnectorFiltering; import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.sortFilteringRulesByOrder; import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.MANAGED_CONNECTOR_INDEX_PREFIX; +import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN; /** * A service that manages persistent {@link Connector} configurations. */ public class ConnectorIndexService { - private final Client client; + // The client to interact with the system index (internal user). + private final Client clientWithOrigin; public static final String CONNECTOR_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN; @@ -88,7 +91,7 @@ public class ConnectorIndexService { * @param client A client for executing actions on the connector index */ public ConnectorIndexService(Client client) { - this.client = client; + this.clientWithOrigin = new OriginSettingClient(client, CONNECTORS_ORIGIN); } /** @@ -134,7 +137,7 @@ public void createConnector( indexRequest = indexRequest.id(connectorId); } - client.index( + clientWithOrigin.index( indexRequest, listener.delegateFailureAndWrap( (ll, indexResponse) -> ll.onResponse( @@ -201,7 +204,7 @@ public void getConnector(String connectorId, boolean includeDeleted, ActionListe try { final GetRequest getRequest = new GetRequest(CONNECTOR_INDEX_NAME).id(connectorId).realtime(true); - client.get(getRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, getResponse) -> { + clientWithOrigin.get(getRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, getResponse) -> { if (getResponse.isExists() == false) { l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); return; @@ -248,13 +251,16 @@ public void deleteConnector(String connectorId, boolean shouldDeleteSyncJobs, Ac .id(connectorId) .source(Map.of(Connector.IS_DELETED_FIELD.getPreferredName(), true)) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> { + clientWithOrigin.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> { if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); return; } if (shouldDeleteSyncJobs) { - new ConnectorSyncJobIndexService(client).deleteAllSyncJobsByConnectorId(connectorId, ll.map(r -> updateResponse)); + new ConnectorSyncJobIndexService(clientWithOrigin).deleteAllSyncJobsByConnectorId( + connectorId, + ll.map(r -> updateResponse) + ); } else { ll.onResponse(updateResponse); } @@ -294,7 +300,7 @@ public void listConnectors( .fetchSource(true) .sort(Connector.INDEX_NAME_FIELD.getPreferredName(), SortOrder.ASC); final SearchRequest req = new SearchRequest(CONNECTOR_INDEX_NAME).source(source); - client.search(req, new ActionListener<>() { + clientWithOrigin.search(req, new ActionListener<>() { @Override public void onResponse(SearchResponse searchResponse) { try { @@ -476,7 +482,7 @@ else if (configurationValues != null) { return; } - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> { + clientWithOrigin.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> { if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); return; @@ -513,13 +519,16 @@ public void updateConnectorError(String connectorId, String error, ActionListene } }) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + }) + ); } catch (Exception e) { listener.onFailure(e); } @@ -541,13 +550,16 @@ public void updateConnectorNameOrDescription(UpdateConnectorNameAction.Request r .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + }) + ); } catch (Exception e) { listener.onFailure(e); } @@ -568,13 +580,16 @@ public void updateConnectorFiltering(String connectorId, List(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + }) + ); } catch (Exception e) { listener.onFailure(e); } @@ -595,13 +610,16 @@ public void updateConnectorFeatures(String connectorId, ConnectorFeatures featur .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(Map.of(Connector.FEATURES_FIELD.getPreferredName(), features)) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + }) + ); } catch (Exception e) { listener.onFailure(e); } @@ -657,13 +675,16 @@ public void updateConnectorFilteringDraft( .source(Map.of(Connector.FILTERING_FIELD.getPreferredName(), List.of(connectorFilteringWithUpdatedDraft))) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (ll, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - ll.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (ll, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + ll.onResponse(updateResponse); + }) + ); })); } catch (Exception e) { @@ -705,7 +726,7 @@ public void updateConnectorDraftFilteringValidation( .source(Map.of(Connector.FILTERING_FIELD.getPreferredName(), List.of(activatedConnectorFiltering))) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> { + clientWithOrigin.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> { if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); return; @@ -762,7 +783,7 @@ public void activateConnectorDraftFiltering(String connectorId, ActionListener(connectorId, l, (ll, updateResponse) -> { + clientWithOrigin.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> { if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); return; @@ -790,13 +811,16 @@ public void checkInConnector(String connectorId, ActionListener .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(Map.of(Connector.LAST_SEEN_FIELD.getPreferredName(), Instant.now())) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + }) + ); } catch (Exception e) { listener.onFailure(e); } @@ -817,13 +841,16 @@ public void updateConnectorLastSyncStats(UpdateConnectorLastSyncStatsAction.Requ .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + }) + ); } catch (Exception e) { listener.onFailure(e); } @@ -887,13 +914,16 @@ public void updateConnectorNative(UpdateConnectorNativeAction.Request request, A ) ) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (ll, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - ll.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (ll, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + ll.onResponse(updateResponse); + }) + ); })); } catch (Exception e) { listener.onFailure(e); @@ -916,13 +946,16 @@ public void updateConnectorPipeline(UpdateConnectorPipelineAction.Request reques .source(Map.of(Connector.PIPELINE_FIELD.getPreferredName(), request.getPipeline())) .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + }) + ); } catch (Exception e) { listener.onFailure(e); } @@ -981,7 +1014,7 @@ public void updateConnectorIndexName(UpdateConnectorIndexNameAction.Request requ } }) ); - client.update( + clientWithOrigin.update( updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (lll, updateResponse) -> { if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { @@ -1014,13 +1047,16 @@ public void updateConnectorScheduling(UpdateConnectorSchedulingAction.Request re .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(Map.of(Connector.SCHEDULING_FIELD.getPreferredName(), request.getScheduling())) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + }) + ); } catch (Exception e) { listener.onFailure(e); } @@ -1056,7 +1092,7 @@ public void updateConnectorServiceType(UpdateConnectorServiceTypeAction.Request ) ); - client.update( + clientWithOrigin.update( updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (updateListener, updateResponse) -> { if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { @@ -1099,7 +1135,7 @@ public void updateConnectorStatus(UpdateConnectorStatusAction.Request request, A .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(Map.of(Connector.STATUS_FIELD.getPreferredName(), request.getStatus())) ); - client.update( + clientWithOrigin.update( updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (updateListener, updateResponse) -> { if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { @@ -1127,13 +1163,16 @@ public void updateConnectorApiKeyIdOrApiKeySecretId( .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + }) + ); } catch (Exception e) { listener.onFailure(e); } @@ -1211,7 +1250,7 @@ private void isDataIndexNameAlreadyInUse(String indexName, String connectorId, A final SearchSourceBuilder searchSource = new SearchSourceBuilder().query(boolFilterQueryBuilder); final SearchRequest searchRequest = new SearchRequest(CONNECTOR_INDEX_NAME).source(searchSource); - client.search(searchRequest, new ActionListener<>() { + clientWithOrigin.search(searchRequest, new ActionListener<>() { @Override public void onResponse(SearchResponse searchResponse) { boolean indexNameIsInUse = searchResponse.getHits().getTotalHits().value() > 0L; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorActionRequest.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorActionRequest.java index 66f347bc4dbb4..723ff12b3b1e1 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorActionRequest.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorActionRequest.java @@ -9,12 +9,9 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.IndicesRequest; -import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.indices.InvalidIndexNameException; -import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; import java.io.IOException; @@ -22,10 +19,9 @@ import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.MANAGED_CONNECTOR_INDEX_PREFIX; /** - * Abstract base class for action requests targeting the connectors index. Implements {@link org.elasticsearch.action.IndicesRequest} - * to ensure index-level privilege support. This class defines the connectors index as the target for all derived action requests. + * Abstract base class for action requests targeting the connectors index. */ -public abstract class ConnectorActionRequest extends ActionRequest implements IndicesRequest { +public abstract class ConnectorActionRequest extends ActionRequest { public ConnectorActionRequest() { super(); @@ -78,14 +74,4 @@ public ActionRequestValidationException validateManagedConnectorIndexPrefix( } return validationException; } - - @Override - public String[] indices() { - return new String[] { ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN }; - } - - @Override - public IndicesOptions indicesOptions() { - return IndicesOptions.lenientExpandHidden(); - } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java index 930068a2a46ef..5d98f9703ecea 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java @@ -18,7 +18,6 @@ import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; import java.io.IOException; import java.util.Objects; @@ -28,7 +27,7 @@ public class DeleteConnectorAction { - public static final String NAME = "indices:data/write/xpack/connector/delete"; + public static final String NAME = "cluster:admin/xpack/connector/delete"; public static final ActionType INSTANCE = new ActionType<>(NAME); private DeleteConnectorAction() {/* no instances */} @@ -71,14 +70,6 @@ public boolean shouldDeleteSyncJobs() { return deleteSyncJobs; } - @Override - public String[] indices() { - // When deleting a connector, corresponding sync jobs can also be deleted - return new String[] { - ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN, - ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN }; - } - @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java index 82f6cf77def58..6f3401e6d85b9 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java @@ -30,7 +30,7 @@ public class GetConnectorAction { - public static final String NAME = "indices:data/read/xpack/connector/get"; + public static final String NAME = "cluster:admin/xpack/connector/get"; public static final ActionType INSTANCE = new ActionType<>(NAME); private GetConnectorAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java index 1220c3c58fbb8..048f651ab2a1c 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java @@ -35,7 +35,7 @@ public class ListConnectorAction { - public static final String NAME = "indices:data/read/xpack/connector/list"; + public static final String NAME = "cluster:admin/xpack/connector/list"; public static final ActionType INSTANCE = new ActionType<>(NAME); private ListConnectorAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PostConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PostConnectorAction.java index b1c38637298c4..b5087634a77f2 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PostConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PostConnectorAction.java @@ -25,7 +25,7 @@ public class PostConnectorAction { - public static final String NAME = "indices:data/write/xpack/connector/post"; + public static final String NAME = "cluster:admin/xpack/connector/post"; public static final ActionType INSTANCE = new ActionType<>(NAME); private PostConnectorAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java index f3e8ed6b6e76d..c5922ebfafe1b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java @@ -9,7 +9,6 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -27,12 +26,12 @@ public class PutConnectorAction { - public static final String NAME = "indices:data/write/xpack/connector/put"; + public static final String NAME = "cluster:admin/xpack/connector/put"; public static final ActionType INSTANCE = new ActionType<>(NAME); private PutConnectorAction() {/* no instances */} - public static class Request extends ConnectorActionRequest implements IndicesRequest, ToXContentObject { + public static class Request extends ConnectorActionRequest implements ToXContentObject { @Nullable private final String connectorId; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorActiveFilteringAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorActiveFilteringAction.java index 7b4ce08ef8320..a7bd9c2382331 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorActiveFilteringAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorActiveFilteringAction.java @@ -22,7 +22,7 @@ public class UpdateConnectorActiveFilteringAction { - public static final String NAME = "indices:data/write/xpack/connector/update_filtering/activate"; + public static final String NAME = "cluster:admin/xpack/connector/update_filtering/activate"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorActiveFilteringAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdAction.java index 7f726f21ce225..e76c5191a58a6 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdAction.java @@ -27,7 +27,7 @@ public class UpdateConnectorApiKeyIdAction { - public static final String NAME = "indices:data/write/xpack/connector/update_api_key_id"; + public static final String NAME = "cluster:admin/xpack/connector/update_api_key_id"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorApiKeyIdAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java index 5d36c5f886ea0..6948667fa7351 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java @@ -32,7 +32,7 @@ public class UpdateConnectorConfigurationAction { - public static final String NAME = "indices:data/write/xpack/connector/update_configuration"; + public static final String NAME = "cluster:admin/xpack/connector/update_configuration"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorConfigurationAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java index 3e506fc835f65..a8c2b334cfee3 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java @@ -27,7 +27,7 @@ public class UpdateConnectorErrorAction { - public static final String NAME = "indices:data/write/xpack/connector/update_error"; + public static final String NAME = "cluster:admin/xpack/connector/update_error"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorErrorAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java index 56656855583aa..4bd51794c1f9e 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java @@ -27,7 +27,7 @@ public class UpdateConnectorFeaturesAction { - public static final String NAME = "indices:data/write/xpack/connector/update_features"; + public static final String NAME = "cluster:admin/xpack/connector/update_features"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorFeaturesAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java index 660956b2e9d7f..e527e79161f2f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java @@ -32,7 +32,7 @@ public class UpdateConnectorFilteringAction { - public static final String NAME = "indices:data/write/xpack/connector/update_filtering"; + public static final String NAME = "cluster:admin/xpack/connector/update_filtering"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorFilteringAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringValidationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringValidationAction.java index 92291506d0719..a22b08152f509 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringValidationAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringValidationAction.java @@ -27,7 +27,7 @@ public class UpdateConnectorFilteringValidationAction { - public static final String NAME = "indices:data/write/xpack/connector/update_filtering/draft_validation"; + public static final String NAME = "cluster:admin/xpack/connector/update_filtering/draft_validation"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorFilteringValidationAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameAction.java index e7840e1f84fad..ddc98d4756dc4 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameAction.java @@ -27,7 +27,7 @@ public class UpdateConnectorIndexNameAction { - public static final String NAME = "indices:data/write/xpack/connector/update_index_name"; + public static final String NAME = "cluster:admin/xpack/connector/update_index_name"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorIndexNameAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java index f72938ec8dba2..deae10d901c13 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java @@ -23,7 +23,7 @@ public class UpdateConnectorLastSeenAction { - public static final String NAME = "indices:data/write/xpack/connector/update_last_seen"; + public static final String NAME = "cluster:admin/xpack/connector/update_last_seen"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorLastSeenAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java index ae3be3801786c..029e261e51fab 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java @@ -32,7 +32,7 @@ public class UpdateConnectorLastSyncStatsAction { - public static final String NAME = "indices:data/write/xpack/connector/update_last_sync_stats"; + public static final String NAME = "cluster:admin/xpack/connector/update_last_sync_stats"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorLastSyncStatsAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java index bbc1f992b48e2..5a63bb106747f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java @@ -27,7 +27,7 @@ public class UpdateConnectorNameAction { - public static final String NAME = "indices:data/write/xpack/connector/update_name"; + public static final String NAME = "cluster:admin/xpack/connector/update_name"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorNameAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNativeAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNativeAction.java index 7b3f2e4577f4e..d20b535d5cc57 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNativeAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNativeAction.java @@ -26,7 +26,7 @@ public class UpdateConnectorNativeAction { - public static final String NAME = "indices:data/write/xpack/connector/update_native"; + public static final String NAME = "cluster:admin/xpack/connector/update_native"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorNativeAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java index e58d614f4ef21..2e738033c3384 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java @@ -27,7 +27,7 @@ public class UpdateConnectorPipelineAction { - public static final String NAME = "indices:data/write/xpack/connector/update_pipeline"; + public static final String NAME = "cluster:admin/xpack/connector/update_pipeline"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorPipelineAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java index 578639f065a0b..65cfd645ea60b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java @@ -32,7 +32,7 @@ public class UpdateConnectorSchedulingAction { - public static final String NAME = "indices:data/write/xpack/connector/update_scheduling"; + public static final String NAME = "cluster:admin/xpack/connector/update_scheduling"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorSchedulingAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorServiceTypeAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorServiceTypeAction.java index de07a6db21bab..1d4df12e10eba 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorServiceTypeAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorServiceTypeAction.java @@ -26,7 +26,7 @@ public class UpdateConnectorServiceTypeAction { - public static final String NAME = "indices:data/write/xpack/connector/update_service_type"; + public static final String NAME = "cluster:admin/xpack/connector/update_service_type"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorServiceTypeAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusAction.java index aebaa0afb9052..79f097db2dec9 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusAction.java @@ -28,7 +28,7 @@ public class UpdateConnectorStatusAction { - public static final String NAME = "indices:data/write/xpack/connector/update_status"; + public static final String NAME = "cluster:admin/xpack/connector/update_status"; public static final ActionType INSTANCE = new ActionType<>(NAME); public UpdateConnectorStatusAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java index ce6f7f0dbf2b2..f46d915a7123f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.common.Strings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.engine.DocumentMissingException; @@ -68,6 +69,7 @@ import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.application.connector.ConnectorIndexService.CONNECTOR_INDEX_NAME; +import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN; /** * A service that manages persistent {@link ConnectorSyncJob} configurations. @@ -76,7 +78,8 @@ public class ConnectorSyncJobIndexService { private static final Long ZERO = 0L; - private final Client client; + // The client to interact with the system index (internal user). + private final Client clientWithOrigin; public static final String CONNECTOR_SYNC_JOB_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN; @@ -84,7 +87,7 @@ public class ConnectorSyncJobIndexService { * @param client A client for executing actions on the connectors sync jobs index. */ public ConnectorSyncJobIndexService(Client client) { - this.client = client; + this.clientWithOrigin = new OriginSettingClient(client, CONNECTORS_ORIGIN); } /** @@ -149,7 +152,7 @@ public void createConnectorSyncJob( indexRequest.source(syncJob.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); - client.index( + clientWithOrigin.index( indexRequest, l.delegateFailureAndWrap( (ll, indexResponse) -> ll.onResponse(new PostConnectorSyncJobAction.Response(indexResponse.getId())) @@ -175,7 +178,7 @@ public void deleteConnectorSyncJob(String connectorSyncJobId, ActionListener(connectorSyncJobId, listener, (l, deleteResponse) -> { if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { @@ -205,7 +208,7 @@ public void checkInConnectorSyncJob(String connectorSyncJobId, ActionListener(connectorSyncJobId, listener, (l, updateResponse) -> { if (updateResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { @@ -230,7 +233,7 @@ public void getConnectorSyncJob(String connectorSyncJobId, ActionListener(connectorSyncJobId, listener, (l, getResponse) -> { if (getResponse.isExists() == false) { @@ -306,7 +309,7 @@ public void cancelConnectorSyncJob(String connectorSyncJobId, ActionListener( connectorSyncJobId, @@ -355,7 +358,7 @@ public void listConnectorSyncJobs( final SearchRequest searchRequest = new SearchRequest(CONNECTOR_SYNC_JOB_INDEX_NAME).source(searchSource); - client.search(searchRequest, new ActionListener<>() { + clientWithOrigin.search(searchRequest, new ActionListener<>() { @Override public void onResponse(SearchResponse searchResponse) { try { @@ -474,7 +477,7 @@ public void updateConnectorSyncJobIngestionStats( ).doc(fieldsToUpdate); try { - client.update( + clientWithOrigin.update( updateRequest, new DelegatingIndexNotFoundOrDocumentMissingActionListener<>(syncJobId, listener, (l, updateResponse) -> { if (updateResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { @@ -501,7 +504,7 @@ private void getSyncJobConnectorInfo(String connectorId, ConnectorSyncJobType jo final GetRequest request = new GetRequest(CONNECTOR_INDEX_NAME, connectorId); - client.get(request, new ActionListener<>() { + clientWithOrigin.get(request, new ActionListener<>() { @Override public void onResponse(GetResponse response) { final boolean connectorDoesNotExist = response.isExists() == false; @@ -594,7 +597,7 @@ public void updateConnectorSyncJobError(String connectorSyncJobId, String error, ) ); - client.update( + clientWithOrigin.update( updateRequest, new DelegatingIndexNotFoundOrDocumentMissingActionListener<>( connectorSyncJobId, @@ -629,7 +632,7 @@ public void deleteAllSyncJobsByConnectorId(String connectorId, ActionListener { + clientWithOrigin.execute(DeleteByQueryAction.INSTANCE, deleteByQueryRequest, listener.delegateFailureAndWrap((l, r) -> { final List bulkDeleteFailures = r.getBulkFailures(); if (bulkDeleteFailures.isEmpty() == false) { l.onFailure( @@ -681,7 +684,7 @@ public void claimConnectorSyncJob( WriteRequest.RefreshPolicy.IMMEDIATE ).doc(document); - client.update( + clientWithOrigin.update( updateRequest, new DelegatingIndexNotFoundOrDocumentMissingActionListener<>( connectorSyncJobId, diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java index 160eda3aeef7c..658d48e9752d0 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java @@ -28,7 +28,7 @@ public class CancelConnectorSyncJobAction { - public static final String NAME = "indices:data/write/xpack/connector/sync_job/cancel"; + public static final String NAME = "cluster:admin/xpack/connector/sync_job/cancel"; public static final ActionType INSTANCE = new ActionType(NAME); private CancelConnectorSyncJobAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java index aa6dea50de464..ae44813354eb9 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java @@ -28,7 +28,7 @@ public class CheckInConnectorSyncJobAction { - public static final String NAME = "indices:data/write/xpack/connector/sync_job/check_in"; + public static final String NAME = "cluster:admin/xpack/connector/sync_job/check_in"; public static final ActionType INSTANCE = new ActionType<>(NAME); private CheckInConnectorSyncJobAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobAction.java index b108116a5e68c..84e3183c2830c 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobAction.java @@ -31,7 +31,7 @@ public class ClaimConnectorSyncJobAction { public static final ParseField CONNECTOR_SYNC_JOB_ID_FIELD = new ParseField("connector_sync_job_id"); - public static final String NAME = "indices:data/write/xpack/connector/sync_job/claim"; + public static final String NAME = "cluster:admin/xpack/connector/sync_job/claim"; public static final ActionType INSTANCE = new ActionType<>(NAME); private ClaimConnectorSyncJobAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ConnectorSyncJobActionRequest.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ConnectorSyncJobActionRequest.java index bb83fd78151df..11a3334aed4e6 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ConnectorSyncJobActionRequest.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ConnectorSyncJobActionRequest.java @@ -8,19 +8,14 @@ package org.elasticsearch.xpack.application.connector.syncjob.action; import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.IndicesRequest; -import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; import java.io.IOException; /** * Abstract base class for action requests targeting the connector sync job index. - * Implements {@link org.elasticsearch.action.IndicesRequest} to ensure index-level privilege support. - * This class defines the connectors sync job index as the target for all derived action requests. */ -public abstract class ConnectorSyncJobActionRequest extends ActionRequest implements IndicesRequest { +public abstract class ConnectorSyncJobActionRequest extends ActionRequest { public ConnectorSyncJobActionRequest() { super(); @@ -29,14 +24,4 @@ public ConnectorSyncJobActionRequest() { public ConnectorSyncJobActionRequest(StreamInput in) throws IOException { super(in); } - - @Override - public String[] indices() { - return new String[] { ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN }; - } - - @Override - public IndicesOptions indicesOptions() { - return IndicesOptions.lenientExpandHidden(); - } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobAction.java index c9c84ead2e4b0..935aa1fd6b7d1 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobAction.java @@ -28,7 +28,7 @@ public class DeleteConnectorSyncJobAction { - public static final String NAME = "indices:data/write/xpack/connector/sync_job/delete"; + public static final String NAME = "cluster:admin/xpack/connector/sync_job/delete"; public static final ActionType INSTANCE = new ActionType<>(NAME); private DeleteConnectorSyncJobAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobAction.java index 7ff82b9881fb4..485e516c80c39 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobAction.java @@ -29,7 +29,7 @@ public class GetConnectorSyncJobAction { - public static final String NAME = "indices:data/read/xpack/connector/sync_job/get"; + public static final String NAME = "cluster:admin/xpack/connector/sync_job/get"; public static final ActionType INSTANCE = new ActionType<>(NAME); private GetConnectorSyncJobAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsAction.java index 5b34e05ae37f0..04b765f3b5687 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsAction.java @@ -33,7 +33,7 @@ public class ListConnectorSyncJobsAction { - public static final String NAME = "indices:data/read/xpack/connector/sync_job/list"; + public static final String NAME = "cluster:admin/xpack/connector/sync_job/list"; public static final ActionType INSTANCE = new ActionType<>(NAME); private ListConnectorSyncJobsAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/PostConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/PostConnectorSyncJobAction.java index 8c1d24e466daa..0d17a6dba6c35 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/PostConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/PostConnectorSyncJobAction.java @@ -18,7 +18,6 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.Connector; -import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTriggerMethod; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobType; @@ -32,7 +31,7 @@ public class PostConnectorSyncJobAction { - public static final String NAME = "indices:data/write/xpack/connector/sync_job/post"; + public static final String NAME = "cluster:admin/xpack/connector/sync_job/post"; public static final ActionType INSTANCE = new ActionType<>(NAME); private PostConnectorSyncJobAction() {/* no instances */} @@ -140,14 +139,6 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(id, jobType, triggerMethod); } - - @Override - public String[] indices() { - // Creating a new sync job requires reading from connector index - return new String[] { - ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN, - ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN }; - } } public static class Response extends ActionResponse implements ToXContentObject { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java index 2235ba7cfe720..0d5f57c202d9f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java @@ -28,7 +28,7 @@ public class UpdateConnectorSyncJobErrorAction { - public static final String NAME = "indices:data/write/xpack/connector/sync_job/update_error"; + public static final String NAME = "cluster:admin/xpack/connector/sync_job/update_error"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorSyncJobErrorAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java index 0fd9b6dec8184..c890ca0d69bc6 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java @@ -35,7 +35,7 @@ public class UpdateConnectorSyncJobIngestionStatsAction { - public static final String NAME = "indices:data/write/xpack/connector/sync_job/update_stats"; + public static final String NAME = "cluster:admin/xpack/connector/sync_job/update_stats"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorSyncJobIngestionStatsAction() {/* no instances */} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 62687e42b0912..82bad85fa34dc 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -134,40 +134,40 @@ public class Constants { "cluster:admin/xpack/ccr/auto_follow_pattern/put", "cluster:admin/xpack/ccr/pause_follow", "cluster:admin/xpack/ccr/resume_follow", - "indices:data/write/xpack/connector/delete", - "indices:data/read/xpack/connector/get", - "indices:data/read/xpack/connector/list", - "indices:data/write/xpack/connector/post", - "indices:data/write/xpack/connector/put", - "indices:data/write/xpack/connector/update_api_key_id", - "indices:data/write/xpack/connector/update_configuration", - "indices:data/write/xpack/connector/update_error", - "indices:data/write/xpack/connector/update_features", - "indices:data/write/xpack/connector/update_filtering", - "indices:data/write/xpack/connector/update_filtering/activate", - "indices:data/write/xpack/connector/update_filtering/draft_validation", - "indices:data/write/xpack/connector/update_index_name", - "indices:data/write/xpack/connector/update_last_seen", - "indices:data/write/xpack/connector/update_last_sync_stats", - "indices:data/write/xpack/connector/update_name", - "indices:data/write/xpack/connector/update_native", - "indices:data/write/xpack/connector/update_pipeline", - "indices:data/write/xpack/connector/update_scheduling", - "indices:data/write/xpack/connector/update_service_type", - "indices:data/write/xpack/connector/update_status", + "cluster:admin/xpack/connector/delete", + "cluster:admin/xpack/connector/get", + "cluster:admin/xpack/connector/list", + "cluster:admin/xpack/connector/post", + "cluster:admin/xpack/connector/put", + "cluster:admin/xpack/connector/update_api_key_id", + "cluster:admin/xpack/connector/update_configuration", + "cluster:admin/xpack/connector/update_error", + "cluster:admin/xpack/connector/update_features", + "cluster:admin/xpack/connector/update_filtering", + "cluster:admin/xpack/connector/update_filtering/activate", + "cluster:admin/xpack/connector/update_filtering/draft_validation", + "cluster:admin/xpack/connector/update_index_name", + "cluster:admin/xpack/connector/update_last_seen", + "cluster:admin/xpack/connector/update_last_sync_stats", + "cluster:admin/xpack/connector/update_name", + "cluster:admin/xpack/connector/update_native", + "cluster:admin/xpack/connector/update_pipeline", + "cluster:admin/xpack/connector/update_scheduling", + "cluster:admin/xpack/connector/update_service_type", + "cluster:admin/xpack/connector/update_status", "cluster:admin/xpack/connector/secret/delete", "cluster:admin/xpack/connector/secret/get", "cluster:admin/xpack/connector/secret/post", "cluster:admin/xpack/connector/secret/put", - "indices:data/write/xpack/connector/sync_job/cancel", - "indices:data/write/xpack/connector/sync_job/check_in", - "indices:data/write/xpack/connector/sync_job/claim", - "indices:data/write/xpack/connector/sync_job/delete", - "indices:data/read/xpack/connector/sync_job/get", - "indices:data/read/xpack/connector/sync_job/list", - "indices:data/write/xpack/connector/sync_job/post", - "indices:data/write/xpack/connector/sync_job/update_error", - "indices:data/write/xpack/connector/sync_job/update_stats", + "cluster:admin/xpack/connector/sync_job/cancel", + "cluster:admin/xpack/connector/sync_job/check_in", + "cluster:admin/xpack/connector/sync_job/claim", + "cluster:admin/xpack/connector/sync_job/delete", + "cluster:admin/xpack/connector/sync_job/get", + "cluster:admin/xpack/connector/sync_job/list", + "cluster:admin/xpack/connector/sync_job/post", + "cluster:admin/xpack/connector/sync_job/update_error", + "cluster:admin/xpack/connector/sync_job/update_stats", "cluster:admin/xpack/deprecation/info", "cluster:admin/xpack/deprecation/nodes/info", "cluster:admin/xpack/enrich/delete", From 5f9efb5e856a72290ca338c0040b41d1c2719fe8 Mon Sep 17 00:00:00 2001 From: Matteo Piergiovanni <134913285+piergm@users.noreply.github.com> Date: Fri, 10 Jan 2025 11:48:02 +0100 Subject: [PATCH 03/13] fix testCancellationDuringTimeSeriesAggregation (#119925) --- .../aggregations/bucket/SearchCancellationIT.java | 7 +++++-- muted-tests.yml | 3 --- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java index 7adf6a09e9a19..d225ccc9d173f 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java @@ -19,6 +19,7 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.aggregations.AggregationsPlugin; import org.elasticsearch.aggregations.bucket.timeseries.TimeSeriesAggregationBuilder; +import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; @@ -97,9 +98,11 @@ public void testCancellationDuringTimeSeriesAggregation() throws Exception { logger.info("Executing search"); // we have to explicitly set error_trace=true for the later exception check for `TimeSeriesIndexSearcher` - client().threadPool().getThreadContext().putHeader("error_trace", "true"); + Client client = client(); + client.threadPool().getThreadContext().putHeader("error_trace", "true"); TimeSeriesAggregationBuilder timeSeriesAggregationBuilder = new TimeSeriesAggregationBuilder("test_agg"); - ActionFuture searchResponse = prepareSearch("test").setQuery(matchAllQuery()) + ActionFuture searchResponse = client.prepareSearch("test") + .setQuery(matchAllQuery()) .addAggregation( timeSeriesAggregationBuilder.subAggregation( new ScriptedMetricAggregationBuilder("sub_agg").initScript( diff --git a/muted-tests.yml b/muted-tests.yml index 82b9e021521e7..6b7f068ce4593 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -202,9 +202,6 @@ tests: - class: org.elasticsearch.cluster.service.MasterServiceTests method: testThreadContext issue: https://github.com/elastic/elasticsearch/issues/118914 -- class: org.elasticsearch.aggregations.bucket.SearchCancellationIT - method: testCancellationDuringTimeSeriesAggregation - issue: https://github.com/elastic/elasticsearch/issues/118992 - class: org.elasticsearch.xpack.security.authc.AuthenticationServiceTests method: testInvalidToken issue: https://github.com/elastic/elasticsearch/issues/119019 From 5baf5af757359131f88bdca2e8e9161e88180a03 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Fri, 10 Jan 2025 13:22:54 +0200 Subject: [PATCH 04/13] Configure index sorting through index settings for logsdb (#118968) * Skip injecting `host.name` for incompatible mappings in logsdb mode * spotless * Update docs/changelog/118856.yaml * fix tsdb * Configure index sorting through index settings for logsdb * fix synthetic source usage * skip injecting host.name * fix test * fix compat * more tests * add index versioning * add index versioning * add index versioning * minor refactoring * Update docs/changelog/118968.yaml * address comments * inject host.name when possible * check subobjects * private settings --- docs/changelog/118968.yaml | 6 + rest-api-spec/build.gradle | 6 + .../rest-api-spec/test/logsdb/10_settings.yml | 268 +++----------- server/src/main/java/module-info.java | 1 + .../common/settings/IndexScopedSettings.java | 2 + .../elasticsearch/index/IndexFeatures.java | 30 ++ .../org/elasticsearch/index/IndexMode.java | 18 +- .../elasticsearch/index/IndexSettings.java | 27 ++ .../elasticsearch/index/IndexSortConfig.java | 58 ++- .../elasticsearch/index/IndexVersions.java | 3 +- .../elasticsearch/index/mapper/Mapping.java | 2 +- ...lasticsearch.features.FeatureSpecification | 1 + .../index/LogsIndexModeTests.java | 81 +++- .../xpack/logsdb/LogsDBPlugin.java | 11 +- .../LogsdbIndexModeSettingsProvider.java | 84 +++-- .../LogsdbIndexModeSettingsProviderTests.java | 349 +++++++++++++++--- ...dexSettingsProviderLegacyLicenseTests.java | 6 +- .../test/30_logsdb_default_mapping.yml | 93 +++-- 18 files changed, 686 insertions(+), 360 deletions(-) create mode 100644 docs/changelog/118968.yaml create mode 100644 server/src/main/java/org/elasticsearch/index/IndexFeatures.java diff --git a/docs/changelog/118968.yaml b/docs/changelog/118968.yaml new file mode 100644 index 0000000000000..799cd32471f2f --- /dev/null +++ b/docs/changelog/118968.yaml @@ -0,0 +1,6 @@ +pr: 118968 +summary: Configure index sorting through index settings for logsdb +area: Logs +type: enhancement +issues: + - 118686 diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index a104ec675adc2..1da8e906582b1 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -60,6 +60,12 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> task.skipTest("cat.aliases/10_basic/Deprecated local parameter", "CAT APIs not covered by compatibility policy") task.skipTest("cat.shards/10_basic/Help", "sync_id is removed in 9.0") task.skipTest("search/500_date_range/from, to, include_lower, include_upper deprecated", "deprecated parameters are removed in 9.0") + task.skipTest("logsdb/10_settings/logsdb with default ignore dynamic beyond limit and default sorting", "skip until pr/118968 gets backported") + task.skipTest("logsdb/10_settings/logsdb with default ignore dynamic beyond limit and too low limit", "skip until pr/118968 gets backported") + task.skipTest("logsdb/10_settings/logsdb with default ignore dynamic beyond limit and subobjects false", "skip until pr/118968 gets backported") + task.skipTest("logsdb/10_settings/override sort missing settings", "skip until pr/118968 gets backported") + task.skipTest("logsdb/10_settings/override sort order settings", "skip until pr/118968 gets backported") + task.skipTest("logsdb/10_settings/override sort mode settings", "skip until pr/118968 gets backported") task.skipTest("search.vectors/41_knn_search_bbq_hnsw/Test knn search", "Scoring has changed in latest versions") task.skipTest("search.vectors/42_knn_search_bbq_flat/Test knn search", "Scoring has changed in latest versions") task.skipTest("search.vectors/180_update_dense_vector_type/Test create and update dense vector mapping with bulk indexing", "waiting for #118774 backport") diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml index 5f4314f724c23..07af3fb52b92f 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml @@ -10,14 +10,6 @@ setup: --- create logs index: - - requires: - test_runner_features: [ capabilities ] - capabilities: - - method: PUT - path: /{index} - capabilities: [ logsdb_index_mode ] - reason: "Support for 'logsdb' index mode capability required" - - do: indices.create: index: test @@ -78,14 +70,6 @@ create logs index: --- using default timestamp field mapping: - - requires: - test_runner_features: [ capabilities ] - capabilities: - - method: PUT - path: /{index} - capabilities: [ logsdb_index_mode ] - reason: "Support for 'logsdb' index mode capability required" - - do: indices.create: index: test-timestamp-missing @@ -110,14 +94,6 @@ using default timestamp field mapping: --- missing hostname field: - - requires: - test_runner_features: [ capabilities ] - capabilities: - - method: PUT - path: /{index} - capabilities: [ logsdb_index_mode ] - reason: "Support for 'logsdb' index mode capability required" - - do: indices.create: index: test-hostname-missing @@ -149,14 +125,6 @@ missing hostname field: --- missing sort field: - - requires: - test_runner_features: [ capabilities ] - capabilities: - - method: PUT - path: /{index} - capabilities: [ logsdb_index_mode ] - reason: "Support for 'logsdb' index mode capability required" - - do: catch: bad_request indices.create: @@ -190,14 +158,6 @@ missing sort field: --- non-default sort settings: - - requires: - test_runner_features: [ capabilities ] - capabilities: - - method: PUT - path: /{index} - capabilities: [ logsdb_index_mode ] - reason: "Support for 'logsdb' index mode capability required" - - do: indices.create: index: test-sort @@ -244,14 +204,10 @@ non-default sort settings: --- override sort order settings: - requires: - test_runner_features: [ capabilities ] - capabilities: - - method: PUT - path: /{index} - capabilities: [ logsdb_index_mode ] - reason: "Support for 'logsdb' index mode capability required" - + cluster_features: [ "index.logsdb_no_host_name_field" ] + reason: "Change in default sort config for logsdb" - do: + catch: bad_request indices.create: index: test-sort-order body: @@ -278,28 +234,16 @@ override sort order settings: message: type: text - - do: - indices.get_settings: - index: test-sort-order - - - is_true: test-sort-order - - match: { test-sort-order.settings.index.mode: "logsdb" } - - match: { test-sort-order.settings.index.sort.field.0: null } - - match: { test-sort-order.settings.index.sort.field.1: null } - - match: { test-sort-order.settings.index.sort.order.0: "asc" } - - match: { test-sort-order.settings.index.sort.order.1: "asc" } + - match: { error.type: "illegal_argument_exception" } + - match: { error.reason: "index.sort.fields:[] index.sort.order:[asc, asc], size mismatch" } --- override sort missing settings: - requires: - test_runner_features: [ capabilities ] - capabilities: - - method: PUT - path: /{index} - capabilities: [ logsdb_index_mode ] - reason: "Support for 'logsdb' index mode capability required" - + cluster_features: [ "index.logsdb_no_host_name_field" ] + reason: "Change in default sort config for logsdb" - do: + catch: bad_request indices.create: index: test-sort-missing body: @@ -326,28 +270,16 @@ override sort missing settings: message: type: text - - do: - indices.get_settings: - index: test-sort-missing - - - is_true: test-sort-missing - - match: { test-sort-missing.settings.index.mode: "logsdb" } - - match: { test-sort-missing.settings.index.sort.field.0: null } - - match: { test-sort-missing.settings.index.sort.field.1: null } - - match: { test-sort-missing.settings.index.sort.missing.0: "_last" } - - match: { test-sort-missing.settings.index.sort.missing.1: "_first" } + - match: { error.type: "illegal_argument_exception" } + - match: { error.reason: "index.sort.fields:[] index.sort.missing:[_last, _first], size mismatch" } --- override sort mode settings: - requires: - test_runner_features: [ capabilities ] - capabilities: - - method: PUT - path: /{index} - capabilities: [ logsdb_index_mode ] - reason: "Support for 'logsdb' index mode capability required" - + cluster_features: [ "index.logsdb_no_host_name_field" ] + reason: "Change in default sort config for logsdb" - do: + catch: bad_request indices.create: index: test-sort-mode body: @@ -374,16 +306,8 @@ override sort mode settings: message: type: text - - do: - indices.get_settings: - index: test-sort-mode - - - is_true: test-sort-mode - - match: { test-sort-mode.settings.index.mode: "logsdb" } - - match: { test-sort-mode.settings.index.sort.field.0: null } - - match: { test-sort-mode.settings.index.sort.field.1: null } - - match: { test-sort-mode.settings.index.sort.mode.0: "max" } - - match: { test-sort-mode.settings.index.sort.mode.1: "max" } + - match: { error.type: "illegal_argument_exception" } + - match: { error.reason: "index.sort.fields:[] index.sort.mode:[MAX, MAX], size mismatch" } --- override sort field using nested field type in sorting: @@ -436,12 +360,7 @@ override sort field using nested field type in sorting: override sort field using nested field type: - requires: cluster_features: ["mapper.index_sorting_on_nested"] - test_runner_features: [ capabilities ] - capabilities: - - method: PUT - path: /{index} - capabilities: [ logsdb_index_mode ] - reason: "Support for 'logsdb' index mode capability required" + reason: "Support for index sorting on indexes with nested objects required" - do: indices.create: @@ -475,14 +394,6 @@ override sort field using nested field type: --- routing path not allowed in logs mode: - - requires: - test_runner_features: [ capabilities ] - capabilities: - - method: PUT - path: /{index} - capabilities: [ logsdb_index_mode ] - reason: "Support for 'logsdb' index mode capability required" - - do: catch: bad_request indices.create: @@ -557,14 +468,6 @@ routing path allowed in logs mode with routing on sort fields: --- start time not allowed in logs mode: - - requires: - test_runner_features: [ capabilities ] - capabilities: - - method: PUT - path: /{index} - capabilities: [ logsdb_index_mode ] - reason: "Support for 'logsdb' index mode capability required" - - do: catch: bad_request indices.create: @@ -598,14 +501,6 @@ start time not allowed in logs mode: --- end time not allowed in logs mode: - - requires: - test_runner_features: [ capabilities ] - capabilities: - - method: PUT - path: /{index} - capabilities: [ logsdb_index_mode ] - reason: "Support for 'logsdb' index mode capability required" - - do: catch: bad_request indices.create: @@ -685,10 +580,10 @@ ignore dynamic beyond limit logsdb override value: - match: { test-ignore-dynamic-override.settings.index.mapping.total_fields.ignore_dynamic_beyond_limit: "false" } --- -logsdb with default ignore dynamic beyond limit and default sorting: +default ignore dynamic beyond limit and default sorting: - requires: - cluster_features: ["mapper.logsdb_default_ignore_dynamic_beyond_limit"] - reason: requires default value for ignore_dynamic_beyond_limit + cluster_features: [ "index.logsdb_no_host_name_field" ] + reason: "No host.name field injection" - do: indices.create: @@ -698,19 +593,8 @@ logsdb with default ignore dynamic beyond limit and default sorting: index: mode: logsdb mapping: - # NOTE: When the index mode is set to `logsdb`, the `host.name` field is automatically injected if - # sort settings are not overridden. - # With `subobjects` set to `true` (default), this creates a `host` object field and a nested `name` - # keyword field (`host.name`). - # - # As a result, there are always at least 4 statically mapped fields (`@timestamp`, `host`, `host.name` - # and `name`). We cannot use a field limit lower than 4 because these fields are always present. - # - # Indeed, if `index.mapping.total_fields.ignore_dynamic_beyond_limit` is `true`, any dynamically - # mapped fields beyond the limit `index.mapping.total_fields.limit` are ignored, but the statically - # mapped fields are always counted. total_fields: - limit: 4 + limit: 2 mappings: properties: "@timestamp": @@ -730,9 +614,9 @@ logsdb with default ignore dynamic beyond limit and default sorting: refresh: true body: - '{ "index": { } }' - - '{ "@timestamp": "2024-08-13T12:30:00Z", "name": "foo", "host.name": "92f4a67c", "value": 10, "message": "the quick brown fox", "region": "us-west", "pid": 153462 }' + - '{ "@timestamp": "2024-08-13T12:30:00Z", "name": "foo", "value": 10, "message": "the quick brown fox", "region": "us-west", "pid": 153462 }' - '{ "index": { } }' - - '{ "@timestamp": "2024-08-13T12:01:00Z", "name": "bar", "host.name": "24eea278", "value": 20, "message": "jumps over the lazy dog", "region": "us-central", "pid": 674972 }' + - '{ "@timestamp": "2024-08-13T12:01:00Z", "name": "bar", "value": 20, "message": "jumps over the lazy dog", "region": "us-central", "pid": 674972 }' - match: { errors: false } - do: @@ -754,132 +638,82 @@ logsdb with default ignore dynamic beyond limit and default sorting: - match: { hits.hits.1._ignored: [ "message", "pid", "region", "value" ] } --- -logsdb with default ignore dynamic beyond limit and non-default sorting: +default ignore dynamic beyond limit and default sorting with hostname: - requires: - cluster_features: ["mapper.logsdb_default_ignore_dynamic_beyond_limit"] - reason: requires default value for ignore_dynamic_beyond_limit + cluster_features: [ "index.logsdb_no_host_name_field" ] + reason: "No host.name field injection" - do: indices.create: - index: test-logsdb-non-default-sort + index: test-logsdb-default-sort body: settings: index: - sort.field: [ "name" ] - sort.order: [ "desc" ] mode: logsdb mapping: - # NOTE: Here sort settings are overridden and we do not have any additional statically mapped field other - # than `name` and `timestamp`. As a result, there are only 2 statically mapped fields. total_fields: - limit: 2 + limit: 3 mappings: properties: "@timestamp": type: date - name: + host.name: type: keyword - do: indices.get_settings: - index: test-logsdb-non-default-sort + index: test-logsdb-default-sort - - match: { test-logsdb-non-default-sort.settings.index.mode: "logsdb" } + - match: { test-logsdb-default-sort.settings.index.mode: "logsdb" } - do: bulk: - index: test-logsdb-non-default-sort + index: test-logsdb-default-sort refresh: true body: - '{ "index": { } }' - - '{ "@timestamp": "2024-08-13T12:30:00Z", "name": "foo", "host.name": "92f4a67c", "value": 10, "message": "the quick brown fox", "region": "us-west", "pid": 153462 }' + - '{ "@timestamp": "2024-08-13T12:30:00Z", "host.name": "foo", "value": 10, "message": "the quick brown fox", "region": "us-west", "pid": 153462 }' - '{ "index": { } }' - - '{ "@timestamp": "2024-08-13T12:01:00Z", "name": "bar", "host.name": "24eea278", "value": 20, "message": "jumps over the lazy dog", "region": "us-central", "pid": 674972 }' + - '{ "@timestamp": "2024-08-13T12:01:00Z", "host.name": "bar", "value": 20, "message": "jumps over the lazy dog", "region": "us-central", "pid": 674972 }' - match: { errors: false } - do: search: - index: test-logsdb-non-default-sort + index: test-logsdb-default-sort body: query: match_all: {} sort: "@timestamp" - match: { hits.total.value: 2 } - - match: { hits.hits.0._source.name: "bar" } + - match: { hits.hits.0._source.host.name: "bar" } - match: { hits.hits.0._source.value: 20 } - match: { hits.hits.0._source.message: "jumps over the lazy dog" } - - match: { hits.hits.0._ignored: [ "host", "message", "pid", "region", "value" ] } - - match: { hits.hits.1._source.name: "foo" } + - match: { hits.hits.0._ignored: [ "message", "pid", "region", "value" ] } + - match: { hits.hits.1._source.host.name: "foo" } - match: { hits.hits.1._source.value: 10 } - match: { hits.hits.1._source.message: "the quick brown fox" } - - match: { hits.hits.1._ignored: [ "host", "message", "pid", "region", "value" ] } - ---- -logsdb with default ignore dynamic beyond limit and too low limit: - - requires: - cluster_features: ["mapper.logsdb_default_ignore_dynamic_beyond_limit"] - reason: requires default value for ignore_dynamic_beyond_limit - - - do: - catch: bad_request - indices.create: - index: test-logsdb-low-limit - body: - settings: - index: - mode: logsdb - mapping: - # NOTE: When the index mode is set to `logsdb`, the `host.name` field is automatically injected if - # sort settings are not overridden. - # With `subobjects` set to `true` (default), this creates a `host` object field and a nested `name` - # keyword field (`host.name`). - # - # As a result, there are always at least 4 statically mapped fields (`@timestamp`, `host`, `host.name` - # and `name`). We cannot use a field limit lower than 4 because these fields are always present. - # - # Indeed, if `index.mapping.total_fields.ignore_dynamic_beyond_limit` is `true`, any dynamically - # mapped fields beyond the limit `index.mapping.total_fields.limit` are ignored, but the statically - # mapped fields are always counted. - total_fields: - limit: 3 - mappings: - properties: - "@timestamp": - type: date - name: - type: keyword - - match: { error.type: "illegal_argument_exception" } - - match: { error.reason: "Limit of total fields [3] has been exceeded" } + - match: { hits.hits.1._ignored: [ "message", "pid", "region", "value" ] } --- -logsdb with default ignore dynamic beyond limit and subobjects false: +default ignore dynamic beyond limit and non-default sorting: - requires: - cluster_features: ["mapper.logsdb_default_ignore_dynamic_beyond_limit"] - reason: requires default value for ignore_dynamic_beyond_limit + cluster_features: [ "index.logsdb_no_host_name_field" ] + reason: "No host.name field injection" - do: indices.create: - index: test-logsdb-subobjects-false + index: test-logsdb-non-default-sort body: settings: index: + sort.field: [ "name" ] + sort.order: [ "desc" ] mode: logsdb mapping: - # NOTE: When the index mode is set to `logsdb`, the `host.name` field is automatically injected if - # sort settings are not overridden. - # With `subobjects` set to `false` anyway, a single `host.name` keyword field is automatically mapped. - # - # As a result, there are just 3 statically mapped fields (`@timestamp`, `host.name` and `name`). - # We cannot use a field limit lower than 3 because these fields are always present. - # - # Indeed, if `index.mapping.total_fields.ignore_dynamic_beyond_limit` is `true`, any dynamically - # mapped fields beyond the limit `index.mapping.total_fields.limit` are ignored, but the statically - # mapped fields are always counted. total_fields: - limit: 3 + limit: 2 mappings: - subobjects: false properties: "@timestamp": type: date @@ -888,13 +722,13 @@ logsdb with default ignore dynamic beyond limit and subobjects false: - do: indices.get_settings: - index: test-logsdb-subobjects-false + index: test-logsdb-non-default-sort - - match: { test-logsdb-subobjects-false.settings.index.mode: "logsdb" } + - match: { test-logsdb-non-default-sort.settings.index.mode: "logsdb" } - do: bulk: - index: test-logsdb-subobjects-false + index: test-logsdb-non-default-sort refresh: true body: - '{ "index": { } }' @@ -905,7 +739,7 @@ logsdb with default ignore dynamic beyond limit and subobjects false: - do: search: - index: test-logsdb-subobjects-false + index: test-logsdb-non-default-sort body: query: match_all: {} @@ -915,8 +749,8 @@ logsdb with default ignore dynamic beyond limit and subobjects false: - match: { hits.hits.0._source.name: "bar" } - match: { hits.hits.0._source.value: 20 } - match: { hits.hits.0._source.message: "jumps over the lazy dog" } - - match: { hits.hits.0._ignored: [ "message", "pid", "region", "value" ] } + - match: { hits.hits.0._ignored: [ "host", "message", "pid", "region", "value" ] } - match: { hits.hits.1._source.name: "foo" } - match: { hits.hits.1._source.value: 10 } - match: { hits.hits.1._source.message: "the quick brown fox" } - - match: { hits.hits.1._ignored: [ "message", "pid", "region", "value" ] } + - match: { hits.hits.1._ignored: [ "host", "message", "pid", "region", "value" ] } diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 4fac2c967c3f6..4112290fa4e04 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -434,6 +434,7 @@ org.elasticsearch.action.admin.cluster.allocation.AllocationStatsFeatures, org.elasticsearch.rest.action.admin.cluster.ClusterRerouteFeatures, org.elasticsearch.index.mapper.MapperFeatures, + org.elasticsearch.index.IndexFeatures, org.elasticsearch.ingest.IngestGeoIpFeatures, org.elasticsearch.search.SearchFeatures, org.elasticsearch.script.ScriptFeatures, diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 48099edeffdaf..afee2491672ef 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -184,6 +184,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexSettings.LIFECYCLE_PARSE_ORIGINATION_DATE_SETTING, IndexSettings.TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING, IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS, + IndexSettings.LOGSDB_SORT_ON_HOST_NAME, + IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD, IndexSettings.PREFER_ILM_SETTING, DataStreamFailureStoreDefinition.FAILURE_STORE_DEFINITION_VERSION_SETTING, FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING, diff --git a/server/src/main/java/org/elasticsearch/index/IndexFeatures.java b/server/src/main/java/org/elasticsearch/index/IndexFeatures.java new file mode 100644 index 0000000000000..f940e87e51391 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/IndexFeatures.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index; + +import org.elasticsearch.features.FeatureSpecification; +import org.elasticsearch.features.NodeFeature; + +import java.util.Set; + +public class IndexFeatures implements FeatureSpecification { + + @Override + public Set getFeatures() { + return Set.of(); + } + + public static final NodeFeature LOGSDB_NO_HOST_NAME_FIELD = new NodeFeature("index.logsdb_no_host_name_field"); + + @Override + public Set getTestFeatures() { + return Set.of(LOGSDB_NO_HOST_NAME_FIELD); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index a138407991b68..f0814577bd203 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -178,7 +178,7 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma @Override public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) { - return DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING; + return DEFAULT_MAPPING_TIMESTAMP; } @Override @@ -260,9 +260,9 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma @Override public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) { - return indexSettings != null && indexSettings.getIndexSortConfig().hasPrimarySortOnField(HOST_NAME) - ? DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME - : DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING; + return indexSettings != null && indexSettings.logsdbAddHostNameField() + ? DEFAULT_MAPPING_TIMESTAMP_HOSTNAME + : DEFAULT_MAPPING_TIMESTAMP; } @Override @@ -392,7 +392,7 @@ public SourceFieldMapper.Mode defaultSourceMode() { } }; - private static final String HOST_NAME = "host.name"; + static final String HOST_NAME = "host.name"; private static void validateRoutingPathSettings(Map, Object> settings) { settingRequiresTimeSeries(settings, IndexMetadata.INDEX_ROUTING_PATH); @@ -432,14 +432,14 @@ private static CompressedXContent createDefaultMapping(boolean includeHostName) }); } - private static final CompressedXContent DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING; + private static final CompressedXContent DEFAULT_MAPPING_TIMESTAMP; - private static final CompressedXContent DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME; + private static final CompressedXContent DEFAULT_MAPPING_TIMESTAMP_HOSTNAME; static { try { - DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING = createDefaultMapping(false); - DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME = createDefaultMapping(true); + DEFAULT_MAPPING_TIMESTAMP = createDefaultMapping(false); + DEFAULT_MAPPING_TIMESTAMP_HOSTNAME = createDefaultMapping(true); } catch (IOException e) { throw new AssertionError(e); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index e5e8508d06ee9..68f334b10ea52 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -711,6 +711,22 @@ public boolean isES87TSDBCodecEnabled() { Property.Final ); + public static final Setting LOGSDB_ADD_HOST_NAME_FIELD = Setting.boolSetting( + "index.logsdb.add_host_name_field", + false, + Property.IndexScope, + Property.PrivateIndex, + Property.Final + ); + + public static final Setting LOGSDB_SORT_ON_HOST_NAME = Setting.boolSetting( + "index.logsdb.sort_on_host_name", + false, + Property.IndexScope, + Property.PrivateIndex, + Property.Final + ); + /** * The {@link IndexMode "mode"} of the index. */ @@ -833,6 +849,8 @@ private static String getIgnoreAboveDefaultValue(final Settings settings) { private volatile long softDeleteRetentionOperations; private final boolean es87TSDBCodecEnabled; private final boolean logsdbRouteOnSortFields; + private final boolean logsdbSortOnHostName; + private final boolean logsdbAddHostNameField; private volatile long retentionLeaseMillis; @@ -950,6 +968,13 @@ public boolean logsdbRouteOnSortFields() { return logsdbRouteOnSortFields; } + /** + * Returns true if the index is in logsdb mode and needs a [host.name] keyword field. The default is false + */ + public boolean logsdbAddHostNameField() { + return logsdbAddHostNameField; + } + /** * Creates a new {@link IndexSettings} instance. The given node settings will be merged with the settings in the metadata * while index level settings will overwrite node settings. @@ -1043,6 +1068,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti sourceKeepMode = scopedSettings.get(Mapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING); es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING); logsdbRouteOnSortFields = scopedSettings.get(LOGSDB_ROUTE_ON_SORT_FIELDS); + logsdbSortOnHostName = scopedSettings.get(LOGSDB_SORT_ON_HOST_NAME); + logsdbAddHostNameField = scopedSettings.get(LOGSDB_ADD_HOST_NAME_FIELD); skipIgnoredSourceWrite = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); skipIgnoredSourceRead = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); indexMappingSourceMode = scopedSettings.get(INDEX_MAPPER_SOURCE_MODE_SETTING); diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java index 6c044ab999899..94d1cc5182457 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java @@ -13,7 +13,6 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.search.SortedSetSortField; -import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Setting; @@ -103,12 +102,23 @@ public final class IndexSortConfig { Setting.Property.ServerlessPublic ); - public static final FieldSortSpec[] TIME_SERIES_SORT; - + public static final FieldSortSpec[] TIME_SERIES_SORT, TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_BWC_SORT; static { FieldSortSpec timeStampSpec = new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH); timeStampSpec.order = SortOrder.DESC; TIME_SERIES_SORT = new FieldSortSpec[] { new FieldSortSpec(TimeSeriesIdFieldMapper.NAME), timeStampSpec }; + TIMESTAMP_SORT = new FieldSortSpec[] { timeStampSpec }; + + FieldSortSpec hostnameSpec = new FieldSortSpec(IndexMode.HOST_NAME); + hostnameSpec.order = SortOrder.ASC; + hostnameSpec.missingValue = "_last"; + hostnameSpec.mode = MultiValueMode.MIN; + HOSTNAME_TIMESTAMP_SORT = new FieldSortSpec[] { hostnameSpec, timeStampSpec }; + + // Older indexes use ascending ordering for host name and timestamp. + HOSTNAME_TIMESTAMP_BWC_SORT = new FieldSortSpec[] { + new FieldSortSpec(IndexMode.HOST_NAME), + new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH) }; } private static String validateMissingValue(String missing) { @@ -148,26 +158,40 @@ public IndexSortConfig(IndexSettings indexSettings) { this.indexName = indexSettings.getIndex().getName(); this.indexMode = indexSettings.getMode(); - if (this.indexMode == IndexMode.TIME_SERIES) { - this.sortSpecs = TIME_SERIES_SORT; + if (indexMode == IndexMode.TIME_SERIES) { + sortSpecs = TIME_SERIES_SORT; return; } List fields = INDEX_SORT_FIELD_SETTING.get(settings); - if (this.indexMode == IndexMode.LOGSDB && fields.isEmpty()) { - fields = List.of("host.name", DataStream.TIMESTAMP_FIELD_NAME); + if (indexMode == IndexMode.LOGSDB && INDEX_SORT_FIELD_SETTING.exists(settings) == false) { + if (INDEX_SORT_ORDER_SETTING.exists(settings)) { + var order = INDEX_SORT_ORDER_SETTING.get(settings); + throw new IllegalArgumentException("index.sort.fields:" + fields + " index.sort.order:" + order + ", size mismatch"); + } + if (INDEX_SORT_MODE_SETTING.exists(settings)) { + var mode = INDEX_SORT_MODE_SETTING.get(settings); + throw new IllegalArgumentException("index.sort.fields:" + fields + " index.sort.mode:" + mode + ", size mismatch"); + } + if (INDEX_SORT_MISSING_SETTING.exists(settings)) { + var missing = INDEX_SORT_MISSING_SETTING.get(settings); + throw new IllegalArgumentException("index.sort.fields:" + fields + " index.sort.missing:" + missing + ", size mismatch"); + } + var version = indexSettings.getIndexVersionCreated(); + if (version.onOrAfter(IndexVersions.LOGSB_OPTIONAL_SORTING_ON_HOST_NAME) + || version.between(IndexVersions.LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0)) { + sortSpecs = (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(settings)) ? HOSTNAME_TIMESTAMP_SORT : TIMESTAMP_SORT; + } else { + sortSpecs = HOSTNAME_TIMESTAMP_BWC_SORT; + } + return; } - this.sortSpecs = fields.stream().map(FieldSortSpec::new).toArray(FieldSortSpec[]::new); + sortSpecs = fields.stream().map(FieldSortSpec::new).toArray(FieldSortSpec[]::new); if (INDEX_SORT_ORDER_SETTING.exists(settings)) { List orders = INDEX_SORT_ORDER_SETTING.get(settings); - if (this.indexMode == IndexMode.LOGSDB && orders.isEmpty()) { - orders = List.of(SortOrder.DESC, SortOrder.DESC); - } if (orders.size() != sortSpecs.length) { - throw new IllegalArgumentException( - "index.sort.field:" + fields + " index.sort.order:" + orders.toString() + ", size mismatch" - ); + throw new IllegalArgumentException("index.sort.field:" + fields + " index.sort.order:" + orders + ", size mismatch"); } for (int i = 0; i < sortSpecs.length; i++) { sortSpecs[i].order = orders.get(i); @@ -176,9 +200,6 @@ public IndexSortConfig(IndexSettings indexSettings) { if (INDEX_SORT_MODE_SETTING.exists(settings)) { List modes = INDEX_SORT_MODE_SETTING.get(settings); - if (this.indexMode == IndexMode.LOGSDB && modes.isEmpty()) { - modes = List.of(MultiValueMode.MIN, MultiValueMode.MIN); - } if (modes.size() != sortSpecs.length) { throw new IllegalArgumentException("index.sort.field:" + fields + " index.sort.mode:" + modes + ", size mismatch"); } @@ -189,9 +210,6 @@ public IndexSortConfig(IndexSettings indexSettings) { if (INDEX_SORT_MISSING_SETTING.exists(settings)) { List missingValues = INDEX_SORT_MISSING_SETTING.get(settings); - if (this.indexMode == IndexMode.LOGSDB && missingValues.isEmpty()) { - missingValues = List.of("_first", "_first"); - } if (missingValues.size() != sortSpecs.length) { throw new IllegalArgumentException( "index.sort.field:" + fields + " index.sort.missing:" + missingValues + ", size mismatch" diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 8d6404e0530e5..69ebcd4ba3fe6 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -134,13 +134,14 @@ private static Version parseUnchecked(String version) { public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BACKPORT = def(8_522_00_0, Version.LUCENE_9_12_0); public static final IndexVersion UPGRADE_TO_LUCENE_9_12_1 = def(8_523_00_0, parseUnchecked("9.12.1")); public static final IndexVersion INFERENCE_METADATA_FIELDS_BACKPORT = def(8_524_00_0, parseUnchecked("9.12.1")); + public static final IndexVersion LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT = def(8_525_00_0, parseUnchecked("9.12.1")); public static final IndexVersion UPGRADE_TO_LUCENE_10_0_0 = def(9_000_00_0, Version.LUCENE_10_0_0); public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT = def(9_001_00_0, Version.LUCENE_10_0_0); public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID = def(9_002_00_0, Version.LUCENE_10_0_0); public static final IndexVersion DEPRECATE_SOURCE_MODE_MAPPER = def(9_003_00_0, Version.LUCENE_10_0_0); public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY = def(9_004_00_0, Version.LUCENE_10_0_0); public static final IndexVersion INFERENCE_METADATA_FIELDS = def(9_005_00_0, Version.LUCENE_10_0_0); - + public static final IndexVersion LOGSB_OPTIONAL_SORTING_ON_HOST_NAME = def(9_006_00_0, Version.LUCENE_10_0_0); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java index 907f7265e98af..7ec175799ddec 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java @@ -81,7 +81,7 @@ public CompressedXContent toCompressedXContent() { /** * Returns the root object for the current mapping */ - RootObjectMapper getRoot() { + public RootObjectMapper getRoot() { return root; } diff --git a/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification index 12965152f260c..8fa188efbd4a3 100644 --- a/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification +++ b/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification @@ -16,6 +16,7 @@ org.elasticsearch.rest.RestFeatures org.elasticsearch.repositories.RepositoriesFeatures org.elasticsearch.action.admin.cluster.allocation.AllocationStatsFeatures org.elasticsearch.rest.action.admin.cluster.ClusterRerouteFeatures +org.elasticsearch.index.IndexFeatures org.elasticsearch.index.mapper.MapperFeatures org.elasticsearch.ingest.IngestGeoIpFeatures org.elasticsearch.search.SearchFeatures diff --git a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java index 23fc788a89bde..6f3077defa4a3 100644 --- a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.index.IndexVersionUtils; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -25,11 +26,89 @@ public void testLogsIndexModeSetting() { public void testDefaultHostNameSortField() { final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", buildSettings()); assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB)); - final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); + boolean sortOnHostName = randomBoolean(); + final IndexSettings settings = new IndexSettings( + metadata, + Settings.builder().put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), sortOnHostName).build() + ); + assertThat(settings.getIndexSortConfig().hasPrimarySortOnField("host.name"), equalTo(sortOnHostName)); + } + + public void testDefaultHostNameSortFieldAndMapping() { + final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", buildSettings()); + assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB)); + final IndexSettings settings = new IndexSettings( + metadata, + Settings.builder() + .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true) + .put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true) + .build() + ); assertThat(settings.getIndexSortConfig().hasPrimarySortOnField("host.name"), equalTo(true)); assertThat(IndexMode.LOGSDB.getDefaultMapping(settings).string(), containsString("host.name")); } + public void testDefaultHostNameSortFieldBwc() { + final IndexMetadata metadata = IndexMetadata.builder("test") + .settings( + indexSettings(IndexVersionUtils.getPreviousVersion(IndexVersions.LOGSB_OPTIONAL_SORTING_ON_HOST_NAME), 1, 1).put( + buildSettings() + ) + ) + .build(); + assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB)); + final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); + assertThat(settings.getIndexSortConfig().hasPrimarySortOnField("host.name"), equalTo(true)); + } + + public void testDefaultHostNameSortWithOrder() { + final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", buildSettings()); + assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB)); + var exception = expectThrows( + IllegalArgumentException.class, + () -> new IndexSettings( + metadata, + Settings.builder() + .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), randomBoolean()) + .put(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), "desc") + .build() + ) + ); + assertEquals("index.sort.fields:[] index.sort.order:[desc], size mismatch", exception.getMessage()); + } + + public void testDefaultHostNameSortWithMode() { + final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", buildSettings()); + assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB)); + var exception = expectThrows( + IllegalArgumentException.class, + () -> new IndexSettings( + metadata, + Settings.builder() + .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), randomBoolean()) + .put(IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey(), "MAX") + .build() + ) + ); + assertEquals("index.sort.fields:[] index.sort.mode:[MAX], size mismatch", exception.getMessage()); + } + + public void testDefaultHostNameSortWithMissing() { + final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", buildSettings()); + assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB)); + var exception = expectThrows( + IllegalArgumentException.class, + () -> new IndexSettings( + metadata, + Settings.builder() + .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), randomBoolean()) + .put(IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey(), "_first") + .build() + ) + ); + assertEquals("index.sort.fields:[] index.sort.missing:[_first], size mismatch", exception.getMessage()); + } + public void testCustomSortField() { final Settings sortSettings = Settings.builder() .put(buildSettings()) diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java index a8085f3d50a82..266847209f495 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java @@ -67,12 +67,11 @@ public Collection createComponents(PluginServices services) { @Override public Collection getAdditionalIndexSettingProviders(IndexSettingProvider.Parameters parameters) { - if (DiscoveryNode.isStateless(settings) == false) { - logsdbIndexModeSettingsProvider.init( - parameters.mapperServiceFactory(), - () -> parameters.clusterService().state().nodes().getMinSupportedIndexVersion() - ); - } + logsdbIndexModeSettingsProvider.init( + parameters.mapperServiceFactory(), + () -> parameters.clusterService().state().nodes().getMinSupportedIndexVersion(), + DiscoveryNode.isStateless(settings) == false + ); return List.of(logsdbIndexModeSettingsProvider); } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index f95b64f8c0ec9..3316ac3603e9e 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -24,7 +24,11 @@ import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import java.io.IOException; @@ -44,6 +48,7 @@ final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider { private final SyntheticSourceLicenseService syntheticSourceLicenseService; private final SetOnce> mapperServiceFactory = new SetOnce<>(); private final SetOnce> createdIndexVersion = new SetOnce<>(); + private final SetOnce supportFallbackToStoredSource = new SetOnce<>(); private volatile boolean isLogsdbEnabled; @@ -56,13 +61,14 @@ void updateClusterIndexModeLogsdbEnabled(boolean isLogsdbEnabled) { this.isLogsdbEnabled = isLogsdbEnabled; } - void init(CheckedFunction factory, Supplier indexVersion) { - mapperServiceFactory.set(factory); - createdIndexVersion.set(indexVersion); - } - - private boolean supportFallbackToStoredSource() { - return mapperServiceFactory.get() != null; + void init( + CheckedFunction factory, + Supplier indexVersion, + boolean supportFallbackToStoredSource + ) { + this.mapperServiceFactory.set(factory); + this.createdIndexVersion.set(indexVersion); + this.supportFallbackToStoredSource.set(supportFallbackToStoredSource); } @Override @@ -90,14 +96,14 @@ public Settings getAdditionalIndexSettings( && resolveIndexMode(settings.get(IndexSettings.MODE.getKey())) == null && matchesLogsPattern(dataStreamName)) { settingsBuilder = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()); - if (supportFallbackToStoredSource()) { - settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()).put(settings).build(); - } + settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()).put(settings).build(); isLogsDB = true; } + MappingHints mappingHints = getMappingHints(indexName, templateIndexMode, settings, combinedTemplateMappings); + // Inject stored source mode if synthetic source if not available per licence. - if (supportFallbackToStoredSource()) { + if (mappingHints.hasSyntheticSourceUsage && supportFallbackToStoredSource.get()) { // This index name is used when validating component and index templates, we should skip this check in that case. // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method) boolean isTemplateValidation = "validate-index-name".equals(indexName); @@ -106,11 +112,7 @@ && matchesLogsPattern(dataStreamName)) { indexName, dataStreamName ); - if (newIndexHasSyntheticSourceUsage(indexName, templateIndexMode, settings, combinedTemplateMappings) - && syntheticSourceLicenseService.fallbackToStoredSource( - isTemplateValidation, - legacyLicensedUsageOfSyntheticSourceAllowed - )) { + if (syntheticSourceLicenseService.fallbackToStoredSource(isTemplateValidation, legacyLicensedUsageOfSyntheticSourceAllowed)) { LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName); if (settingsBuilder == null) { settingsBuilder = Settings.builder(); @@ -120,6 +122,18 @@ && matchesLogsPattern(dataStreamName)) { } if (isLogsDB) { + // Inject sorting on [host.name], in addition to [@timestamp]. + if (mappingHints.sortOnHostName) { + if (settingsBuilder == null) { + settingsBuilder = Settings.builder(); + } + if (mappingHints.addHostNameField) { + // Inject keyword field [host.name] too. + settingsBuilder.put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true); + } + settingsBuilder.put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true); + } + // Inject routing path matching sort fields. if (settings.getAsBoolean(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), false)) { List sortFields = new ArrayList<>(settings.getAsList(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())); @@ -155,7 +169,7 @@ && matchesLogsPattern(dataStreamName)) { if (settingsBuilder == null) { settingsBuilder = Settings.builder(); } - settingsBuilder.putList(INDEX_ROUTING_PATH.getKey(), sortFields).build(); + settingsBuilder.putList(INDEX_ROUTING_PATH.getKey(), sortFields); } } } @@ -164,6 +178,10 @@ && matchesLogsPattern(dataStreamName)) { } + record MappingHints(boolean hasSyntheticSourceUsage, boolean sortOnHostName, boolean addHostNameField) { + static MappingHints EMPTY = new MappingHints(false, false, false); + } + private static boolean matchesLogsPattern(final String name) { return Regex.simpleMatch(LOGS_PATTERN, name); } @@ -172,7 +190,7 @@ private static IndexMode resolveIndexMode(final String mode) { return mode != null ? Enum.valueOf(IndexMode.class, mode.toUpperCase(Locale.ROOT)) : null; } - boolean newIndexHasSyntheticSourceUsage( + MappingHints getMappingHints( String indexName, IndexMode templateIndexMode, Settings indexTemplateAndCreateRequestSettings, @@ -181,12 +199,13 @@ boolean newIndexHasSyntheticSourceUsage( if ("validate-index-name".equals(indexName)) { // This index name is used when validating component and index templates, we should skip this check in that case. // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method) - return false; + return MappingHints.EMPTY; } try { var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, templateIndexMode, indexTemplateAndCreateRequestSettings); var indexMode = tmpIndexMetadata.getIndexMode(); + boolean hasSyntheticSourceUsage = false; if (SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.exists(tmpIndexMetadata.getSettings()) || indexMode == IndexMode.LOGSDB || indexMode == IndexMode.TIME_SERIES) { @@ -196,10 +215,18 @@ boolean newIndexHasSyntheticSourceUsage( // to disabled, because that isn't allowed with logsdb/tsdb. In other words setting index.mapping.source.mode setting to // stored when _source.mode mapping attribute is stored is fine as it has no effect, but avoids creating MapperService. var sourceMode = SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(tmpIndexMetadata.getSettings()); - return sourceMode == SourceFieldMapper.Mode.SYNTHETIC; + hasSyntheticSourceUsage = sourceMode == SourceFieldMapper.Mode.SYNTHETIC; + if (IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(indexTemplateAndCreateRequestSettings).isEmpty() == false) { + // Custom sort config, no point for further checks on [host.name] field. + return new MappingHints(hasSyntheticSourceUsage, false, false); + } + if (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(indexTemplateAndCreateRequestSettings) + && IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(indexTemplateAndCreateRequestSettings)) { + // Settings for adding and sorting on [host.name] are already set, propagate them. + return new MappingHints(hasSyntheticSourceUsage, true, true); + } } - // TODO: remove this when _source.mode attribute has been removed: try (var mapperService = mapperServiceFactory.get().apply(tmpIndexMetadata)) { // combinedTemplateMappings can be null when creating system indices // combinedTemplateMappings can be empty when creating a normal index that doesn't match any template and without mapping. @@ -207,14 +234,25 @@ boolean newIndexHasSyntheticSourceUsage( combinedTemplateMappings = List.of(new CompressedXContent("{}")); } mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE); - return mapperService.documentMapper().sourceMapper().isSynthetic(); + Mapper hostName = mapperService.mappingLookup().getMapper("host.name"); + hasSyntheticSourceUsage = hasSyntheticSourceUsage || mapperService.documentMapper().sourceMapper().isSynthetic(); + boolean addHostNameField = IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(indexTemplateAndCreateRequestSettings) + || (hostName == null + && mapperService.mappingLookup().objectMappers().get("host.name") == null + && (mapperService.mappingLookup().getMapper("host") == null + || mapperService.mappingLookup().getMapping().getRoot().subobjects() == ObjectMapper.Subobjects.DISABLED)); + boolean sortOnHostName = IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(indexTemplateAndCreateRequestSettings) + || addHostNameField + || ((hostName instanceof NumberFieldMapper nfm && nfm.fieldType().hasDocValues()) + || (hostName instanceof KeywordFieldMapper kfm && kfm.fieldType().hasDocValues())); + return new MappingHints(hasSyntheticSourceUsage, sortOnHostName, addHostNameField); } } catch (AssertionError | Exception e) { // In case invalid mappings or setting are provided, then mapper service creation can fail. // In that case it is ok to return false here. The index creation will fail anyway later, so no need to fallback to stored // source. LOGGER.info(() -> Strings.format("unable to create mapper service for index [%s]", indexName), e); - return false; + return MappingHints.EMPTY; } } diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index 5d3cb7b2a9967..71d52842ecbae 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -82,7 +82,7 @@ public void setup() throws Exception { syntheticSourceLicenseService.setLicenseService(mockLicenseService); } - LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabled) { + private LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabled) { newMapperServiceCounter.set(0); var provider = new LogsdbIndexModeSettingsProvider( syntheticSourceLicenseService, @@ -91,11 +91,37 @@ LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabl provider.init(im -> { newMapperServiceCounter.incrementAndGet(); return MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()); - }, IndexVersion::current); + }, IndexVersion::current, true); return provider; } - public void testLogsDbDisabled() throws IOException { + private Settings generateLogsdbSettings(Settings settings) throws IOException { + return generateLogsdbSettings(settings, null); + } + + private Settings generateLogsdbSettings(Settings settings, String mapping) throws IOException { + Metadata metadata = Metadata.EMPTY_METADATA; + var provider = new LogsdbIndexModeSettingsProvider( + syntheticSourceLicenseService, + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + provider.init(im -> { + newMapperServiceCounter.incrementAndGet(); + return MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()); + }, IndexVersion::current, true); + var result = provider.getAdditionalIndexSettings( + DataStream.getDefaultBackingIndexName(DATA_STREAM_NAME, 0), + DATA_STREAM_NAME, + IndexMode.LOGSDB, + metadata, + Instant.now(), + settings, + mapping == null ? List.of() : List.of(new CompressedXContent(mapping)) + ); + return builder().put(result).build(); + } + + public void testDisabled() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( syntheticSourceLicenseService, Settings.builder().put("cluster.logsdb.enabled", false).build() @@ -408,7 +434,8 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException { } } """; - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + .hasSyntheticSourceUsage(); assertTrue(result); assertThat(newMapperServiceCounter.get(), equalTo(1)); assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); @@ -444,7 +471,8 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException { } """; } - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + .hasSyntheticSourceUsage(); assertFalse(result); assertThat(newMapperServiceCounter.get(), equalTo(2)); if (withSourceMode) { @@ -471,7 +499,8 @@ public void testValidateIndexName() throws IOException { """; Settings settings = Settings.EMPTY; LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + .hasSyntheticSourceUsage(); assertFalse(result); } @@ -492,30 +521,27 @@ public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); { Settings settings = Settings.builder().put("index.mode", "logsdb").build(); - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + .hasSyntheticSourceUsage(); assertTrue(result); - assertThat(newMapperServiceCounter.get(), equalTo(0)); + assertThat(newMapperServiceCounter.get(), equalTo(1)); } { Settings settings = Settings.builder().put("index.mode", "logsdb").build(); - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of()); + boolean result = provider.getMappingHints(indexName, null, settings, List.of()).hasSyntheticSourceUsage(); assertTrue(result); - assertThat(newMapperServiceCounter.get(), equalTo(0)); + assertThat(newMapperServiceCounter.get(), equalTo(2)); } { - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, Settings.EMPTY, List.of()); + boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of()).hasSyntheticSourceUsage(); assertFalse(result); - assertThat(newMapperServiceCounter.get(), equalTo(1)); + assertThat(newMapperServiceCounter.get(), equalTo(3)); } { - boolean result = provider.newIndexHasSyntheticSourceUsage( - indexName, - null, - Settings.EMPTY, - List.of(new CompressedXContent(mapping)) - ); + boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(mapping))) + .hasSyntheticSourceUsage(); assertFalse(result); - assertThat(newMapperServiceCounter.get(), equalTo(2)); + assertThat(newMapperServiceCounter.get(), equalTo(4)); } } @@ -537,25 +563,22 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); { Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build(); - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + .hasSyntheticSourceUsage(); assertTrue(result); } { Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build(); - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of()); + boolean result = provider.getMappingHints(indexName, null, settings, List.of()).hasSyntheticSourceUsage(); assertTrue(result); } { - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, Settings.EMPTY, List.of()); + boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of()).hasSyntheticSourceUsage(); assertFalse(result); } { - boolean result = provider.newIndexHasSyntheticSourceUsage( - indexName, - null, - Settings.EMPTY, - List.of(new CompressedXContent(mapping)) - ); + boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(mapping))) + .hasSyntheticSourceUsage(); assertFalse(result); } } @@ -580,7 +603,8 @@ public void testNewIndexHasSyntheticSourceUsage_invalidSettings() throws IOExcep } } """; - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + .hasSyntheticSourceUsage(); assertFalse(result); assertThat(newMapperServiceCounter.get(), equalTo(1)); } @@ -596,7 +620,8 @@ public void testNewIndexHasSyntheticSourceUsage_invalidSettings() throws IOExcep } } """; - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + .hasSyntheticSourceUsage(); assertFalse(result); assertThat(newMapperServiceCounter.get(), equalTo(2)); } @@ -628,7 +653,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws List.of() ); assertThat(result.size(), equalTo(0)); - assertThat(newMapperServiceCounter.get(), equalTo(0)); + assertThat(newMapperServiceCounter.get(), equalTo(1)); syntheticSourceLicenseService.setSyntheticSourceFallback(true); result = provider.getAdditionalIndexSettings( @@ -642,7 +667,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws ); assertThat(result.size(), equalTo(1)); assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); - assertThat(newMapperServiceCounter.get(), equalTo(0)); + assertThat(newMapperServiceCounter.get(), equalTo(2)); result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), @@ -655,7 +680,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws ); assertThat(result.size(), equalTo(1)); assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); - assertThat(newMapperServiceCounter.get(), equalTo(0)); + assertThat(newMapperServiceCounter.get(), equalTo(3)); result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), @@ -666,9 +691,11 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws settings, List.of() ); - assertThat(result.size(), equalTo(1)); + assertThat(result.size(), equalTo(3)); assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); - assertThat(newMapperServiceCounter.get(), equalTo(0)); + assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); + assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertThat(newMapperServiceCounter.get(), equalTo(4)); } public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch() throws IOException { @@ -719,9 +746,11 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch( settings, List.of() ); - assertThat(result.size(), equalTo(2)); + assertThat(result.size(), equalTo(4)); assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); assertEquals(IndexMode.LOGSDB, IndexSettings.MODE.get(result)); + assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); + assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), @@ -735,7 +764,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch( assertThat(result.size(), equalTo(0)); } - public void testLogsdbRoutingPathOnSortFields() throws Exception { + public void testRoutingPathOnSortFields() throws Exception { var settings = Settings.builder() .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message") .put(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), true) @@ -744,7 +773,7 @@ public void testLogsdbRoutingPathOnSortFields() throws Exception { assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), contains("host", "message")); } - public void testLogsdbRoutingPathOnSortFieldsFilterTimestamp() throws Exception { + public void testRoutingPathOnSortFieldsFilterTimestamp() throws Exception { var settings = Settings.builder() .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message,@timestamp") .put(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), true) @@ -753,7 +782,7 @@ public void testLogsdbRoutingPathOnSortFieldsFilterTimestamp() throws Exception assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), contains("host", "message")); } - public void testLogsdbRoutingPathOnSortSingleField() throws Exception { + public void testRoutingPathOnSortSingleField() throws Exception { var settings = Settings.builder() .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host") .put(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), true) @@ -770,7 +799,7 @@ public void testLogsdbRoutingPathOnSortSingleField() throws Exception { ); } - public void testLogsdbExplicitRoutingPathMatchesSortFields() throws Exception { + public void testExplicitRoutingPathMatchesSortFields() throws Exception { var settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB) .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message,@timestamp") @@ -781,7 +810,7 @@ public void testLogsdbExplicitRoutingPathMatchesSortFields() throws Exception { assertTrue(result.isEmpty()); } - public void testLogsdbExplicitRoutingPathDoesNotMatchSortFields() throws Exception { + public void testExplicitRoutingPathDoesNotMatchSortFields() throws Exception { var settings = Settings.builder() .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message,@timestamp") .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "host,message,foo") @@ -800,21 +829,227 @@ public void testLogsdbExplicitRoutingPathDoesNotMatchSortFields() throws Excepti ); } - private Settings generateLogsdbSettings(Settings settings) throws IOException { - Metadata metadata = Metadata.EMPTY_METADATA; - var provider = new LogsdbIndexModeSettingsProvider( - syntheticSourceLicenseService, - Settings.builder().put("cluster.logsdb.enabled", true).build() - ); - var result = provider.getAdditionalIndexSettings( - DataStream.getDefaultBackingIndexName(DATA_STREAM_NAME, 0), - DATA_STREAM_NAME, - IndexMode.LOGSDB, - metadata, - Instant.now(), - settings, - List.of() - ); - return builder().put(result).build(); + public void testSortAndHostNamePropagateValue() throws Exception { + var settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB) + .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true) + .put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true) + .build(); + Settings result = generateLogsdbSettings(settings); + assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); + assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertEquals(0, newMapperServiceCounter.get()); + } + + public void testSortAndHostNameWithCustomSortConfig() throws Exception { + var settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "foo,bar") + .build(); + Settings result = generateLogsdbSettings(settings); + assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); + assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertEquals(0, newMapperServiceCounter.get()); + } + + public void testSortAndHostNameKeyword() throws Exception { + var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); + var mappings = """ + { + "_doc": { + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "keyword" + } + } + } + } + """; + Settings result = generateLogsdbSettings(settings, mappings); + assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); + assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertEquals(1, newMapperServiceCounter.get()); + } + + public void testSortAndHostNameKeywordNoDocvalues() throws Exception { + var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); + var mappings = """ + { + "_doc": { + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "keyword", + "doc_values": false + } + } + } + } + """; + Settings result = generateLogsdbSettings(settings, mappings); + assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); + assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertEquals(1, newMapperServiceCounter.get()); + } + + public void testSortAndHostNameInteger() throws Exception { + var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); + var mappings = """ + { + "_doc": { + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "integer" + } + } + } + } + """; + Settings result = generateLogsdbSettings(settings, mappings); + assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); + assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertEquals(1, newMapperServiceCounter.get()); + } + + public void testSortAndHostNameIntegerNoDocvalues() throws Exception { + var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); + var mappings = """ + { + "_doc": { + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "integer", + "doc_values": false + } + } + } + } + """; + Settings result = generateLogsdbSettings(settings, mappings); + assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); + assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertEquals(1, newMapperServiceCounter.get()); + } + + public void testSortAndHostNameBoolean() throws Exception { + var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); + var mappings = """ + { + "_doc": { + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "boolean" + } + } + } + } + """; + Settings result = generateLogsdbSettings(settings, mappings); + assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); + assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertEquals(1, newMapperServiceCounter.get()); + } + + public void testSortAndHostObject() throws Exception { + var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); + var mappings = """ + { + "_doc": { + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "type": "object" + } + } + } + } + """; + Settings result = generateLogsdbSettings(settings, mappings); + assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); + assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertEquals(1, newMapperServiceCounter.get()); + } + + public void testSortAndHostField() throws Exception { + var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); + var mappings = """ + { + "_doc": { + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "type": "keyword" + } + } + } + } + """; + Settings result = generateLogsdbSettings(settings, mappings); + assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); + assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertEquals(1, newMapperServiceCounter.get()); + } + + public void testSortAndHostFieldSubobjectsFalse() throws Exception { + var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); + var mappings = """ + { + "_doc": { + "subobjects": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "type": "keyword" + } + } + } + } + """; + Settings result = generateLogsdbSettings(settings, mappings); + assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); + assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertEquals(1, newMapperServiceCounter.get()); + } + + public void testSortAndHostNameObject() throws Exception { + var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); + var mappings = """ + { + "_doc": { + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name.sub": { + "type": "keyword" + } + } + } + } + """; + Settings result = generateLogsdbSettings(settings, mappings); + assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); + assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertEquals(1, newMapperServiceCounter.get()); } } diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java index 8a4adf18b3e67..4921344278845 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java @@ -52,7 +52,8 @@ public void setup() throws Exception { provider = new LogsdbIndexModeSettingsProvider(syntheticSourceLicenseService, Settings.EMPTY); provider.init( im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()), - IndexVersion::current + IndexVersion::current, + true ); } @@ -113,7 +114,8 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception provider = new LogsdbIndexModeSettingsProvider(syntheticSourceLicenseService, Settings.EMPTY); provider.init( im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()), - IndexVersion::current + IndexVersion::current, + true ); Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml index 2f320c2cad966..1fd10c2028fd2 100644 --- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml @@ -72,6 +72,44 @@ create logsdb data stream with host.name as keyword and timestamp as date: - is_true: acknowledged +--- +create logsdb data stream with host.name as integer and timestamp as date: + - requires: + test_runner_features: [ "allowed_warnings" ] + - requires: + cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] + reason: support for normalizer on keyword fields + + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + host.name: + type: "integer" + "@timestamp": + type: "date" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + + - is_true: acknowledged + --- create logsdb data stream with host as keyword: - requires: @@ -103,12 +141,10 @@ create logsdb data stream with host as keyword: - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" - do: - catch: bad_request indices.create_data_stream: name: "logsdb" - - match: { error.type: "mapper_parsing_exception" } - - match: { error.reason: "Failed to parse mapping: can't merge a non object mapping [host] with an object mapping" } + - is_true: acknowledged --- create logsdb data stream with host as text and multi fields: @@ -148,12 +184,10 @@ create logsdb data stream with host as text and multi fields: - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" - do: - catch: bad_request indices.create_data_stream: name: "logsdb" - - match: { error.type: "mapper_parsing_exception" } - - match: { error.reason: "Failed to parse mapping: can't merge a non object mapping [host] with an object mapping" } + - is_true: acknowledged --- create logsdb data stream with host as text: @@ -189,12 +223,10 @@ create logsdb data stream with host as text: - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" - do: - catch: bad_request indices.create_data_stream: name: "logsdb" - - match: { error.type: "mapper_parsing_exception" } - - match: { error.reason: "Failed to parse mapping: can't merge a non object mapping [host] with an object mapping" } + - is_true: acknowledged --- create logsdb data stream with host as text and name as double: @@ -233,12 +265,10 @@ create logsdb data stream with host as text and name as double: - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" - do: - catch: bad_request indices.create_data_stream: name: "logsdb" - - match: { error.type: "mapper_parsing_exception" } - - match: { error.reason: "Failed to parse mapping: can't merge a non object mapping [host] with an object mapping" } + - is_true: acknowledged --- create logsdb data stream with timestamp object mapping: @@ -388,7 +418,7 @@ create logsdb data stream with custom sorting and host object: - match: { .$backing_index.mappings.properties.host.properties.ip.type: ip } - match: { .$backing_index.mappings.properties.host.properties.hostname.type: keyword } - match: { .$backing_index.mappings.properties.host.properties.region.type: keyword } - - match: { .$backing_index.mappings.properties.host.properties.name.type: integer } # Overrides LogsDB injected + - match: { .$backing_index.mappings.properties.host.properties.name.type: integer } --- create logsdb data stream with custom sorting and dynamically mapped host.name: @@ -647,8 +677,8 @@ create logsdb data stream with custom sorting and missing host.name field mappin template: settings: index: - sort.field: [ host.name, host.hostname ] - sort.order: [ desc, desc ] + sort.field: [ host.hostname ] + sort.order: [ desc ] mode: logsdb mappings: properties: @@ -677,8 +707,7 @@ create logsdb data stream with custom sorting and missing host.name field mappin - match: { .$backing_index.mappings.properties.@timestamp.type: date } - match: { .$backing_index.mappings.properties.host.properties.hostname.type: keyword } - - match: { .$backing_index.mappings.properties.host.properties.name.type: keyword } - - match: { .$backing_index.mappings.properties.host.properties.name.ignore_above: 1024 } + - match: { .$backing_index.mappings.properties.host.properties.name: null } --- create logsdb data stream with custom sorting and host.name field without doc values: @@ -797,12 +826,21 @@ create logsdb data stream with no sorting and host.name as text: - is_true: acknowledged - do: - catch: bad_request indices.create_data_stream: name: logsdb-non-keyword + - is_true: acknowledged - - match: { error.type: "illegal_argument_exception" } - - match: { error.reason: "docvalues not found for index sort field:[host.name]" } + - do: + indices.get_data_stream: + name: logsdb-non-keyword + + - set: { data_streams.0.indices.0.index_name: backing_index } + - do: + indices.get_mapping: + index: $backing_index + + - match: { .$backing_index.mappings.properties.@timestamp.type: date } + - match: { .$backing_index.mappings.properties.host.properties.name.type: text } --- create logsdb data stream without index sorting and ignore_above on host.name: @@ -963,12 +1001,21 @@ create logsdb data stream with multi-fields on host.name and no sorting: data_stream: {} - do: - catch: bad_request indices.create_data_stream: name: logsdb-no-sort-multi-fields + - is_true: acknowledged - - match: { error.type: "illegal_argument_exception" } - - match: { error.reason: "docvalues not found for index sort field:[host.name]" } + - do: + indices.get_data_stream: + name: logsdb-no-sort-multi-fields + + - set: { data_streams.0.indices.0.index_name: backing_index } + - do: + indices.get_mapping: + index: $backing_index + + - match: { .$backing_index.mappings.properties.@timestamp.type: date } + - match: { .$backing_index.mappings.properties.host.properties.name.type: text } --- create logsdb data stream with custom empty sorting: From da42d7710b1c685e3b20628b8c646be6124f38aa Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Fri, 10 Jan 2025 11:39:18 +0000 Subject: [PATCH 05/13] Reinstate old TransportVersion BwC condition (#119669) QueryBuilderBWCIT needs to read serialized data created by <8.8 nodes throughout 9.x releases --- muted-tests.yml | 6 ------ .../upgrades/QueryBuilderBWCIT.java | 18 +++++++++++++++++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 6b7f068ce4593..10bb16f97f39b 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -155,12 +155,6 @@ tests: - class: org.elasticsearch.xpack.ml.integration.RegressionIT method: testTwoJobsWithSameRandomizeSeedUseSameTrainingSet issue: https://github.com/elastic/elasticsearch/issues/117805 -- class: org.elasticsearch.upgrades.QueryBuilderBWCIT - method: testQueryBuilderBWC {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/116990 -- class: org.elasticsearch.xpack.restart.QueryBuilderBWCIT - method: testQueryBuilderBWC {p0=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/116989 - class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS2UnavailableRemotesIT method: testEsqlRcs2UnavailableRemoteScenarios issue: https://github.com/elastic/elasticsearch/issues/117419 diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java index aac2c661dea9f..8b74657becb24 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java @@ -12,6 +12,8 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.Strings; @@ -21,6 +23,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; import org.elasticsearch.index.query.DisMaxQueryBuilder; @@ -51,6 +54,7 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.cluster.ClusterState.VERSION_INTRODUCING_TRANSPORT_VERSIONS; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; /** @@ -245,7 +249,19 @@ public void testQueryBuilderBWC() throws Exception { InputStream in = new ByteArrayInputStream(qbSource, 0, qbSource.length); StreamInput input = new NamedWriteableAwareStreamInput(new InputStreamStreamInput(in), registry) ) { - input.setTransportVersion(TransportVersion.readVersion(input)); + @UpdateForV10(owner = UpdateForV10.Owner.SEARCH_FOUNDATIONS) // won't need to read <8.8 data anymore + boolean originalClusterHasTransportVersion = parseLegacyVersion(getOldClusterVersion()).map( + v -> v.onOrAfter(VERSION_INTRODUCING_TRANSPORT_VERSIONS) + ).orElse(true); + TransportVersion transportVersion; + if (originalClusterHasTransportVersion == false) { + transportVersion = TransportVersion.fromId( + parseLegacyVersion(getOldClusterVersion()).map(Version::id).orElse(TransportVersions.MINIMUM_COMPATIBLE.id()) + ); + } else { + transportVersion = TransportVersion.readVersion(input); + } + input.setTransportVersion(transportVersion); QueryBuilder queryBuilder = input.readNamedWriteable(QueryBuilder.class); assert in.read() == -1; assertEquals(expectedQueryBuilder, queryBuilder); From 31f11c3c0cb343f01a1e76421ed8d2bf927ffd6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20FOUCRET?= Date: Fri, 10 Jan 2025 12:49:28 +0100 Subject: [PATCH 06/13] [ES|QL] Enable KQL function as a tech preview (#119730) --- docs/changelog/119730.yaml | 5 +++++ docs/reference/esql/esql-limitations.asciidoc | 2 +- .../esql/functions/kibana/definition/kql.json | 2 +- .../esql/functions/search-functions.asciidoc | 2 ++ .../org/elasticsearch/TransportVersions.java | 1 + .../rest/action/search/SearchCapabilities.java | 6 +----- .../xpack/esql/plugin/KqlFunctionIT.java | 8 -------- .../xpack/esql/action/EsqlCapabilities.java | 2 +- .../function/EsqlFunctionRegistry.java | 6 ++++-- .../function/fulltext/FullTextWritables.java | 4 +--- .../xpack/esql/analysis/VerifierTests.java | 17 ----------------- .../expression/function/fulltext/KqlTests.java | 7 ------- .../LocalPhysicalPlanOptimizerTests.java | 15 --------------- .../org/elasticsearch/xpack/kql/KqlPlugin.java | 7 +------ .../xpack/kql/query/KqlQueryBuilder.java | 2 +- .../xpack/kql/query/KqlQueryBuilderTests.java | 8 -------- .../rest-api-spec/test/esql/60_usage.yml | 2 +- 17 files changed, 20 insertions(+), 76 deletions(-) create mode 100644 docs/changelog/119730.yaml diff --git a/docs/changelog/119730.yaml b/docs/changelog/119730.yaml new file mode 100644 index 0000000000000..09ff1362b3b1f --- /dev/null +++ b/docs/changelog/119730.yaml @@ -0,0 +1,5 @@ +pr: 119730 +summary: Enable KQL function as a tech preview +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/reference/esql/esql-limitations.asciidoc b/docs/reference/esql/esql-limitations.asciidoc index 0710acf3749d2..8ce8064a8161c 100644 --- a/docs/reference/esql/esql-limitations.asciidoc +++ b/docs/reference/esql/esql-limitations.asciidoc @@ -150,7 +150,7 @@ FROM books Note that, because of <>, any queries on `text` fields that do not explicitly use the full-text functions, -<> or <>, will behave as if the fields are actually `keyword` fields: +<>, <> or <>, will behave as if the fields are actually `keyword` fields: they are case-sensitive and need to match the full string. [discrete] diff --git a/docs/reference/esql/functions/kibana/definition/kql.json b/docs/reference/esql/functions/kibana/definition/kql.json index 6960681fbbf0d..440786ec63e77 100644 --- a/docs/reference/esql/functions/kibana/definition/kql.json +++ b/docs/reference/esql/functions/kibana/definition/kql.json @@ -33,5 +33,5 @@ "FROM books \n| WHERE KQL(\"author: Faulkner\")\n| KEEP book_no, author \n| SORT book_no \n| LIMIT 5;" ], "preview" : true, - "snapshot_only" : true + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/search-functions.asciidoc b/docs/reference/esql/functions/search-functions.asciidoc index bfdd288542782..a36473a7e8869 100644 --- a/docs/reference/esql/functions/search-functions.asciidoc +++ b/docs/reference/esql/functions/search-functions.asciidoc @@ -18,9 +18,11 @@ See <> for infor {esql} supports these full-text search functions: // tag::search_list[] +* experimental:[] <> * experimental:[] <> * experimental:[] <> // end::search_list[] +include::layout/kql.asciidoc[] include::layout/match.asciidoc[] include::layout/qstr.asciidoc[] diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index c5bb47ce1e4f7..f0f3d27c6e86c 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -155,6 +155,7 @@ static TransportVersion def(int id) { public static final TransportVersion TRACK_INDEX_FAILED_DUE_TO_VERSION_CONFLICT_METRIC = def(8_820_00_0); public static final TransportVersion REPLACE_FAILURE_STORE_OPTIONS_WITH_SELECTOR_SYNTAX = def(8_821_00_0); public static final TransportVersion ELASTIC_INFERENCE_SERVICE_UNIFIED_CHAT_COMPLETIONS_INTEGRATION = def(8_822_00_0); + public static final TransportVersion KQL_QUERY_TECH_PREVIEW = def(8_823_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java index c343141490799..214d257f0ed6a 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java @@ -9,8 +9,6 @@ package org.elasticsearch.rest.action.search; -import org.elasticsearch.Build; - import java.util.HashSet; import java.util.Set; @@ -60,9 +58,7 @@ private SearchCapabilities() {} capabilities.add(KNN_QUANTIZED_VECTOR_RESCORE); capabilities.add(MOVING_FN_RIGHT_MATH); capabilities.add(K_DEFAULT_TO_SIZE); - if (Build.current().isSnapshot()) { - capabilities.add(KQL_QUERY_SUPPORTED); - } + capabilities.add(KQL_QUERY_SUPPORTED); capabilities.add(HIGHLIGHT_MAX_ANALYZED_OFFSET_DEFAULT); CAPABILITIES = Set.copyOf(capabilities); } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java index 0e84ac7588ad6..39c963e938ac5 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java @@ -15,10 +15,8 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.kql.KqlPlugin; import org.junit.Before; -import org.junit.BeforeClass; import java.util.Collection; import java.util.List; @@ -27,12 +25,6 @@ import static org.hamcrest.CoreMatchers.containsString; public class KqlFunctionIT extends AbstractEsqlIntegTestCase { - - @BeforeClass - protected static void ensureKqlFunctionEnabled() { - assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); - } - @Before public void setupIndex() { createAndPopulateIndex(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 574c0f176bcf8..999ada6feba03 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -452,7 +452,7 @@ public enum Cap { /** * KQL function */ - KQL_FUNCTION(Build.current().isSnapshot()), + KQL_FUNCTION, /** * Hash function diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index d310973d6dbe7..bb170f546b54e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -424,7 +424,10 @@ private static FunctionDefinition[][] functions() { def(MvSum.class, MvSum::new, "mv_sum"), def(Split.class, Split::new, "split") }, // fulltext functions - new FunctionDefinition[] { def(Match.class, bi(Match::new), "match"), def(QueryString.class, uni(QueryString::new), "qstr") } }; + new FunctionDefinition[] { + def(Kql.class, uni(Kql::new), "kql"), + def(Match.class, bi(Match::new), "match"), + def(QueryString.class, uni(QueryString::new), "qstr") } }; } @@ -434,7 +437,6 @@ private static FunctionDefinition[][] snapshotFunctions() { // The delay() function is for debug/snapshot environments only and should never be enabled in a non-snapshot build. // This is an experimental function and can be removed without notice. def(Delay.class, Delay::new, "delay"), - def(Kql.class, uni(Kql::new), "kql"), def(Rate.class, Rate::withUnresolvedTimestamp, "rate"), def(Term.class, bi(Term::new), "term") } }; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java index 245aca5b7328e..a3a14004b1c89 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java @@ -25,10 +25,8 @@ public static List getNamedWriteables() { entries.add(MultiMatchQueryPredicate.ENTRY); entries.add(QueryString.ENTRY); entries.add(Match.ENTRY); + entries.add(Kql.ENTRY); - if (EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()) { - entries.add(Kql.ENTRY); - } if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { entries.add(Term.ENTRY); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index b2362b5c2aa68..d78c4bfa21ced 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1288,9 +1288,6 @@ public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception { } public void testKqlFunctionsNotAllowedAfterCommands() throws Exception { - // Skip test if the kql function is not enabled. - assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); - // Source commands assertEquals("1:13: [KQL] function cannot be used after SHOW", error("show info | where kql(\"8.16.0\")")); assertEquals("1:17: [KQL] function cannot be used after ROW", error("row a= \"Anna\" | where kql(\"Anna\")")); @@ -1348,9 +1345,6 @@ public void testQueryStringFunctionOnlyAllowedInWhere() throws Exception { } public void testKqlFunctionOnlyAllowedInWhere() throws Exception { - // Skip test if the kql function is not enabled. - assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); - assertEquals("1:9: [KQL] function is only supported in WHERE commands", error("row a = kql(\"Anna\")")); checkFullTextFunctionsOnlyAllowedInWhere("KQL", "kql(\"Anna\")", "function"); } @@ -1402,9 +1396,6 @@ public void testQueryStringFunctionArgNotNullOrConstant() throws Exception { } public void testKqlFunctionArgNotNullOrConstant() throws Exception { - // Skip test if the kql function is not enabled. - assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); - assertEquals( "1:19: argument of [kql(first_name)] must be a constant, received [first_name]", error("from test | where kql(first_name)") @@ -1418,9 +1409,6 @@ public void testQueryStringWithDisjunctions() { } public void testKqlFunctionWithDisjunctions() { - // Skip test if the kql function is not enabled. - assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); - checkWithDisjunctions("KQL", "kql(\"first_name: Anna\")", "function"); } @@ -1463,8 +1451,6 @@ public void testFullTextFunctionsDisjunctions() { checkWithFullTextFunctionsDisjunctions("MATCH", "match(last_name, \"Smith\")", "function"); checkWithFullTextFunctionsDisjunctions(":", "last_name : \"Smith\"", "operator"); checkWithFullTextFunctionsDisjunctions("QSTR", "qstr(\"last_name: Smith\")", "function"); - - assumeTrue("KQL function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); checkWithFullTextFunctionsDisjunctions("KQL", "kql(\"last_name: Smith\")", "function"); } @@ -1493,9 +1479,6 @@ public void testQueryStringFunctionWithNonBooleanFunctions() { } public void testKqlFunctionWithNonBooleanFunctions() { - // Skip test if the kql function is not enabled. - assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); - checkFullTextFunctionsWithNonBooleanFunctions("KQL", "kql(\"first_name: Anna\")", "function"); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java index d97be6b169eef..118bf56c5d6b9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java @@ -10,21 +10,14 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; -import org.junit.BeforeClass; import java.util.List; import java.util.function.Supplier; public class KqlTests extends NoneFieldFullTextFunctionTestCase { - @BeforeClass - protected static void ensureKqlFunctionEnabled() { - assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); - } - public KqlTests(@Name("TestCase") Supplier testCaseSupplier) { super(testCaseSupplier); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index fd12b7eef7e7e..6dee34323443d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -796,9 +796,6 @@ public void testMatchFunctionMultipleMatchClauses() { * \_EsQueryExec[test], indexMode[standard], query[{"kql":{"query":"last_name: Smith"}}] */ public void testKqlFunction() { - // Skip test if the kql function is not enabled. - assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); - var plan = plannerOptimizer.plan(""" from test | where kql("last_name: Smith") @@ -827,9 +824,6 @@ public void testKqlFunction() { * "boost":1.0}}][_doc{f}#1423], limit[1000], sort[] estimatedRowSize[324] */ public void testKqlFunctionConjunctionWhereOperands() { - // Skip test if the kql function is not enabled. - assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); - String queryText = """ from test | where kql("last_name: Smith") and emp_no > 10010 @@ -864,9 +858,6 @@ public void testKqlFunctionConjunctionWhereOperands() { * "source":"cidr_match(ip, \"127.0.0.1/32\")@2:38"}}],"boost":1.0}}][_doc{f}#21], limit[1000], sort[] estimatedRowSize[354] */ public void testKqlFunctionWithFunctionsPushedToLucene() { - // Skip test if the kql function is not enabled. - assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); - String queryText = """ from test | where kql("last_name: Smith") and cidr_match(ip, "127.0.0.1/32") @@ -902,9 +893,6 @@ public void testKqlFunctionWithFunctionsPushedToLucene() { * "boost":1.0}}][_doc{f}#1167], limit[1000], sort[] estimatedRowSize[324] */ public void testKqlFunctionMultipleWhereClauses() { - // Skip test if the kql function is not enabled. - assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); - String queryText = """ from test | where kql("last_name: Smith") @@ -939,9 +927,6 @@ public void testKqlFunctionMultipleWhereClauses() { * {"kql":{"query":"emp_no > 10010"}}],"boost":1.0}}] */ public void testKqlFunctionMultipleKqlClauses() { - // Skip test if the kql function is not enabled. - assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); - String queryText = """ from test | where kql("last_name: Smith") and kql("emp_no > 10010") diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/KqlPlugin.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/KqlPlugin.java index 217513bd2c0da..e31db37953d43 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/KqlPlugin.java +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/KqlPlugin.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.kql; -import org.elasticsearch.Build; import org.elasticsearch.plugins.ExtensiblePlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; @@ -18,10 +17,6 @@ public class KqlPlugin extends Plugin implements SearchPlugin, ExtensiblePlugin { @Override public List> getQueries() { - if (Build.current().isSnapshot()) { - return List.of(new SearchPlugin.QuerySpec<>(KqlQueryBuilder.NAME, KqlQueryBuilder::new, KqlQueryBuilder::fromXContent)); - } - - return List.of(); + return List.of(new SearchPlugin.QuerySpec<>(KqlQueryBuilder.NAME, KqlQueryBuilder::new, KqlQueryBuilder::fromXContent)); } } diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java index e2817665d8f79..a77e1453e66ca 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java @@ -97,7 +97,7 @@ public static KqlQueryBuilder fromXContent(XContentParser parser) { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.KQL_QUERY_ADDED; + return TransportVersions.KQL_QUERY_TECH_PREVIEW; } public String queryString() { diff --git a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilderTests.java b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilderTests.java index 7323f7d6d1a4e..2643e95df8946 100644 --- a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilderTests.java +++ b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilderTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.kql.query; import org.apache.lucene.search.Query; -import org.elasticsearch.Build; import org.elasticsearch.core.Strings; import org.elasticsearch.index.query.MultiMatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -22,7 +21,6 @@ import org.elasticsearch.test.AbstractQueryTestCase; import org.elasticsearch.xpack.kql.KqlPlugin; import org.hamcrest.Matchers; -import org.junit.BeforeClass; import java.io.IOException; import java.util.Collection; @@ -36,12 +34,6 @@ import static org.hamcrest.Matchers.nullValue; public class KqlQueryBuilderTests extends AbstractQueryTestCase { - @BeforeClass - protected static void ensureSnapshotBuild() { - assumeTrue("requires snapshot builds", Build.current().isSnapshot()); - } - - @Override protected Collection> getPlugins() { return List.of(KqlPlugin.class); } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index 35e5d66e91d97..0d9c66012dbfc 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -163,4 +163,4 @@ setup: - match: {esql.functions.cos: $functions_cos} - gt: {esql.functions.to_long: $functions_to_long} - match: {esql.functions.coalesce: $functions_coalesce} - - length: {esql.functions: 129} # check the "sister" test above for a likely update to the same esql.functions length check + - length: {esql.functions: 130} # check the "sister" test above for a likely update to the same esql.functions length check From b68fc8e2440780fc881b64c4c3d7a32e4dfa8186 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 10 Jan 2025 13:25:46 +0100 Subject: [PATCH 07/13] Remove synthetic role names for legacy API keys (#119930) Forward-ports: https://github.com/elastic/elasticsearch/pull/119844 --- .../core/security/authc/Authentication.java | 33 ++++++++++++- .../AuthenticationSerializationTests.java | 48 +++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java index 13c65c457d96b..561a94dd3cc05 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java @@ -55,6 +55,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import static org.elasticsearch.common.Strings.EMPTY_ARRAY; import static org.elasticsearch.transport.RemoteClusterPortSettings.TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; @@ -170,21 +171,49 @@ public Authentication(StreamInput in) throws IOException { type = AuthenticationType.REALM; metadata = Map.of(); } + if (innerUser != null) { - authenticatingSubject = new Subject(innerUser, authenticatedBy, version, metadata); + authenticatingSubject = new Subject( + copyUserWithRolesRemovedForLegacyApiKeys(version, innerUser), + authenticatedBy, + version, + metadata + ); // The lookup user for run-as currently doesn't have authentication metadata associated with them because // lookupUser only returns the User object. The lookup user for authorization delegation does have // authentication metadata, but the realm does not expose this difference between authenticatingUser and // delegateUser so effectively this is handled together with the authenticatingSubject not effectiveSubject. + // Note: we do not call copyUserWithRolesRemovedForLegacyApiKeys here because an API key is never the target of run-as effectiveSubject = new Subject(outerUser, lookedUpBy, version, Map.of()); } else { - authenticatingSubject = effectiveSubject = new Subject(outerUser, authenticatedBy, version, metadata); + authenticatingSubject = effectiveSubject = new Subject( + copyUserWithRolesRemovedForLegacyApiKeys(version, outerUser), + authenticatedBy, + version, + metadata + ); } + if (Assertions.ENABLED) { checkConsistency(); } } + private User copyUserWithRolesRemovedForLegacyApiKeys(TransportVersion version, User user) { + // API keys prior to 7.8 had synthetic role names. Strip these out to maintain the invariant that API keys don't have role names + if (type == AuthenticationType.API_KEY && version.onOrBefore(TransportVersions.V_7_8_0) && user.roles().length > 0) { + logger.debug( + "Stripping [{}] roles from API key user [{}] for legacy version [{}]", + user.roles().length, + user.principal(), + version + ); + return new User(user.principal(), EMPTY_ARRAY, user.fullName(), user.email(), user.metadata(), user.enabled()); + } else { + return user; + } + } + /** * Get the {@link Subject} that performs the actual authentication. This normally means it provides a credentials. */ diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java index 11a8f894d7cf7..d9d235c8ce41b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java @@ -13,16 +13,21 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.TransportVersionUtils; import org.elasticsearch.transport.RemoteClusterPortSettings; +import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer; import org.elasticsearch.xpack.core.security.user.ElasticUser; import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.KibanaSystemUser; import org.elasticsearch.xpack.core.security.user.KibanaUser; import org.elasticsearch.xpack.core.security.user.User; +import java.io.IOException; import java.util.Arrays; +import java.util.Map; import static org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationSerializationHelper; +import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; @@ -171,4 +176,47 @@ public void testReservedUserSerialization() throws Exception { assertEquals(kibanaSystemUser, readFrom); } + + public void testRolesRemovedFromUserForLegacyApiKeys() throws IOException { + TransportVersion transportVersion = TransportVersionUtils.randomVersionBetween( + random(), + TransportVersions.V_7_0_0, + TransportVersions.V_7_8_0 + ); + Subject authenticatingSubject = new Subject( + new User("foo", "role"), + new Authentication.RealmRef(AuthenticationField.API_KEY_REALM_NAME, AuthenticationField.API_KEY_REALM_TYPE, "node"), + transportVersion, + Map.of(AuthenticationField.API_KEY_ID_KEY, "abc") + ); + Subject effectiveSubject = new Subject( + new User("bar", "role"), + new Authentication.RealmRef("native", "native", "node"), + transportVersion, + Map.of() + ); + + { + Authentication actual = AuthenticationContextSerializer.decode( + Authentication.doEncode(authenticatingSubject, authenticatingSubject, Authentication.AuthenticationType.API_KEY) + ); + assertThat(actual.getAuthenticatingSubject().getUser().roles(), is(emptyArray())); + } + + { + Authentication actual = AuthenticationContextSerializer.decode( + Authentication.doEncode(effectiveSubject, authenticatingSubject, Authentication.AuthenticationType.API_KEY) + ); + assertThat(actual.getAuthenticatingSubject().getUser().roles(), is(emptyArray())); + assertThat(actual.getEffectiveSubject().getUser().roles(), is(arrayContaining("role"))); + } + + { + // do not strip roles for authentication methods other than API key + Authentication actual = AuthenticationContextSerializer.decode( + Authentication.doEncode(effectiveSubject, effectiveSubject, Authentication.AuthenticationType.REALM) + ); + assertThat(actual.getAuthenticatingSubject().getUser().roles(), is(arrayContaining("role"))); + } + } } From 6ca7e755bd23537abbe4a2859782f9baa17bd0d6 Mon Sep 17 00:00:00 2001 From: Artem Prigoda Date: Fri, 10 Jan 2025 13:28:54 +0100 Subject: [PATCH 08/13] Add possibility to acquire permits on primary shards with different checks (#119794) Since #42241 we check that the shard must be in a primary mode for acquiring a primary permit on it. We would like customize this check and an option to perform different checks before running the `onPermitAcquired` listener. For example, we would to skip the primary mode check when we acquire primary permits during recovering of a hollow indexing shard. See ES-10487 --- .../elasticsearch/index/shard/IndexShard.java | 77 +++++++++++++++---- .../index/shard/IndexShardTests.java | 15 ++++ 2 files changed, 75 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 2c9741a87c335..ec260a40452b5 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -189,6 +189,7 @@ import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.index.seqno.RetentionLeaseActions.RETAIN_ALL; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; +import static org.elasticsearch.index.shard.IndexShard.PrimaryPermitCheck.CHECK_PRIMARY_MODE; public class IndexShard extends AbstractIndexShardComponent implements IndicesClusterStateService.Shard { @@ -3568,24 +3569,48 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) { ); } + /** + * Check to run before running the primary permit operation + */ + public enum PrimaryPermitCheck { + CHECK_PRIMARY_MODE, + /** + * IMPORTANT: Currently intented to be used only for acquiring primary permits during the recovery of hollow shards. + * Don't disable primary mode checks unless you're really sure. + */ + NONE + } + /** * Acquire a primary operation permit whenever the shard is ready for indexing. If a permit is directly available, the provided * ActionListener will be called on the calling thread. During relocation hand-off, permit acquisition can be delayed. The provided * ActionListener will then be called using the provided executor. - * */ public void acquirePrimaryOperationPermit(ActionListener onPermitAcquired, Executor executorOnDelay) { - acquirePrimaryOperationPermit(onPermitAcquired, executorOnDelay, false); + acquirePrimaryOperationPermit(onPermitAcquired, executorOnDelay, false, CHECK_PRIMARY_MODE); } public void acquirePrimaryOperationPermit( ActionListener onPermitAcquired, Executor executorOnDelay, boolean forceExecution + ) { + acquirePrimaryOperationPermit(onPermitAcquired, executorOnDelay, forceExecution, CHECK_PRIMARY_MODE); + } + + public void acquirePrimaryOperationPermit( + ActionListener onPermitAcquired, + Executor executorOnDelay, + boolean forceExecution, + PrimaryPermitCheck primaryPermitCheck ) { verifyNotClosed(); assert shardRouting.primary() : "acquirePrimaryOperationPermit should only be called on primary shard: " + shardRouting; - indexShardOperationPermits.acquire(wrapPrimaryOperationPermitListener(onPermitAcquired), executorOnDelay, forceExecution); + indexShardOperationPermits.acquire( + wrapPrimaryOperationPermitListener(primaryPermitCheck, onPermitAcquired), + executorOnDelay, + forceExecution + ); } public boolean isPrimaryMode() { @@ -3593,33 +3618,51 @@ public boolean isPrimaryMode() { return replicationTracker.isPrimaryMode(); } + public void acquireAllPrimaryOperationsPermits(final ActionListener onPermitAcquired, final TimeValue timeout) { + acquireAllPrimaryOperationsPermits(onPermitAcquired, timeout, CHECK_PRIMARY_MODE); + } + /** * Acquire all primary operation permits. Once all permits are acquired, the provided ActionListener is called. * It is the responsibility of the caller to close the {@link Releasable}. */ - public void acquireAllPrimaryOperationsPermits(final ActionListener onPermitAcquired, final TimeValue timeout) { + public void acquireAllPrimaryOperationsPermits( + final ActionListener onPermitAcquired, + final TimeValue timeout, + final PrimaryPermitCheck primaryPermitCheck + ) { verifyNotClosed(); assert shardRouting.primary() : "acquireAllPrimaryOperationsPermits should only be called on primary shard: " + shardRouting; - asyncBlockOperations(wrapPrimaryOperationPermitListener(onPermitAcquired), timeout.duration(), timeout.timeUnit()); + asyncBlockOperations( + wrapPrimaryOperationPermitListener(primaryPermitCheck, onPermitAcquired), + timeout.duration(), + timeout.timeUnit() + ); } /** - * Wraps the action to run on a primary after acquiring permit. This wrapping is used to check if the shard is in primary mode before - * executing the action. + * Wraps the action to run on a primary after acquiring permit. * + * @param primaryPermitCheck check to run before the primary mode operation * @param listener the listener to wrap * @return the wrapped listener */ - private ActionListener wrapPrimaryOperationPermitListener(final ActionListener listener) { - return listener.delegateFailure((l, r) -> { - if (isPrimaryMode()) { - l.onResponse(r); - } else { - r.close(); - l.onFailure(new ShardNotInPrimaryModeException(shardId, state)); - } - }); + private ActionListener wrapPrimaryOperationPermitListener( + final PrimaryPermitCheck primaryPermitCheck, + final ActionListener listener + ) { + return switch (primaryPermitCheck) { + case CHECK_PRIMARY_MODE -> listener.delegateFailure((l, r) -> { + if (isPrimaryMode()) { + l.onResponse(r); + } else { + r.close(); + l.onFailure(new ShardNotInPrimaryModeException(shardId, state)); + } + }); + case NONE -> listener; + }; } private void asyncBlockOperations(ActionListener onPermitAcquired, long timeout, TimeUnit timeUnit) { @@ -3657,7 +3700,7 @@ public void runUnderPrimaryPermit(final Runnable runnable, final Consumer void bumpPrimaryTerm( diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 7d436ab5d8d22..0bbf63b07c4cb 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -790,6 +790,21 @@ public void onFailure(final Exception e) { } }, TimeValue.timeValueSeconds(30)); latch.await(); + + // It's possible to acquire permits if we skip the primary mode check + var permitAcquiredLatch = new CountDownLatch(1); + indexShard.acquirePrimaryOperationPermit(ActionListener.wrap(r -> { + r.close(); + permitAcquiredLatch.countDown(); + }, Assert::assertNotNull), EsExecutors.DIRECT_EXECUTOR_SERVICE, false, IndexShard.PrimaryPermitCheck.NONE); + safeAwait(permitAcquiredLatch); + + var allPermitsAcquiredLatch = new CountDownLatch(1); + indexShard.acquireAllPrimaryOperationsPermits(ActionListener.wrap(r -> { + r.close(); + allPermitsAcquiredLatch.countDown(); + }, Assert::assertNotNull), TimeValue.timeValueSeconds(30), IndexShard.PrimaryPermitCheck.NONE); + safeAwait(allPermitsAcquiredLatch); } if (Assertions.ENABLED && indexShard.routingEntry().isRelocationTarget() == false) { From 25a04481ee7a388eddcfc885ccb44b405d19e158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20FOUCRET?= Date: Fri, 10 Jan 2025 15:21:43 +0100 Subject: [PATCH 09/13] Fix InferenceCrudIT.testGetServicesWithCompletionTaskType failure on non snapshot buiild (#119928) --- .../xpack/inference/InferenceCrudIT.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java index 6634eecc2c959..50f83dc7b02eb 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java @@ -242,7 +242,12 @@ public void testGetServicesWithRerankTaskType() throws IOException { @SuppressWarnings("unchecked") public void testGetServicesWithCompletionTaskType() throws IOException { List services = getServices(TaskType.COMPLETION); - assertThat(services.size(), equalTo(10)); + if ((ElasticInferenceServiceFeature.DEPRECATED_ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled() + || ElasticInferenceServiceFeature.ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled())) { + assertThat(services.size(), equalTo(10)); + } else { + assertThat(services.size(), equalTo(9)); + } String[] providers = new String[services.size()]; for (int i = 0; i < services.size(); i++) { @@ -250,8 +255,7 @@ public void testGetServicesWithCompletionTaskType() throws IOException { providers[i] = (String) serviceConfig.get("service"); } - assertArrayEquals( - providers, + var providerList = new ArrayList<>( List.of( "alibabacloud-ai-search", "amazonbedrock", @@ -259,12 +263,18 @@ public void testGetServicesWithCompletionTaskType() throws IOException { "azureaistudio", "azureopenai", "cohere", - "elastic", "googleaistudio", "openai", "streaming_completion_test_service" - ).toArray() + ) ); + + if ((ElasticInferenceServiceFeature.DEPRECATED_ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled() + || ElasticInferenceServiceFeature.ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled())) { + providerList.add(6, "elastic"); + } + + assertArrayEquals(providers, providerList.toArray()); } @SuppressWarnings("unchecked") From 4f5ad9db69e3c2dd620aaec2e9810fc67e2b4dd8 Mon Sep 17 00:00:00 2001 From: Ioana Tagirta Date: Fri, 10 Jan 2025 15:40:42 +0100 Subject: [PATCH 10/13] Unmute CSV tests (#119940) --- muted-tests.yml | 2 -- .../src/test/java/org/elasticsearch/xpack/esql/CsvTests.java | 4 ++++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 10bb16f97f39b..9a87b4b7e1452 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -245,8 +245,6 @@ tests: - class: org.elasticsearch.xpack.security.authz.IndicesAndAliasesResolverTests method: testBackingIndicesAreNotVisibleWhenNotIncludedByRequestWithoutWildcard issue: https://github.com/elastic/elasticsearch/issues/119909 -- class: org.elasticsearch.xpack.esql.CsvTests - issue: https://github.com/elastic/elasticsearch/issues/119918 # Examples: # diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index fe9a5e569669d..7d4374934ab82 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -270,6 +270,10 @@ public final void test() throws Throwable { "can't use TERM function in csv tests", testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.TERM_FUNCTION.capabilityName()) ); + assumeFalse( + "CSV tests cannot correctly handle the field caps change", + testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.SEMANTIC_TEXT_FIELD_CAPS.capabilityName()) + ); if (Build.current().isSnapshot()) { assertThat( "Capability is not included in the enabled list capabilities on a snapshot build. Spelling mistake?", From f2d069e2bf59e7c54b3972f334fdb3603c77978f Mon Sep 17 00:00:00 2001 From: Artem Shelkovnikov Date: Fri, 10 Jan 2025 15:56:38 +0100 Subject: [PATCH 11/13] Update Sharepoint Online connector documentation (#119933) Co-authored-by: Liam Thompson <32779855+leemthompo@users.noreply.github.com> --- .../connectors-sharepoint-online.asciidoc | 77 +++++++++++++++---- 1 file changed, 63 insertions(+), 14 deletions(-) diff --git a/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc b/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc index 2680e3ff840a6..d09e089f194ad 100644 --- a/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc +++ b/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc @@ -75,12 +75,10 @@ Follow these steps: * Leave the *Redirect URIs* blank for now. * *Register* the application. * Find and keep the **Application (client) ID** and **Directory (tenant) ID** handy. -* Locate the **Secret** by navigating to **Client credentials: Certificates & Secrets**. -* Select **New client secret** -* Pick a name for your client secret. -Select an expiration date. (At this expiration date, you will need to generate a new secret and update your connector configuration.) -** Save the client secret **Secret ID** before leaving this screen. -** Save the client secret **Value** before leaving this screen. +* Create a certificate and private key. This can, for example, be done by running `openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout azure_app.key -out azure_app.crt` command. Store both in a safe and secure place +* Locate the **Certificates** by navigating to **Client credentials: Certificates & Secrets**. +* Select **Upload certificate** +* Upload the certificate created in one of previous steps: `azure_app.crt` * Set up the permissions the OAuth App will request from the Azure Portal service account. ** Navigate to **API Permissions** and click **Add Permission**. ** Add **application permissions** until the list looks like the following: @@ -114,6 +112,24 @@ When entities are not available via the Graph API the connector falls back to us [discrete#es-connectors-sharepoint-online-oauth-app-permissions] ====== SharePoint permissions +Microsoft is https://learn.microsoft.com/en-us/sharepoint/dev/sp-add-ins/retirement-announcement-for-azure-acs[retiring Azure Access Control Service (ACS)]. This affects permission configuration: + +* *Tenants created after November 1st, 2024*: Certificate authentication is required +* *Tenants created before November 1st, 2024*: Secret-based authentication must be migrated to certificate authentication by April 2nd, 2026 + +[discrete#es-connectors-sharepoint-online-oauth-app-certificate-auth] +===== Certificate Authentication + +This authentication method does not require additional setup other than creating and uploading certificates to the OAuth App. + +[discrete#es-connectors-sharepoint-online-oauth-app-secret-auth] +===== Secret Authentication + +[IMPORTANT] +==== +This method is only applicable to tenants created before November 1st, 2024. This method will be fully retired as of April 2nd, 2026. +==== + Refer to the following documentation for setting https://learn.microsoft.com/en-us/sharepoint/dev/solution-guidance/security-apponly-azureacs[SharePoint permissions^]. * To set `DisableCustomAppAuthentication` to false, connect to SharePoint using PowerShell and run `set-spotenant -DisableCustomAppAuthentication $false` @@ -219,8 +235,17 @@ The tenant name for the Azure account hosting the Sharepoint Online instance. Client ID:: The client id to authenticate with SharePoint Online. +Authentication Method:: +Authentication method to use to connector to Sharepoint Online and Rest APIs. `secret` is deprecated and `certificate` is recommended. + Secret value:: -The secret value to authenticate with SharePoint Online. +The secret value to authenticate with SharePoint Online, if Authentication Method: `secret` is chosen. + +Content of certificate file:: +Content of certificate file if Authentication Method: `certificate` is chosen. + +Content of private key file:: +Content of private key file if Authentication Method: `certificate` is chosen. Comma-separated list of sites:: List of site collection names or paths to fetch from SharePoint. @@ -588,12 +613,10 @@ Follow these steps: * Leave the *Redirect URIs* blank for now. * *Register* the application. * Find and keep the **Application (client) ID** and **Directory (tenant) ID** handy. -* Locate the **Secret** by navigating to **Client credentials: Certificates & Secrets**. -* Select **New client secret** -* Pick a name for your client secret. -Select an expiration date. (At this expiration date, you will need to generate a new secret and update your connector configuration.) -** Save the client secret **Secret ID** before leaving this screen. -** Save the client secret **Value** before leaving this screen. +* Create a certificate and private key. This can, for example, be done by running `openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout azure_app.key -out azure_app.crt` command. Store both in a safe and secure place +* Locate the **Certificates** by navigating to **Client credentials: Certificates & Secrets**. +* Select **Upload certificate** +* Upload the certificate created in one of previous steps: `azure_app.crt` * Set up the permissions the OAuth App will request from the Azure Portal service account. ** Navigate to **API Permissions** and click **Add Permission**. ** Add **application permissions** until the list looks like the following: @@ -627,6 +650,23 @@ When entities are not available via the Graph API the connector falls back to us [discrete#es-connectors-sharepoint-online-client-oauth-app-permissions] ====== SharePoint permissions +Microsoft is https://learn.microsoft.com/en-us/sharepoint/dev/sp-add-ins/retirement-announcement-for-azure-acs[retiring Azure Access Control Service (ACS)]. This affects permission configuration: +* *Tenants created after November 1st, 2024*: Certificate authentication is required +* *Tenants created before November 1st, 2024*: Secret-based authentication must be migrated to certificate authentication by April 2nd, 2026 + +[discrete#es-connectors-sharepoint-online-client-oauth-app-certificate-auth] +===== Certificate Authentication + +This authentication method does not require additional setup other than creating and uploading certificates to the OAuth App. + +[discrete#es-connectors-sharepoint-online-client-oauth-app-secret-auth] +===== Secret Authentication + +[IMPORTANT] +==== +This method is only applicable to tenants created before November 1st, 2024. This method will be fully retired as of April 2nd, 2026. +==== + Refer to the following documentation for setting https://learn.microsoft.com/en-us/sharepoint/dev/solution-guidance/security-apponly-azureacs[SharePoint permissions^]. * To set `DisableCustomAppAuthentication` to false, connect to SharePoint using PowerShell and run `set-spotenant -DisableCustomAppAuthentication $false` @@ -742,8 +782,17 @@ The tenant name for the Azure account hosting the Sharepoint Online instance. `client_id`:: The client id to authenticate with SharePoint Online. +`auth_method`:: +Authentication method to use to connector to Sharepoint Online and Rest APIs. `secret` is deprecated and `certificate` is recommended. + `secret_value`:: -The secret value to authenticate with SharePoint Online. +The secret value to authenticate with SharePoint Online, if auth_method: `secret` is chosen. + +`certificate`:: +Content of certificate file if auth_method: `certificate` is chosen. + +`private_key`:: +Content of private key file if auth_method: `certificate` is chosen. `site_collections`:: List of site collection names or paths to fetch from SharePoint. From e6f9d6b5a977df968d87f30cde9a22a9de5a7de0 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sat, 11 Jan 2025 02:05:47 +1100 Subject: [PATCH 12/13] Mute org.elasticsearch.xpack.inference.InferenceCrudIT testGetServicesWithCompletionTaskType #119959 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 9a87b4b7e1452..36c47d3e0bd32 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -245,6 +245,9 @@ tests: - class: org.elasticsearch.xpack.security.authz.IndicesAndAliasesResolverTests method: testBackingIndicesAreNotVisibleWhenNotIncludedByRequestWithoutWildcard issue: https://github.com/elastic/elasticsearch/issues/119909 +- class: org.elasticsearch.xpack.inference.InferenceCrudIT + method: testGetServicesWithCompletionTaskType + issue: https://github.com/elastic/elasticsearch/issues/119959 # Examples: # From c4024dcfe33da5c586ad54ba674674bc6222aad5 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Fri, 10 Jan 2025 09:13:51 -0600 Subject: [PATCH 13/13] Add index and reindex request settings to speed up reindex (#119780) - set slices:auto on the reindex request - set refresh_interval: -1 on destination index before reindexing into it - set number_of_replicas: 0 on destination index before reindexing into it - reset refresh_interval and number_of_replicas to previous value or default after reindex --- docs/changelog/119780.yaml | 5 +++ ...indexDatastreamIndexTransportActionIT.java | 41 ++++++++++++++++++- ...ReindexDataStreamIndexTransportAction.java | 35 ++++++++++++++++ 3 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/119780.yaml diff --git a/docs/changelog/119780.yaml b/docs/changelog/119780.yaml new file mode 100644 index 0000000000000..5b7226741a416 --- /dev/null +++ b/docs/changelog/119780.yaml @@ -0,0 +1,5 @@ +pr: 119780 +summary: Add index and reindex request settings to speed up reindex +area: Data streams +type: enhancement +issues: [] diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java index 0ca58ecf0f0d5..0902f6ce6468a 100644 --- a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java @@ -15,6 +15,8 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; +import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; @@ -155,7 +157,11 @@ public void testSettingsAddedBeforeReindex() throws Exception { // update with a dynamic setting var numReplicas = randomIntBetween(0, 10); - var dynamicSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas).build(); + var refreshInterval = randomIntBetween(1, 100) + "s"; + var dynamicSettings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), refreshInterval) + .build(); indicesAdmin().updateSettings(new UpdateSettingsRequest(dynamicSettings, sourceIndex)).actionGet(); // call reindex @@ -167,6 +173,7 @@ public void testSettingsAddedBeforeReindex() throws Exception { var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet(); assertEquals(numReplicas, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS))); assertEquals(numShards, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS))); + assertEquals(refreshInterval, settingsResponse.getSetting(destIndex, IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey())); } public void testMappingsAddedToDestIndex() throws Exception { @@ -229,6 +236,38 @@ public void testReadOnlyAddedBack() { removeReadOnly(destIndex); } + public void testUpdateSettingsDefaultsRestored() { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + // ESIntegTestCase creates a template random_index_template which contains a value for number_of_replicas. + // Since this test checks the behavior of default settings, there cannot be a value for number_of_replicas, + // so we delete the template within this method. This has no effect on other tests which will still + // have the template created during their setup. + assertAcked( + indicesAdmin().execute(TransportDeleteIndexTemplateAction.TYPE, new DeleteIndexTemplateRequest("random_index_template")) + ); + + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + assertAcked(indicesAdmin().create(new CreateIndexRequest(sourceIndex))); + + // call reindex + var destIndex = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + .actionGet() + .getDestIndex(); + + var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(sourceIndex, destIndex)).actionGet(); + var destSettings = settingsResponse.getIndexToSettings().get(destIndex); + + assertEquals( + IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getDefault(destSettings), + IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.get(destSettings) + ); + assertEquals( + IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getDefault(destSettings), + IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.get(destSettings) + ); + } + public void testSettingsAndMappingsFromTemplate() throws IOException { assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java index 6dd38d35ca9f7..d86885ce0fbe4 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.admin.indices.readonly.AddIndexBlockRequest; import org.elasticsearch.action.admin.indices.readonly.AddIndexBlockResponse; import org.elasticsearch.action.admin.indices.readonly.TransportAddIndexBlockAction; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.action.support.IndicesOptions; @@ -25,6 +26,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.index.reindex.ReindexAction; import org.elasticsearch.index.reindex.ReindexRequest; @@ -95,6 +97,7 @@ protected void doExecute( .andThen(l -> deleteDestIfExists(destIndexName, l, taskId)) .andThen(l -> createIndex(sourceIndex, destIndexName, l, taskId)) .andThen(l -> reindex(sourceIndexName, destIndexName, l, taskId)) + .andThen(l -> copyOldSourceSettingsToDest(settingsBefore, destIndexName, l, taskId)) .andThen(l -> addBlockIfFromSource(WRITE, settingsBefore, destIndexName, l, taskId)) .andThen(l -> addBlockIfFromSource(READ_ONLY, settingsBefore, destIndexName, l, taskId)) .andThenApply(ignored -> new ReindexDataStreamIndexAction.Response(destIndexName)) @@ -147,6 +150,8 @@ private void createIndex( var removeReadOnlyOverride = Settings.builder() .putNull(IndexMetadata.SETTING_READ_ONLY) .putNull(IndexMetadata.SETTING_BLOCKS_WRITE) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1) .build(); var request = new CreateIndexFromSourceAction.Request( @@ -168,6 +173,7 @@ private void reindex(String sourceIndexName, String destIndexName, ActionListene reindexRequest.getSearchRequest().source().fetchSource(true); reindexRequest.setDestIndex(destIndexName); reindexRequest.setParentTask(parentTaskId); + reindexRequest.setSlices(0); // equivalent to slices=auto in rest api client.execute(ReindexAction.INSTANCE, reindexRequest, listener); } @@ -186,6 +192,35 @@ private void addBlockIfFromSource( } } + private void copyOldSourceSettingsToDest( + Settings settingsBefore, + String destIndexName, + ActionListener listener, + TaskId parentTaskId + ) { + logger.debug("Updating settings on destination index after reindex completes"); + + var settings = Settings.builder(); + copySettingOrUnset(settingsBefore, settings, IndexMetadata.SETTING_NUMBER_OF_REPLICAS); + copySettingOrUnset(settingsBefore, settings, IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey()); + + var updateSettingsRequest = new UpdateSettingsRequest(settings.build(), destIndexName); + updateSettingsRequest.setParentTask(parentTaskId); + var errorMessage = String.format(Locale.ROOT, "Could not update settings on index [%s]", destIndexName); + client.admin().indices().updateSettings(updateSettingsRequest, failIfNotAcknowledged(listener, errorMessage)); + } + + private static void copySettingOrUnset(Settings settingsBefore, Settings.Builder builder, String setting) { + // if setting was explicitly added to the source index + if (settingsBefore.get(setting) != null) { + // copy it back to the dest index + builder.copy(setting, settingsBefore); + } else { + // otherwise, delete from dest index so that it loads from the settings default + builder.putNull(setting); + } + } + public static String generateDestIndexName(String sourceIndex) { return "migrated-" + sourceIndex; }