Merge remote-tracking branch 'luigidellaquila/esql/remove_like_dead_c…

…ode' into esql/remove_like_dead_code

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"]
+            BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -272,8 +272,8 @@ steps:
         env:
           BWC_VERSION: 8.14.3
 
-      - label: "{{matrix.image}} / 8.15.3 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.3
+      - label: "{{matrix.image}} / 8.15.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.4
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -286,7 +286,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.15.3
+          BWC_VERSION: 8.15.4
 
       - label: "{{matrix.image}} / 8.16.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.0

.buildkite/pipelines/periodic.yml

@@ -287,8 +287,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.15.3 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.3#bwcTest
+      - label: 8.15.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.4#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -297,7 +297,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.15.3
+          BWC_VERSION: 8.15.4
         retry:
           automatic:
             - exit_status: "-1"
@@ -429,7 +429,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"]
+            BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -471,7 +471,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"]
+            BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -14,7 +14,7 @@ BWC_VERSION:
   - "8.12.2"
   - "8.13.4"
   - "8.14.3"
-  - "8.15.3"
+  - "8.15.4"
   - "8.16.0"
   - "8.17.0"
   - "9.0.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
-  - "8.15.3"
+  - "8.15.4"
   - "8.16.0"
   - "8.17.0"
   - "9.0.0"

docs/changelog/112881.yaml

@@ -0,0 +1,5 @@
+pr: 112881
+summary: "ESQL: Remove parent from `FieldAttribute`"
+area: ES|QL
+type: enhancement
+issues: []

docs/reference/connector/docs/connectors-content-extraction.asciidoc

@@ -90,7 +90,7 @@ include::_connectors-list-local-content-extraction.asciidoc[]
 Self-hosted content extraction is handled by a *separate* extraction service.
 
 The versions for the extraction service do not align with the Elastic stack.
-For version `8.11.x`, you should use extraction service version `0.3.x`.
+For versions after `8.11.x` (including {version}), you should use extraction service version `0.3.x`.
 
 You can run the service with the following command:
 

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -175,6 +175,8 @@ static TransportVersion def(int id) {
     public static final TransportVersion ML_INFERENCE_ATTACH_TO_EXISTSING_DEPLOYMENT = def(8_771_00_0);
     public static final TransportVersion CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY = def(8_772_00_0);
     public static final TransportVersion REMOVE_MIN_COMPATIBLE_SHARD_NODE = def(8_773_00_0);
+    public static final TransportVersion REVERT_REMOVE_MIN_COMPATIBLE_SHARD_NODE = def(8_774_00_0);
+    public static final TransportVersion ESQL_FIELD_ATTRIBUTE_PARENT_SIMPLIFIED = def(8_775_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/Version.java

@@ -186,6 +186,7 @@ public class Version implements VersionId<Version>, ToXContentFragment {
     public static final Version V_8_15_1 = new Version(8_15_01_99);
     public static final Version V_8_15_2 = new Version(8_15_02_99);
     public static final Version V_8_15_3 = new Version(8_15_03_99);
+    public static final Version V_8_15_4 = new Version(8_15_04_99);
     public static final Version V_8_16_0 = new Version(8_16_00_99);
     public static final Version V_8_17_0 = new Version(8_17_00_99);
     public static final Version V_9_0_0 = new Version(9_00_00_99);

server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.injection.guice.Inject;
@@ -119,18 +120,27 @@ public void onPrimaryOperationComplete(
             ActionListener<Void> listener
         ) {
             assert replicaRequest.primaryRefreshResult.refreshed() : "primary has not refreshed";
-            UnpromotableShardRefreshRequest unpromotableReplicaRequest = new UnpromotableShardRefreshRequest(
-                indexShardRoutingTable,
-                replicaRequest.primaryRefreshResult.primaryTerm(),
-                replicaRequest.primaryRefreshResult.generation(),
-                false
-            );
-            transportService.sendRequest(
-                transportService.getLocalNode(),
-                TransportUnpromotableShardRefreshAction.NAME,
-                unpromotableReplicaRequest,
-                new ActionListenerResponseHandler<>(listener.safeMap(r -> null), in -> ActionResponse.Empty.INSTANCE, refreshExecutor)
+            boolean fastRefresh = IndexSettings.INDEX_FAST_REFRESH_SETTING.get(
+                clusterService.state().metadata().index(indexShardRoutingTable.shardId().getIndex()).getSettings()
             );
+
+            // Indices marked with fast refresh do not rely on refreshing the unpromotables
+            if (fastRefresh) {
+                listener.onResponse(null);
+            } else {
+                UnpromotableShardRefreshRequest unpromotableReplicaRequest = new UnpromotableShardRefreshRequest(
+                    indexShardRoutingTable,
+                    replicaRequest.primaryRefreshResult.primaryTerm(),
+                    replicaRequest.primaryRefreshResult.generation(),
+                    false
+                );
+                transportService.sendRequest(
+                    transportService.getLocalNode(),
+                    TransportUnpromotableShardRefreshAction.NAME,
+                    unpromotableReplicaRequest,
+                    new ActionListenerResponseHandler<>(listener.safeMap(r -> null), in -> ActionResponse.Empty.INSTANCE, refreshExecutor)
+                );
+            }
         }
     }
 }

server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java

@@ -24,9 +24,6 @@
 
 import java.util.List;
 
-import static org.elasticsearch.TransportVersions.FAST_REFRESH_RCO;
-import static org.elasticsearch.index.IndexSettings.INDEX_FAST_REFRESH_SETTING;
-
 public class TransportUnpromotableShardRefreshAction extends TransportBroadcastUnpromotableAction<
     UnpromotableShardRefreshRequest,
     ActionResponse.Empty> {
@@ -76,18 +73,6 @@ protected void unpromotableShardOperation(
             return;
         }
 
-        // During an upgrade to FAST_REFRESH_RCO, we expect search shards to be first upgraded before the primary is upgraded. Thus,
-        // when the primary is upgraded, and starts to deliver unpromotable refreshes, we expect the search shards to be upgraded already.
-        // Note that the fast refresh setting is final.
-        // TODO: remove assertion (ES-9563)
-        assert INDEX_FAST_REFRESH_SETTING.get(shard.indexSettings().getSettings()) == false
-            || transportService.getLocalNodeConnection().getTransportVersion().onOrAfter(FAST_REFRESH_RCO)
-            : "attempted to refresh a fast refresh search shard "
-                + shard
-                + " on transport version "
-                + transportService.getLocalNodeConnection().getTransportVersion()
-                + " (before FAST_REFRESH_RCO)";
-
         ActionListener.run(responseListener, listener -> {
             shard.waitForPrimaryTermAndGeneration(
                 request.getPrimaryTerm(),

server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java

@@ -126,10 +126,12 @@ protected void asyncShardOperation(GetRequest request, ShardId shardId, ActionLi
         IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
         IndexShard indexShard = indexService.getShard(shardId.id());
         if (indexShard.routingEntry().isPromotableToPrimary() == false) {
+            // TODO: Re-evaluate assertion (ES-8227)
+            // assert indexShard.indexSettings().isFastRefresh() == false
+            // : "a search shard should not receive a TransportGetAction for an index with fast refresh";
             handleGetOnUnpromotableShard(request, indexShard, listener);
             return;
         }
-        // TODO: adapt assertion to assert only that it is not stateless (ES-9563)
         assert DiscoveryNode.isStateless(clusterService.getSettings()) == false || indexShard.indexSettings().isFastRefresh()
             : "in Stateless a promotable to primary shard can receive a TransportGetAction only if an index has the fast refresh setting";
         if (request.realtime()) { // we are not tied to a refresh cycle here anyway

server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java

@@ -124,10 +124,12 @@ protected void asyncShardOperation(MultiGetShardRequest request, ShardId shardId
         IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
         IndexShard indexShard = indexService.getShard(shardId.id());
         if (indexShard.routingEntry().isPromotableToPrimary() == false) {
+            // TODO: Re-evaluate assertion (ES-8227)
+            // assert indexShard.indexSettings().isFastRefresh() == false
+            // : "a search shard should not receive a TransportShardMultiGetAction for an index with fast refresh";
             handleMultiGetOnUnpromotableShard(request, indexShard, listener);
             return;
         }
-        // TODO: adapt assertion to assert only that it is not stateless (ES-9563)
         assert DiscoveryNode.isStateless(clusterService.getSettings()) == false || indexShard.indexSettings().isFastRefresh()
             : "in Stateless a promotable to primary shard can receive a TransportShardMultiGetAction only if an index has "
                 + "the fast refresh setting";

server/src/main/java/org/elasticsearch/action/search/SearchRequest.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.Rewriteable;
@@ -254,8 +255,10 @@ public SearchRequest(StreamInput in) throws IOException {
             finalReduce = true;
         }
         ccsMinimizeRoundtrips = in.readBoolean();
-        if (in.getTransportVersion().before(TransportVersions.REMOVE_MIN_COMPATIBLE_SHARD_NODE) && in.readBoolean()) {
-            Version.readVersion(in); // and drop on the floor
+        if ((in.getTransportVersion().before(TransportVersions.REMOVE_MIN_COMPATIBLE_SHARD_NODE)
+            || in.getTransportVersion().onOrAfter(TransportVersions.REVERT_REMOVE_MIN_COMPATIBLE_SHARD_NODE)) && in.readBoolean()) {
+            @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)  // this can be removed (again) when the v9 transport version can diverge
+            Version v = Version.readVersion(in); // and drop on the floor
         }
         waitForCheckpoints = in.readMap(StreamInput::readLongArray);
         waitForCheckpointsTimeout = in.readTimeValue();
@@ -291,7 +294,8 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(finalReduce);
         }
         out.writeBoolean(ccsMinimizeRoundtrips);
-        if (out.getTransportVersion().before(TransportVersions.REMOVE_MIN_COMPATIBLE_SHARD_NODE)) {
+        if (out.getTransportVersion().before(TransportVersions.REMOVE_MIN_COMPATIBLE_SHARD_NODE)
+            || out.getTransportVersion().onOrAfter(TransportVersions.REVERT_REMOVE_MIN_COMPATIBLE_SHARD_NODE)) {
             out.writeBoolean(false);
         }
         out.writeMap(waitForCheckpoints, StreamOutput::writeLongArray);

server/src/main/java/org/elasticsearch/action/support/replication/PostWriteRefresh.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.translog.Translog;
@@ -52,7 +53,9 @@ public void refreshShard(
             case WAIT_UNTIL -> waitUntil(indexShard, location, new ActionListener<>() {
                 @Override
                 public void onResponse(Boolean forced) {
-                    if (location != null && indexShard.routingEntry().isSearchable() == false) {
+                    // Fast refresh indices do not depend on the unpromotables being refreshed
+                    boolean fastRefresh = IndexSettings.INDEX_FAST_REFRESH_SETTING.get(indexShard.indexSettings().getSettings());
+                    if (location != null && (indexShard.routingEntry().isSearchable() == false && fastRefresh == false)) {
                         refreshUnpromotables(indexShard, location, listener, forced, postWriteRefreshTimeout);
                     } else {
                         listener.onResponse(forced);
@@ -65,7 +68,9 @@ public void onFailure(Exception e) {
                 }
             });
             case IMMEDIATE -> immediate(indexShard, listener.delegateFailureAndWrap((l, r) -> {
-                if (indexShard.getReplicationGroup().getRoutingTable().unpromotableShards().size() > 0) {
+                // Fast refresh indices do not depend on the unpromotables being refreshed
+                boolean fastRefresh = IndexSettings.INDEX_FAST_REFRESH_SETTING.get(indexShard.indexSettings().getSettings());
+                if (indexShard.getReplicationGroup().getRoutingTable().unpromotableShards().size() > 0 && fastRefresh == false) {
                     sendUnpromotableRequests(indexShard, r.generation(), true, l, postWriteRefreshTimeout);
                 } else {
                     l.onResponse(true);

server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java

@@ -32,7 +32,6 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.TransportVersions.FAST_REFRESH_RCO;
 import static org.elasticsearch.index.IndexSettings.INDEX_FAST_REFRESH_SETTING;
 
 public class OperationRouting {
@@ -306,14 +305,8 @@ public ShardId shardId(ClusterState clusterState, String index, String id, @Null
     }
 
     public static boolean canSearchShard(ShardRouting shardRouting, ClusterState clusterState) {
-        // TODO: remove if and always return isSearchable (ES-9563)
         if (INDEX_FAST_REFRESH_SETTING.get(clusterState.metadata().index(shardRouting.index()).getSettings())) {
-            // Until all the cluster is upgraded, we send searches/gets to the primary (even if it has been upgraded) to execute locally.
-            if (clusterState.getMinTransportVersion().onOrAfter(FAST_REFRESH_RCO)) {
-                return shardRouting.isSearchable();
-            } else {
-                return shardRouting.isPromotableToPrimary();
-            }
+            return shardRouting.isPromotableToPrimary();
         } else {
             return shardRouting.isSearchable();
         }