Merge branch 'main' into remove-scale-from-0-flag

davidkyle · Nov 18, 2024 · 9dc2ac4 · 9dc2ac4
2 parents b7d6839 + d6cc86a
commit 9dc2ac4
Show file tree

Hide file tree

Showing 76 changed files with 118 additions and 849 deletions.
diff --git a/build-tools-internal/src/main/groovy/elasticsearch.fips.gradle b/build-tools-internal/src/main/groovy/elasticsearch.fips.gradle
@@ -20,7 +20,7 @@ import org.elasticsearch.gradle.testclusters.TestDistribution
 // Common config when running with a FIPS-140 runtime JVM
 if (buildParams.inFipsJvm) {
   allprojects {
-    String javaSecurityFilename = buildParams.runtimeJavaDetails.toLowerCase().contains('oracle') ? 'fips_java_oracle.security' : 'fips_java.security'
+    String javaSecurityFilename = buildParams.runtimeJavaDetails.get().toLowerCase().contains('oracle') ? 'fips_java_oracle.security' : 'fips_java.security'
     File fipsResourcesDir = new File(project.buildDir, 'fips-resources')
     File fipsSecurity = new File(fipsResourcesDir, javaSecurityFilename)
     File fipsPolicy = new File(fipsResourcesDir, 'fips_java.policy')

diff --git a/docs/changelog/116915.yaml b/docs/changelog/116915.yaml
@@ -0,0 +1,5 @@
+pr: 116915
+summary: Improve message about insecure S3 settings
+area: Snapshot/Restore
+type: enhancement
+issues: []
diff --git a/docs/changelog/116931.yaml b/docs/changelog/116931.yaml
@@ -0,0 +1,5 @@
+pr: 116931
+summary: Enable built-in Inference Endpoints and default for Semantic Text
+area: "Machine Learning"
+type: enhancement
+issues: []
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java
@@ -9,30 +9,22 @@
 
 package org.elasticsearch.datastreams;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.rollover.LazyRolloverAction;
 import org.elasticsearch.action.datastreams.autosharding.DataStreamAutoShardingService;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
 import org.elasticsearch.datastreams.lifecycle.health.DataStreamLifecycleHealthInfoPublisher;
 import org.elasticsearch.features.FeatureSpecification;
 import org.elasticsearch.features.NodeFeature;
 
-import java.util.Map;
 import java.util.Set;
 
 /**
  * Provides the features for data streams that this version of the code supports
  */
 public class DataStreamFeatures implements FeatureSpecification {
 
-    public static final NodeFeature DATA_STREAM_LIFECYCLE = new NodeFeature("data_stream.lifecycle");
     public static final NodeFeature DATA_STREAM_FAILURE_STORE_TSDB_FIX = new NodeFeature("data_stream.failure_store.tsdb_fix");
 
-    @Override
-    public Map<NodeFeature, Version> getHistoricalFeatures() {
-        return Map.of(DATA_STREAM_LIFECYCLE, Version.V_8_11_0);
-    }
-
     @Override
     public Set<NodeFeature> getFeatures() {
         return Set.of(

diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
@@ -318,8 +318,7 @@ class S3Repository extends MeteredBlobStoreRepository {
             deprecationLogger.critical(
                 DeprecationCategory.SECURITY,
                 "s3_repository_secret_settings",
-                "Using s3 access/secret key from repository settings. Instead "
-                    + "store these in named clients and the elasticsearch keystore for secure settings."
+                INSECURE_CREDENTIALS_DEPRECATION_WARNING
             );
         }
 
@@ -336,6 +335,11 @@ class S3Repository extends MeteredBlobStoreRepository {
         );
     }
 
+    static final String INSECURE_CREDENTIALS_DEPRECATION_WARNING = Strings.format("""
+        This repository's settings include a S3 access key and secret key, but repository settings are stored in plaintext and must not be \
+        used for security-sensitive information. Instead, store all secure settings in the keystore. See [%s] for more information.\
+        """, ReferenceDocs.SECURE_SETTINGS);
+
     private static Map<String, String> buildLocation(RepositoryMetadata metadata) {
         return Map.of("base_path", BASE_PATH_SETTING.get(metadata.settings()), "bucket", BUCKET_SETTING.get(metadata.settings()));
     }

diff --git a/...sitory-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java b/...sitory-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java
@@ -107,10 +107,9 @@ public void testRepositoryCredentialsOverrideSecureCredentials() {
         assertThat(credentials.getAWSSecretKey(), is("insecure_aws_secret"));
 
         assertCriticalWarnings(
+            "[access_key] setting was deprecated in Elasticsearch and will be removed in a future release.",
             "[secret_key] setting was deprecated in Elasticsearch and will be removed in a future release.",
-            "Using s3 access/secret key from repository settings. Instead store these in named clients and"
-                + " the elasticsearch keystore for secure settings.",
-            "[access_key] setting was deprecated in Elasticsearch and will be removed in a future release."
+            S3Repository.INSECURE_CREDENTIALS_DEPRECATION_WARNING
         );
     }
 
@@ -194,10 +193,9 @@ public void testReinitSecureCredentials() {
 
         if (hasInsecureSettings) {
             assertCriticalWarnings(
+                "[access_key] setting was deprecated in Elasticsearch and will be removed in a future release.",
                 "[secret_key] setting was deprecated in Elasticsearch and will be removed in a future release.",
-                "Using s3 access/secret key from repository settings. Instead store these in named clients and"
-                    + " the elasticsearch keystore for secure settings.",
-                "[access_key] setting was deprecated in Elasticsearch and will be removed in a future release."
+                S3Repository.INSECURE_CREDENTIALS_DEPRECATION_WARNING
             );
         }
     }
@@ -238,10 +236,7 @@ public void sendResponse(RestResponse response) {
             throw error.get();
         }
 
-        assertWarnings(
-            "Using s3 access/secret key from repository settings. Instead store these in named clients and"
-                + " the elasticsearch keystore for secure settings."
-        );
+        assertWarnings(S3Repository.INSECURE_CREDENTIALS_DEPRECATION_WARNING);
     }
 
     private void createRepository(final String name, final Settings repositorySettings) {

diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/HealthNodeUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/HealthNodeUpgradeIT.java
diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java
@@ -426,7 +426,6 @@
             org.elasticsearch.cluster.service.TransportFeatures,
             org.elasticsearch.cluster.metadata.MetadataFeatures,
             org.elasticsearch.rest.RestFeatures,
-            org.elasticsearch.indices.IndicesFeatures,
             org.elasticsearch.repositories.RepositoriesFeatures,
             org.elasticsearch.action.admin.cluster.allocation.AllocationStatsFeatures,
             org.elasticsearch.rest.action.admin.cluster.ClusterRerouteFeatures,

diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -853,7 +853,7 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster, Predicate<
         registerHandler.accept(new RestClusterStateAction(settingsFilter, threadPool));
         registerHandler.accept(new RestClusterHealthAction());
         registerHandler.accept(new RestClusterUpdateSettingsAction());
-        registerHandler.accept(new RestClusterGetSettingsAction(settings, clusterSettings, settingsFilter, clusterSupportsFeature));
+        registerHandler.accept(new RestClusterGetSettingsAction(settings, clusterSettings, settingsFilter));
         registerHandler.accept(new RestClusterRerouteAction(settingsFilter));
         registerHandler.accept(new RestClusterSearchShardsAction());
         registerHandler.accept(new RestPendingClusterTasksAction());

diff --git a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java
@@ -82,6 +82,7 @@ public enum ReferenceDocs {
     CIRCUIT_BREAKER_ERRORS,
     ALLOCATION_EXPLAIN_NO_COPIES,
     ALLOCATION_EXPLAIN_MAX_RETRY,
+    SECURE_SETTINGS,
     // this comment keeps the ';' on the next line so every entry above has a trailing ',' which makes the diff for adding new links cleaner
     ;
 

diff --git a/server/src/main/java/org/elasticsearch/common/settings/SecureSetting.java b/server/src/main/java/org/elasticsearch/common/settings/SecureSetting.java
@@ -185,9 +185,7 @@ private InsecureStringSetting(String name) {
         @Override
         public SecureString get(Settings settings) {
             if (ALLOW_INSECURE_SETTINGS == false && exists(settings)) {
-                throw new IllegalArgumentException(
-                    "Setting [" + name + "] is insecure, " + "but property [allow_insecure_settings] is not set"
-                );
+                throw new IllegalArgumentException("Setting [" + name + "] is insecure, use the elasticsearch keystore instead");
             }
             return super.get(settings);
         }

diff --git a/server/src/main/java/org/elasticsearch/health/HealthFeatures.java b/server/src/main/java/org/elasticsearch/health/HealthFeatures.java
@@ -9,34 +9,17 @@
 
 package org.elasticsearch.health;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.features.FeatureSpecification;
 import org.elasticsearch.features.NodeFeature;
 
-import java.util.Map;
 import java.util.Set;
 
 public class HealthFeatures implements FeatureSpecification {
 
-    public static final NodeFeature SUPPORTS_HEALTH = new NodeFeature("health.supports_health");
-    public static final NodeFeature SUPPORTS_HEALTH_REPORT_API = new NodeFeature("health.supports_health_report_api");
-    public static final NodeFeature SUPPORTS_SHARDS_CAPACITY_INDICATOR = new NodeFeature("health.shards_capacity_indicator");
     public static final NodeFeature SUPPORTS_EXTENDED_REPOSITORY_INDICATOR = new NodeFeature("health.extended_repository_indicator");
 
     @Override
     public Set<NodeFeature> getFeatures() {
         return Set.of(SUPPORTS_EXTENDED_REPOSITORY_INDICATOR);
     }
-
-    @Override
-    public Map<NodeFeature, Version> getHistoricalFeatures() {
-        return Map.of(
-            SUPPORTS_HEALTH,
-            Version.V_8_5_0,                // health accessible via /_internal/_health
-            SUPPORTS_HEALTH_REPORT_API,
-            Version.V_8_7_0,                // health accessible via /_health_report
-            SUPPORTS_SHARDS_CAPACITY_INDICATOR,
-            Version.V_8_8_0
-        );
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/health/metadata/HealthMetadataService.java b/server/src/main/java/org/elasticsearch/health/metadata/HealthMetadataService.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.gateway.GatewayService;
-import org.elasticsearch.health.HealthFeatures;
 
 import java.util.List;
 import java.util.stream.Stream;
@@ -137,7 +136,7 @@ private void updateOnHealthNodeEnabledChange(boolean enabled) {
 
     private boolean canPostClusterStateUpdates(ClusterState state) {
         // Wait until every node in the cluster supports health checks
-        return isMaster && state.clusterRecovered() && featureService.clusterHasFeature(state, HealthFeatures.SUPPORTS_HEALTH);
+        return isMaster && state.clusterRecovered();
     }
 
     private void updateOnClusterStateChange(ClusterChangedEvent event) {

diff --git a/server/src/main/java/org/elasticsearch/health/node/DiskHealthIndicatorService.java b/server/src/main/java/org/elasticsearch/health/node/DiskHealthIndicatorService.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.health.Diagnosis;
-import org.elasticsearch.health.HealthFeatures;
 import org.elasticsearch.health.HealthIndicatorDetails;
 import org.elasticsearch.health.HealthIndicatorImpact;
 import org.elasticsearch.health.HealthIndicatorResult;
@@ -91,15 +90,6 @@ public HealthIndicatorResult calculate(boolean verbose, int maxAffectedResources
         ClusterState clusterState = clusterService.state();
         Map<String, DiskHealthInfo> diskHealthInfoMap = healthInfo.diskInfoByNode();
         if (diskHealthInfoMap == null || diskHealthInfoMap.isEmpty()) {
-            if (featureService.clusterHasFeature(clusterState, HealthFeatures.SUPPORTS_HEALTH) == false) {
-                return createIndicator(
-                    HealthStatus.GREEN,
-                    "No disk usage data available. The cluster currently has mixed versions (an upgrade may be in progress).",
-                    HealthIndicatorDetails.EMPTY,
-                    List.of(),
-                    List.of()
-                );
-            }
             /*
              * If there is no disk health info, that either means that a new health node was just elected, or something is seriously
              * wrong with health data collection on the health node. Either way, we immediately return UNKNOWN. If there are at least

diff --git a/server/src/main/java/org/elasticsearch/health/node/LocalHealthMonitor.java b/server/src/main/java/org/elasticsearch/health/node/LocalHealthMonitor.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.common.util.concurrent.RunOnce;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.features.FeatureService;
-import org.elasticsearch.health.HealthFeatures;
 import org.elasticsearch.health.metadata.HealthMetadata;
 import org.elasticsearch.health.node.action.HealthNodeNotDiscoveredException;
 import org.elasticsearch.health.node.selection.HealthNode;
@@ -200,7 +199,6 @@ public void clusterChanged(ClusterChangedEvent event) {
             }
         }
         prerequisitesFulfilled = event.state().clusterRecovered()
-            && featureService.clusterHasFeature(event.state(), HealthFeatures.SUPPORTS_HEALTH)
             && HealthMetadata.getFromClusterState(event.state()) != null
             && currentHealthNode != null
             && currentMasterNode != null;

diff --git a/server/src/main/java/org/elasticsearch/health/node/ShardsCapacityHealthIndicatorService.java b/server/src/main/java/org/elasticsearch/health/node/ShardsCapacityHealthIndicatorService.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.health.Diagnosis;
-import org.elasticsearch.health.HealthFeatures;
 import org.elasticsearch.health.HealthIndicatorDetails;
 import org.elasticsearch.health.HealthIndicatorImpact;
 import org.elasticsearch.health.HealthIndicatorResult;
@@ -111,15 +110,6 @@ public HealthIndicatorResult calculate(boolean verbose, int maxAffectedResources
         var state = clusterService.state();
         var healthMetadata = HealthMetadata.getFromClusterState(state);
         if (healthMetadata == null || healthMetadata.getShardLimitsMetadata() == null) {
-            if (featureService.clusterHasFeature(state, HealthFeatures.SUPPORTS_SHARDS_CAPACITY_INDICATOR) == false) {
-                return createIndicator(
-                    HealthStatus.GREEN,
-                    "No shard limits configured yet. The cluster currently has mixed versions (an upgrade may be in progress).",
-                    HealthIndicatorDetails.EMPTY,
-                    List.of(),
-                    List.of()
-                );
-            }
             return unknownIndicator();
         }
 

diff --git a/server/src/main/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutor.java b/server/src/main/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutor.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.features.FeatureService;
-import org.elasticsearch.health.HealthFeatures;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.persistent.AllocatedPersistentTask;
 import org.elasticsearch.persistent.PersistentTaskParams;
@@ -157,11 +156,8 @@ public PersistentTasksCustomMetadata.Assignment getAssignment(
 
     // visible for testing
     void startTask(ClusterChangedEvent event) {
-        // Wait until every node in the cluster supports health checks
-        if (event.localNodeMaster()
-            && event.state().clusterRecovered()
-            && HealthNode.findTask(event.state()) == null
-            && featureService.clusterHasFeature(event.state(), HealthFeatures.SUPPORTS_HEALTH)) {
+        // Wait until master is stable before starting health task
+        if (event.localNodeMaster() && event.state().clusterRecovered() && HealthNode.findTask(event.state()) == null) {
             persistentTasksService.sendStartRequest(
                 TASK_NAME,
                 TASK_NAME,

diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesFeatures.java b/server/src/main/java/org/elasticsearch/indices/IndicesFeatures.java
diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.action.ResolvedIndices;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.TransportAutoPutMappingAction;
-import org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
@@ -79,7 +78,6 @@
 import org.elasticsearch.env.ShardLock;
 import org.elasticsearch.env.ShardLockObtainFailedException;
 import org.elasticsearch.features.FeatureService;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.gateway.MetaStateService;
 import org.elasticsearch.gateway.MetadataStateFormat;
 import org.elasticsearch.index.CloseUtils;
@@ -211,8 +209,6 @@ public class IndicesService extends AbstractLifecycleComponent
         Setting.Property.NodeScope
     );
 
-    static final NodeFeature SUPPORTS_AUTO_PUT = new NodeFeature("indices.auto_put_supported");
-
     /**
      * The node's settings.
      */
@@ -910,9 +906,7 @@ public void createShard(
                     .setConcreteIndex(shardRouting.index())
                     .source(mapping.source().string(), XContentType.JSON);
                 client.execute(
-                    featureService.clusterHasFeature(clusterService.state(), SUPPORTS_AUTO_PUT)
-                        ? TransportAutoPutMappingAction.TYPE
-                        : TransportPutMappingAction.TYPE,
+                    TransportAutoPutMappingAction.TYPE,
                     putMappingRequestAcknowledgedRequest.ackTimeout(TimeValue.MAX_VALUE).masterNodeTimeout(TimeValue.MAX_VALUE),
                     new RefCountAwareThreadedActionListener<>(threadPool.generic(), listener.map(ignored -> null))
                 );