Merge remote-tracking branch 'upstream/main' into mapping_context_bit…

…setproducer

docs/reference/connector/apis/connector-apis.asciidoc

@@ -40,6 +40,8 @@ beta:[]
 beta:[]
 * <<update-connector-configuration-api>>
 beta:[]
+* <<update-connector-features-api>>
+beta:[]
 * <<update-connector-filtering-api>>
 beta:[]
 * <<update-connector-index-name-api>>
@@ -120,6 +122,7 @@ include::list-connectors-api.asciidoc[]
 include::update-connector-api-key-id-api.asciidoc[]
 include::update-connector-configuration-api.asciidoc[]
 include::update-connector-index-name-api.asciidoc[]
+include::update-connector-features-api.asciidoc[]
 include::update-connector-filtering-api.asciidoc[]
 include::update-connector-name-description-api.asciidoc[]
 include::update-connector-pipeline-api.asciidoc[]

docs/reference/connector/apis/update-connector-features-api.asciidoc

@@ -0,0 +1,138 @@
+[[update-connector-features-api]]
+=== Update connector features API
+++++
+<titleabbrev>Update connector features</titleabbrev>
+++++
+
+beta::[]
+
+Manages the `features` of a connector. This endpoint can be used to control the following aspects of a connector:
+
+* document-level security
+* incremental syncs
+* advanced sync rules
+* basic sync rules
+
+Normally, the running connector service automatically manages these features. However, you can use this API to override the default behavior.
+
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
+[[update-connector-features-api-request]]
+==== {api-request-title}
+
+`PUT _connector/<connector_id>/_features`
+
+[[update-connector-features-api-prereq]]
+==== {api-prereq-title}
+
+* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors.
+* The `connector_id` parameter should reference an existing connector.
+
+[[update-connector-features-api-path-params]]
+==== {api-path-parms-title}
+
+`<connector_id>`::
+(Required, string)
+
+[role="child_attributes"]
+[[update-connector-features-api-request-body]]
+==== {api-request-body-title}
+
+`features`::
+(Required, object) An object containing connector features.
+
+* `document_level_security` (Optional, object) Controls whether document-level security is enabled with the `enabled` flag.
+* `incremental_sync` (Optional, object) Controls whether incremental syncs are enabled with the `enabled` flag.
+* `native_connector_api_keys`(Optional, object) Controls whether native connector API keys are enabled with the `enabled` flag.
+* `sync_rules` (Optional, object) Controls sync rules.
+**  `advanced` (Optional, object) Controls whether advanced sync rules are enabled with the `enabled` flag.
+**  `basic`(Optional, object) Controls whether basic sync rules are enabled with the `enabled` flag.
+
+
+
+[[update-connector-features-api-response-codes]]
+==== {api-response-codes-title}
+
+`200`::
+Connector `features` was successfully updated.
+
+`400`::
+The `connector_id` was not provided or the request payload was malformed.
+
+`404` (Missing resources)::
+No connector matching `connector_id` could be found.
+
+[[update-connector-features-api-example]]
+==== {api-examples-title}
+
+The following example updates the `features` field for the connector with ID `my-connector`:
+
+////
+[source, console]
+--------------------------------------------------
+PUT _connector/my-connector
+{
+  "index_name": "search-google-drive",
+  "name": "My Connector",
+  "service_type": "google_drive"
+}
+--------------------------------------------------
+// TESTSETUP
+
+[source,console]
+--------------------------------------------------
+DELETE _connector/my-connector
+--------------------------------------------------
+// TEARDOWN
+////
+
+[source,console]
+----
+PUT _connector/my-connector/_features
+{
+  "features": {
+    "document_level_security": {
+      "enabled": true
+    },
+    "incremental_sync": {
+      "enabled": true
+    },
+    "sync_rules": {
+      "advanced": {
+        "enabled": false
+      },
+      "basic": {
+        "enabled": true
+      }
+    }
+  }
+}
+----
+
+[source,console-result]
+----
+{
+    "result": "updated"
+}
+----
+
+The endpoint supports partial updates of the `features` field. For example, to update only the `document_level_security` feature, you can send the following request:
+
+[source,console]
+----
+PUT _connector/my-connector/_features
+{
+  "features": {
+    "document_level_security": {
+      "enabled": true
+    }
+  }
+}
+----
+
+[source,console-result]
+----
+{
+    "result": "updated"
+}
+----

modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml

@@ -294,11 +294,11 @@ setup:
 ---
 "Configure with no synthetic source":
   - requires:
-      cluster_features: ["gte_v8.9.0"]
-      reason: "Error message fix in 8.9"
+      cluster_features: ["gte_v8.15.0"]
+      reason: "Error message changed in 8.15.0"
 
   - do:
-      catch: '/Time series indices only support synthetic source./'
+      catch: '/Indices with with index mode \[time_series\] only support synthetic source/'
       indices.create:
         index: tsdb_error
         body:

muted-tests.yml

@@ -54,6 +54,12 @@ tests:
 - class: "org.elasticsearch.upgrades.AggregationsIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109322"
   method: "testTerms"
+- class: "org.elasticsearch.xpack.ml.integration.ClassificationHousePricingIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/101598"
+  method: "testFeatureImportanceValues"
+- class: "org.elasticsearch.client.RestClientSingleHostIntegTests"
+  issue: "https://github.com/elastic/elasticsearch/issues/102717"
+  method: "testRequestResetAndAbort"
 
 # Examples:
 #

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml

@@ -77,6 +77,10 @@ create logs index:
   - is_true: test
   - match: { test.settings.index.mode: "logs" }
 
+  - do:
+      indices.get_mapping:
+        index: test
+  - match: { test.mappings._source.mode: synthetic }
 
 ---
 using default timestamp field mapping:

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java

@@ -2267,8 +2267,9 @@ IndexMetadata build(boolean repair) {
             }
 
             final boolean isSearchableSnapshot = SearchableSnapshotsSettings.isSearchableSnapshotStore(settings);
-            final String indexMode = settings.get(IndexSettings.MODE.getKey());
-            final boolean isTsdb = indexMode != null && IndexMode.TIME_SERIES.getName().equals(indexMode.toLowerCase(Locale.ROOT));
+            String indexModeString = settings.get(IndexSettings.MODE.getKey());
+            final IndexMode indexMode = indexModeString != null ? IndexMode.fromString(indexModeString.toLowerCase(Locale.ROOT)) : null;
+            final boolean isTsdb = indexMode == IndexMode.TIME_SERIES;
             return new IndexMetadata(
                 new Index(index, uuid),
                 version,
@@ -2308,7 +2309,7 @@ IndexMetadata build(boolean repair) {
                 AutoExpandReplicas.SETTING.get(settings),
                 isSearchableSnapshot,
                 isSearchableSnapshot && settings.getAsBoolean(SEARCHABLE_SNAPSHOT_PARTIAL_SETTING_KEY, false),
-                isTsdb ? IndexMode.TIME_SERIES : null,
+                indexMode,
                 isTsdb ? IndexSettings.TIME_SERIES_START_TIME.get(settings) : null,
                 isTsdb ? IndexSettings.TIME_SERIES_END_TIME.get(settings) : null,
                 SETTING_INDEX_VERSION_COMPATIBILITY.get(settings),

server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java

@@ -69,6 +69,14 @@ private enum Mode {
         IndexMode.TIME_SERIES
     );
 
+    private static final SourceFieldMapper LOGS_DEFAULT = new SourceFieldMapper(
+        Mode.SYNTHETIC,
+        Explicit.IMPLICIT_TRUE,
+        Strings.EMPTY_ARRAY,
+        Strings.EMPTY_ARRAY,
+        IndexMode.LOGS
+    );
+
     /*
      * Synthetic source was added as the default for TSDB in v.8.7. The legacy field mapper below
      * is used in bwc tests and mixed clusters containing time series indexes created in an earlier version.
@@ -156,7 +164,8 @@ protected Parameter<?>[] getParameters() {
 
         private boolean isDefault() {
             Mode m = mode.get();
-            if (m != null && (((indexMode == IndexMode.TIME_SERIES && m == Mode.SYNTHETIC) == false) || m == Mode.DISABLED)) {
+            if (m != null
+                && (((indexMode != null && indexMode.isSyntheticSourceEnabled() && m == Mode.SYNTHETIC) == false) || m == Mode.DISABLED)) {
                 return false;
             }
             return enabled.get().value() && includes.getValue().isEmpty() && excludes.getValue().isEmpty();
@@ -165,15 +174,19 @@ private boolean isDefault() {
         @Override
         public SourceFieldMapper build() {
             if (enabled.getValue().explicit()) {
-                if (indexMode == IndexMode.TIME_SERIES) {
-                    throw new MapperParsingException("Time series indices only support synthetic source");
+                if (indexMode != null && indexMode.isSyntheticSourceEnabled()) {
+                    throw new MapperParsingException("Indices with with index mode [" + indexMode + "] only support synthetic source");
                 }
                 if (mode.get() != null) {
                     throw new MapperParsingException("Cannot set both [mode] and [enabled] parameters");
                 }
             }
             if (isDefault()) {
-                return indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT : DEFAULT;
+                return switch (indexMode) {
+                    case TIME_SERIES -> TSDB_DEFAULT;
+                    case LOGS -> LOGS_DEFAULT;
+                    default -> DEFAULT;
+                };
             }
             if (supportsNonDefaultParameterValues == false) {
                 List<String> disallowed = new ArrayList<>();
@@ -212,10 +225,21 @@ public SourceFieldMapper build() {
 
     }
 
-    public static final TypeParser PARSER = new ConfigurableTypeParser(
-        c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES
-            ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0) ? TSDB_DEFAULT : TSDB_LEGACY_DEFAULT
-            : DEFAULT,
+    public static final TypeParser PARSER = new ConfigurableTypeParser(c -> {
+        var indexMode = c.getIndexSettings().getMode();
+        if (indexMode.isSyntheticSourceEnabled()) {
+            if (indexMode == IndexMode.TIME_SERIES) {
+                if (c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0)) {
+                    return TSDB_DEFAULT;
+                } else {
+                    return TSDB_LEGACY_DEFAULT;
+                }
+            } else if (indexMode == IndexMode.LOGS) {
+                return LOGS_DEFAULT;
+            }
+        }
+        return DEFAULT;
+    },
         c -> new Builder(
             c.getIndexSettings().getMode(),
             c.getSettings(),
@@ -323,6 +347,7 @@ public void preParse(DocumentParserContext context) throws IOException {
         final BytesReference adaptedSource = applyFilters(originalSource, contentType);
 
         if (adaptedSource != null) {
+            assert indexMode == null || indexMode.isSyntheticSourceEnabled() == false;
             final BytesRef ref = adaptedSource.toBytesRef();
             context.doc().add(new StoredField(fieldType().name(), ref.bytes, ref.offset, ref.length));
         }

server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java

@@ -10,12 +10,13 @@
 
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.mapper.MapperServiceTestCase;
-import org.hamcrest.Matchers;
+import org.elasticsearch.test.ESTestCase;
 
-public class LogsIndexModeTests extends MapperServiceTestCase {
+import static org.hamcrest.Matchers.equalTo;
+
+public class LogsIndexModeTests extends ESTestCase {
     public void testLogsIndexModeSetting() {
-        assertThat(IndexSettings.MODE.get(buildSettings()), Matchers.equalTo(IndexMode.LOGS));
+        assertThat(IndexSettings.MODE.get(buildSettings()), equalTo(IndexMode.LOGS));
     }
 
     public void testSortField() {
@@ -24,8 +25,10 @@ public void testSortField() {
             .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "agent_id")
             .build();
         final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings);
+        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGS));
         final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        assertThat("agent_id", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat(settings.getMode(), equalTo(IndexMode.LOGS));
+        assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
     }
 
     public void testSortMode() {
@@ -35,9 +38,11 @@ public void testSortMode() {
             .put(IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey(), "max")
             .build();
         final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings);
+        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGS));
         final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        assertThat("agent_id", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
-        assertThat("max", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey())));
+        assertThat(settings.getMode(), equalTo(IndexMode.LOGS));
+        assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat("max", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey())));
     }
 
     public void testSortOrder() {
@@ -47,9 +52,11 @@ public void testSortOrder() {
             .put(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), "desc")
             .build();
         final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings);
+        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGS));
         final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        assertThat("agent_id", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
-        assertThat("desc", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey())));
+        assertThat(settings.getMode(), equalTo(IndexMode.LOGS));
+        assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat("desc", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey())));
     }
 
     public void testSortMissing() {
@@ -59,9 +66,11 @@ public void testSortMissing() {
             .put(IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey(), "_last")
             .build();
         final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings);
+        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGS));
         final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        assertThat("agent_id", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
-        assertThat("_last", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey())));
+        assertThat(settings.getMode(), equalTo(IndexMode.LOGS));
+        assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat("_last", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey())));
     }
 
     private Settings buildSettings() {