From d543d91f0257faef0df36c03cf5ab5b8c5c4ca38 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Tue, 4 Jun 2024 11:58:37 +0200 Subject: [PATCH 1/2] [Connector API] Implement _features endpoint (#109248) --- .../api/connector.update_features.json | 38 +++++ .../170_connector_update_features.yml | 108 ++++++++++++++ .../xpack/application/EnterpriseSearch.java | 5 + .../application/connector/Connector.java | 2 +- .../connector/ConnectorFeatures.java | 16 +++ .../connector/ConnectorIndexService.java | 27 ++++ .../RestUpdateConnectorFeaturesAction.java | 48 +++++++ ...ransportUpdateConnectorFeaturesAction.java | 50 +++++++ .../action/UpdateConnectorFeaturesAction.java | 133 ++++++++++++++++++ .../connector/ConnectorIndexServiceTests.java | 73 ++++++++++ .../connector/ConnectorTestUtils.java | 2 +- ...turesActionRequestBWCSerializingTests.java | 51 +++++++ .../xpack/security/operator/Constants.java | 1 + 13 files changed, 552 insertions(+), 2 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_features.json create mode 100644 x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/170_connector_update_features.yml create mode 100644 x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorFeaturesAction.java create mode 100644 x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFeaturesAction.java create mode 100644 x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java create mode 100644 x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesActionRequestBWCSerializingTests.java diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_features.json b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_features.json new file mode 100644 index 0000000000000..b488e19262c2e --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_features.json @@ -0,0 +1,38 @@ +{ + "connector.update_features": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/update-connector-features-api.html", + "description": "Updates the connector features in the connector document." + }, + "stability": "experimental", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_connector/{connector_id}/_features", + "methods": [ + "PUT" + ], + "parts": { + "connector_id": { + "type": "string", + "description": "The unique identifier of the connector to be updated." + } + } + } + ] + }, + "body": { + "description": "An object containing the connector's features definition.", + "required": true + } + } +} diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/170_connector_update_features.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/170_connector_update_features.yml new file mode 100644 index 0000000000000..0964e4f50ebde --- /dev/null +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/170_connector_update_features.yml @@ -0,0 +1,108 @@ +setup: + - requires: + cluster_features: ["gte_v8.15.0"] + reason: Introduced in 8.15.0 + + - do: + connector.put: + connector_id: test-connector + body: + index_name: search-1-test + name: my-connector + language: pl + is_native: false + service_type: super-connector + +--- +"Update Connector Features": + - do: + connector.update_features: + connector_id: test-connector + body: + features: + document_level_security: { enabled: true } + native_connector_api_keys: { enabled: true } + incremental_sync: { enabled: false } + sync_rules: + basic: { enabled: true } + advanced: { enabled: false } + + + - match: { result: updated } + + - do: + connector.get: + connector_id: test-connector + + - match: { features.document_level_security.enabled: true } + - match: { features.native_connector_api_keys.enabled: true } + - match: { features.incremental_sync.enabled: false } + - match: { features.sync_rules.basic.enabled: true } + - match: { features.sync_rules.advanced.enabled: false } + +--- +"Update Connector Features - Partial Update": + - do: + connector.update_features: + connector_id: test-connector + body: + features: + document_level_security: { enabled: true } + + + - match: { result: updated } + + - do: + connector.get: + connector_id: test-connector + + - match: { features.document_level_security.enabled: true } + + + - do: + connector.update_features: + connector_id: test-connector + body: + features: + native_connector_api_keys: { enabled: true } + + + - match: { result: updated } + + - do: + connector.get: + connector_id: test-connector + + # Assert that existing feature remains unchanged + - match: { features.document_level_security.enabled: true } + - match: { features.native_connector_api_keys.enabled: true } + +--- +"Update Connector Features - 404 when connector doesn't exist": + - do: + catch: "missing" + connector.update_features: + connector_id: test-non-existent-connector + body: + features: + native_connector_api_keys: { enabled: true } + +--- +"Update Connector Features - 400 status code when connector_id is empty": + - do: + catch: "bad_request" + connector.update_features: + connector_id: "" + body: + features: + native_connector_api_keys: { enabled: true } + +--- +"Update Connector Features - 400 status code when payload unknown": + - do: + catch: "bad_request" + connector.update_features: + connector_id: test-connector + body: + featuresss: + not_a_feature: 12423 diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java index bc3da1a82fba4..871bf7fb122b9 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java @@ -58,6 +58,7 @@ import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorApiKeyIdAction; import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorConfigurationAction; import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorErrorAction; +import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorFeaturesAction; import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorFilteringAction; import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorFilteringValidationAction; import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorIndexNameAction; @@ -78,6 +79,7 @@ import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorApiKeyIdAction; import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorConfigurationAction; import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorErrorAction; +import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorFeaturesAction; import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorFilteringAction; import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorFilteringValidationAction; import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorIndexNameAction; @@ -93,6 +95,7 @@ import org.elasticsearch.xpack.application.connector.action.UpdateConnectorApiKeyIdAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorConfigurationAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorErrorAction; +import org.elasticsearch.xpack.application.connector.action.UpdateConnectorFeaturesAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorFilteringAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorFilteringValidationAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorIndexNameAction; @@ -267,6 +270,7 @@ protected XPackLicenseState getLicenseState() { new ActionHandler<>(UpdateConnectorApiKeyIdAction.INSTANCE, TransportUpdateConnectorApiKeyIdAction.class), new ActionHandler<>(UpdateConnectorConfigurationAction.INSTANCE, TransportUpdateConnectorConfigurationAction.class), new ActionHandler<>(UpdateConnectorErrorAction.INSTANCE, TransportUpdateConnectorErrorAction.class), + new ActionHandler<>(UpdateConnectorFeaturesAction.INSTANCE, TransportUpdateConnectorFeaturesAction.class), new ActionHandler<>(UpdateConnectorFilteringAction.INSTANCE, TransportUpdateConnectorFilteringAction.class), new ActionHandler<>(UpdateConnectorActiveFilteringAction.INSTANCE, TransportUpdateConnectorActiveFilteringAction.class), new ActionHandler<>( @@ -368,6 +372,7 @@ public List getRestHandlers( new RestUpdateConnectorConfigurationAction(), new RestUpdateConnectorErrorAction(), new RestUpdateConnectorActiveFilteringAction(), + new RestUpdateConnectorFeaturesAction(), new RestUpdateConnectorFilteringValidationAction(), new RestUpdateConnectorFilteringAction(), new RestUpdateConnectorIndexNameAction(), diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java index e9447149c7e6c..62f42d9a16ead 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java @@ -213,7 +213,7 @@ public Connector(StreamInput in) throws IOException { static final ParseField CUSTOM_SCHEDULING_FIELD = new ParseField("custom_scheduling"); public static final ParseField DESCRIPTION_FIELD = new ParseField("description"); public static final ParseField ERROR_FIELD = new ParseField("error"); - static final ParseField FEATURES_FIELD = new ParseField("features"); + public static final ParseField FEATURES_FIELD = new ParseField("features"); public static final ParseField FILTERING_FIELD = new ParseField("filtering"); public static final ParseField INDEX_NAME_FIELD = new ParseField("index_name"); public static final ParseField IS_NATIVE_FIELD = new ParseField("is_native"); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFeatures.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFeatures.java index 1b2e7209e41e5..0b9a72f06ad53 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFeatures.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFeatures.java @@ -102,6 +102,22 @@ public static ConnectorFeatures fromXContentBytes(BytesReference source, XConten } } + public FeatureEnabled getDocumentLevelSecurityEnabled() { + return documentLevelSecurityEnabled; + } + + public FeatureEnabled getIncrementalSyncEnabled() { + return incrementalSyncEnabled; + } + + public FeatureEnabled getNativeConnectorAPIKeysEnabled() { + return nativeConnectorAPIKeysEnabled; + } + + public SyncRulesFeatures getSyncRulesFeatures() { + return syncRulesFeatures; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index 50e2633bb8c76..e5314a20bdccf 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -549,6 +549,33 @@ public void updateConnectorFiltering(String connectorId, List listener) { + try { + final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc( + new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) + .id(connectorId) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(Map.of(Connector.FEATURES_FIELD.getPreferredName(), features)) + ); + client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + })); + } catch (Exception e) { + listener.onFailure(e); + } + } + /** * Updates the draft filtering in a given {@link Connector}. * diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorFeaturesAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorFeaturesAction.java new file mode 100644 index 0000000000000..48bf87b114548 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorFeaturesAction.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.application.EnterpriseSearch; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.PUT; + +@ServerlessScope(Scope.PUBLIC) +public class RestUpdateConnectorFeaturesAction extends BaseRestHandler { + + @Override + public String getName() { + return "connector_update_features_action"; + } + + @Override + public List routes() { + return List.of(new Route(PUT, "/" + EnterpriseSearch.CONNECTOR_API_ENDPOINT + "/{connector_id}/_features")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { + UpdateConnectorFeaturesAction.Request request = UpdateConnectorFeaturesAction.Request.fromXContentBytes( + restRequest.param("connector_id"), + restRequest.content(), + restRequest.getXContentType() + ); + return channel -> client.execute( + UpdateConnectorFeaturesAction.INSTANCE, + request, + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) + ); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFeaturesAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFeaturesAction.java new file mode 100644 index 0000000000000..c86ddf902519f --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFeaturesAction.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.application.connector.ConnectorIndexService; + +public class TransportUpdateConnectorFeaturesAction extends HandledTransportAction< + UpdateConnectorFeaturesAction.Request, + ConnectorUpdateActionResponse> { + + protected final ConnectorIndexService connectorIndexService; + + @Inject + public TransportUpdateConnectorFeaturesAction(TransportService transportService, ActionFilters actionFilters, Client client) { + super( + UpdateConnectorFeaturesAction.NAME, + transportService, + actionFilters, + UpdateConnectorFeaturesAction.Request::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.connectorIndexService = new ConnectorIndexService(client); + } + + @Override + protected void doExecute( + Task task, + UpdateConnectorFeaturesAction.Request request, + ActionListener listener + ) { + connectorIndexService.updateConnectorFeatures( + request.getConnectorId(), + request.getFeatures(), + listener.map(r -> new ConnectorUpdateActionResponse(r.getResult())) + ); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java new file mode 100644 index 0000000000000..c1f62c0efe6e8 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.application.connector.Connector; +import org.elasticsearch.xpack.application.connector.ConnectorFeatures; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class UpdateConnectorFeaturesAction { + + public static final String NAME = "indices:data/write/xpack/connector/update_features"; + public static final ActionType INSTANCE = new ActionType<>(NAME); + + private UpdateConnectorFeaturesAction() {/* no instances */} + + public static class Request extends ConnectorActionRequest implements ToXContentObject { + + private final String connectorId; + + private final ConnectorFeatures features; + + public Request(String connectorId, ConnectorFeatures features) { + this.connectorId = connectorId; + this.features = features; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.connectorId = in.readString(); + this.features = in.readOptionalWriteable(ConnectorFeatures::new); + } + + public String getConnectorId() { + return connectorId; + } + + public ConnectorFeatures getFeatures() { + return features; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + + if (Strings.isNullOrEmpty(connectorId)) { + validationException = addValidationError("[connector_id] cannot be [null] or [\"\"].", validationException); + } + + return validationException; + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "connector_update_features_request", + false, + ((args, connectorId) -> new UpdateConnectorFeaturesAction.Request(connectorId, (ConnectorFeatures) args[0])) + ); + + static { + PARSER.declareObject(optionalConstructorArg(), (p, c) -> ConnectorFeatures.fromXContent(p), Connector.FEATURES_FIELD); + } + + public static UpdateConnectorFeaturesAction.Request fromXContentBytes( + String connectorId, + BytesReference source, + XContentType xContentType + ) { + try (XContentParser parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, source, xContentType)) { + return UpdateConnectorFeaturesAction.Request.fromXContent(parser, connectorId); + } catch (IOException e) { + throw new ElasticsearchParseException("Failed to parse: " + source.utf8ToString(), e); + } + } + + public static UpdateConnectorFeaturesAction.Request fromXContent(XContentParser parser, String connectorId) throws IOException { + return PARSER.parse(parser, connectorId); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field(Connector.FEATURES_FIELD.getPreferredName(), features); + } + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(connectorId); + out.writeOptionalWriteable(features); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(connectorId, request.connectorId) && Objects.equals(features, request.features); + } + + @Override + public int hashCode() { + return Objects.hash(connectorId, features); + } + + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java index 045cb725e477c..21a0fede4675e 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java @@ -56,7 +56,9 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.getRandomConnectorFeatures; import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.getRandomCronExpression; +import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.randomConnectorFeatureEnabled; import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.registerSimplifiedConnectorIndexTemplates; import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.equalTo; @@ -240,6 +242,51 @@ public void testUpdateConnectorPipeline() throws Exception { assertThat(updatedPipeline, equalTo(indexedConnector.getPipeline())); } + public void testUpdateConnectorFeatures() throws Exception { + Connector connector = ConnectorTestUtils.getRandomConnector(); + String connectorId = randomUUID(); + + ConnectorCreateActionResponse resp = awaitCreateConnector(connectorId, connector); + assertThat(resp.status(), anyOf(equalTo(RestStatus.CREATED), equalTo(RestStatus.OK))); + + ConnectorFeatures newFeatures = getRandomConnectorFeatures(); + + DocWriteResponse updateResponse = awaitUpdateConnectorFeatures(connectorId, newFeatures); + assertThat(updateResponse.status(), equalTo(RestStatus.OK)); + Connector indexedConnector = awaitGetConnector(connectorId); + assertThat(newFeatures, equalTo(indexedConnector.getFeatures())); + + } + + public void testUpdateConnectorFeatures_partialUpdate() throws Exception { + Connector connector = ConnectorTestUtils.getRandomConnector(); + String connectorId = randomUUID(); + + ConnectorCreateActionResponse resp = awaitCreateConnector(connectorId, connector); + assertThat(resp.status(), anyOf(equalTo(RestStatus.CREATED), equalTo(RestStatus.OK))); + + ConnectorFeatures features = getRandomConnectorFeatures(); + + awaitUpdateConnectorFeatures(connectorId, features); + + Connector indexedConnector = awaitGetConnector(connectorId); + assertThat(features, equalTo(indexedConnector.getFeatures())); + + // Partial update of DLS feature + ConnectorFeatures dlsFeature = new ConnectorFeatures.Builder().setDocumentLevelSecurityEnabled(randomConnectorFeatureEnabled()) + .build(); + awaitUpdateConnectorFeatures(connectorId, dlsFeature); + indexedConnector = awaitGetConnector(connectorId); + + // Assert that partial update was applied + assertThat(dlsFeature.getDocumentLevelSecurityEnabled(), equalTo(indexedConnector.getFeatures().getDocumentLevelSecurityEnabled())); + + // Assert other features are unchanged + assertThat(features.getSyncRulesFeatures(), equalTo(indexedConnector.getFeatures().getSyncRulesFeatures())); + assertThat(features.getNativeConnectorAPIKeysEnabled(), equalTo(indexedConnector.getFeatures().getNativeConnectorAPIKeysEnabled())); + assertThat(features.getIncrementalSyncEnabled(), equalTo(indexedConnector.getFeatures().getIncrementalSyncEnabled())); + } + public void testUpdateConnectorFiltering() throws Exception { Connector connector = ConnectorTestUtils.getRandomConnector(); String connectorId = randomUUID(); @@ -890,6 +937,32 @@ public void onFailure(Exception e) { return resp.get(); } + private UpdateResponse awaitUpdateConnectorFeatures(String connectorId, ConnectorFeatures features) throws Exception { + CountDownLatch latch = new CountDownLatch(1); + final AtomicReference resp = new AtomicReference<>(null); + final AtomicReference exc = new AtomicReference<>(null); + connectorIndexService.updateConnectorFeatures(connectorId, features, new ActionListener<>() { + @Override + public void onResponse(UpdateResponse indexResponse) { + resp.set(indexResponse); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + exc.set(e); + latch.countDown(); + } + }); + + assertTrue("Timeout waiting for update features request", latch.await(REQUEST_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + if (exc.get() != null) { + throw exc.get(); + } + assertNotNull("Received null response from update features request", resp.get()); + return resp.get(); + } + private UpdateResponse awaitUpdateConnectorFiltering(String connectorId, List filtering) throws Exception { CountDownLatch latch = new CountDownLatch(1); final AtomicReference resp = new AtomicReference<>(null); diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java index 230de44a8f6c5..f052ef79d82fb 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java @@ -371,7 +371,7 @@ public static ConnectorSearchResult getRandomConnectorSearchResult() { .build(); } - private static ConnectorFeatures.FeatureEnabled randomConnectorFeatureEnabled() { + public static ConnectorFeatures.FeatureEnabled randomConnectorFeatureEnabled() { return new ConnectorFeatures.FeatureEnabled(randomBoolean()); } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesActionRequestBWCSerializingTests.java new file mode 100644 index 0000000000000..9a191dba2e525 --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesActionRequestBWCSerializingTests.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.application.connector.ConnectorTestUtils; +import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; + +import java.io.IOException; + +public class UpdateConnectorFeaturesActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase< + UpdateConnectorFeaturesAction.Request> { + + private String connectorId; + + @Override + protected Writeable.Reader instanceReader() { + return UpdateConnectorFeaturesAction.Request::new; + } + + @Override + protected UpdateConnectorFeaturesAction.Request createTestInstance() { + this.connectorId = randomUUID(); + return new UpdateConnectorFeaturesAction.Request(connectorId, ConnectorTestUtils.getRandomConnectorFeatures()); + } + + @Override + protected UpdateConnectorFeaturesAction.Request mutateInstance(UpdateConnectorFeaturesAction.Request instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected UpdateConnectorFeaturesAction.Request doParseInstance(XContentParser parser) throws IOException { + return UpdateConnectorFeaturesAction.Request.fromXContent(parser, this.connectorId); + } + + @Override + protected UpdateConnectorFeaturesAction.Request mutateInstanceForVersion( + UpdateConnectorFeaturesAction.Request instance, + TransportVersion version + ) { + return instance; + } +} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index ae5af54f078dd..5561e14da980a 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -133,6 +133,7 @@ public class Constants { "indices:data/write/xpack/connector/update_api_key_id", "indices:data/write/xpack/connector/update_configuration", "indices:data/write/xpack/connector/update_error", + "indices:data/write/xpack/connector/update_features", "indices:data/write/xpack/connector/update_filtering", "indices:data/write/xpack/connector/update_filtering/activate", "indices:data/write/xpack/connector/update_filtering/draft_validation", From 8ac3e3dd909f3c5c04538e2ee08dfc7d5bc109c9 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Tue, 4 Jun 2024 12:43:02 +0200 Subject: [PATCH 2/2] Update Gradle wrapper to 8.8 (#108021) Fix incompatibility with 8.8 and our internal api usages - Update ospackage to a version that contains a fix we provided - Tweak build logic to avoid deprecation warnings - Use newer permission api - Use custom shadowplugin - Rework ElasticsearchDistribution dependencies resolution - Update Gradle wrapper to 8.8 --- build-conventions/build.gradle | 14 ++ build-tools-internal/build.gradle | 3 + .../gradle/wrapper/gradle-wrapper.properties | 4 +- build-tools-internal/settings.gradle | 12 +- ...nternalDistributionArchiveSetupPlugin.java | 4 +- .../internal/SymbolicLinkPreservingTar.java | 6 +- .../internal/info/GlobalBuildInfoPlugin.java | 2 +- .../DependencyLicensesPrecommitPlugin.java | 25 ++-- .../precommit/DependencyLicensesTask.java | 24 +++- .../ThirdPartyAuditPrecommitPlugin.java | 17 ++- .../internal/util/DependenciesUtils.java | 49 +++++++ .../gradle/internal/util/HdfsUtils.java | 55 -------- .../src/main/resources/minimumGradleVersion | 2 +- build-tools/build.gradle | 12 ++ build-tools/settings.gradle | 2 +- .../gradle/DistributionDownloadPlugin.java | 132 ++++++++++-------- .../gradle/ElasticsearchDistribution.java | 7 +- .../testclusters/ElasticsearchNode.java | 14 +- build.gradle | 14 +- distribution/archives/build.gradle | 26 +++- distribution/build.gradle | 24 +++- distribution/packages/build.gradle | 74 ++++++---- gradle/build.versions.toml | 2 +- gradle/verification-metadata.xml | 32 +++-- gradle/wrapper/gradle-wrapper.properties | 4 +- gradlew | 2 +- .../gradle/wrapper/gradle-wrapper.properties | 4 +- server/build.gradle | 12 +- settings.gradle | 3 + 29 files changed, 351 insertions(+), 230 deletions(-) create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java delete mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/HdfsUtils.java diff --git a/build-conventions/build.gradle b/build-conventions/build.gradle index cd9a548a9901f..94b0312d0d5d3 100644 --- a/build-conventions/build.gradle +++ b/build-conventions/build.gradle @@ -8,6 +8,16 @@ import org.gradle.plugins.ide.eclipse.model.SourceFolder + +buildscript { + repositories { + maven { + url 'https://jitpack.io' + } + mavenCentral() + } +} + plugins { id 'java-gradle-plugin' id 'java-test-fixtures' @@ -59,6 +69,10 @@ gradlePlugin { } repositories { + maven { + url 'https://jitpack.io' + } + mavenCentral() gradlePluginPortal() } diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index 52e72d973f2ed..84e56bbaf03ad 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -257,6 +257,9 @@ tasks.named('licenseHeaders').configure { *****************************************************************************/ repositories { + maven { + url 'https://jitpack.io' + } mavenCentral() gradlePluginPortal() } diff --git a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties index fcbbad6dd644c..515ab9d5f1822 100644 --- a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties +++ b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=194717442575a6f96e1c1befa2c30e9a4fc90f701d7aee33eb879b79e7ff05c0 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-all.zip +distributionSha256Sum=f8b4f4772d302c8ff580bc40d0f56e715de69b163546944f787c87abf209c961 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/build-tools-internal/settings.gradle b/build-tools-internal/settings.gradle index 6423750872ca2..1b4fb1215a59d 100644 --- a/build-tools-internal/settings.gradle +++ b/build-tools-internal/settings.gradle @@ -1,5 +1,13 @@ pluginManagement { - includeBuild "../build-conventions" + repositories { + maven { + url 'https://jitpack.io' + } + mavenCentral() + gradlePluginPortal() + } + + includeBuild "../build-conventions" includeBuild "../build-tools" } @@ -9,4 +17,4 @@ dependencyResolutionManagement { from(files("../gradle/build.versions.toml")) } } -} \ No newline at end of file +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java index bfc38e13043b9..d10cecf7fa50e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java @@ -99,8 +99,8 @@ private void configureGeneralTaskDefaults(Project project) { project.getTasks().withType(AbstractCopyTask.class).configureEach(t -> { t.dependsOn(project.getTasks().withType(EmptyDirTask.class)); t.setIncludeEmptyDirs(true); - t.setDirMode(0755); - t.setFileMode(0644); + t.dirPermissions(permissions -> permissions.unix(0755)); + t.filePermissions(permissions -> permissions.unix(0644)); }); // common config across all archives diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTar.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTar.java index 29c7dfd422547..52000e8c8fd71 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTar.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTar.java @@ -145,7 +145,7 @@ private void visitSymbolicLink(final FileCopyDetailsInternal details) { visitedSymbolicLinks.add(details.getFile()); final TarArchiveEntry entry = new TarArchiveEntry(details.getRelativePath().getPathString(), TarConstants.LF_SYMLINK); entry.setModTime(getModTime(details)); - entry.setMode(UnixStat.LINK_FLAG | details.getMode()); + entry.setMode(UnixStat.LINK_FLAG | details.getPermissions().toUnixNumeric()); try { entry.setLinkName(Files.readSymbolicLink(details.getFile().toPath()).toString()); tar.putArchiveEntry(entry); @@ -158,7 +158,7 @@ private void visitSymbolicLink(final FileCopyDetailsInternal details) { private void visitDirectory(final FileCopyDetailsInternal details) { final TarArchiveEntry entry = new TarArchiveEntry(details.getRelativePath().getPathString() + "/"); entry.setModTime(getModTime(details)); - entry.setMode(UnixStat.DIR_FLAG | details.getMode()); + entry.setMode(UnixStat.DIR_FLAG | details.getPermissions().toUnixNumeric()); try { tar.putArchiveEntry(entry); tar.closeArchiveEntry(); @@ -170,7 +170,7 @@ private void visitDirectory(final FileCopyDetailsInternal details) { private void visitFile(final FileCopyDetailsInternal details) { final TarArchiveEntry entry = new TarArchiveEntry(details.getRelativePath().getPathString()); entry.setModTime(getModTime(details)); - entry.setMode(UnixStat.FILE_FLAG | details.getMode()); + entry.setMode(UnixStat.FILE_FLAG | details.getPermissions().toUnixNumeric()); entry.setSize(details.getSize()); try { tar.putArchiveEntry(entry); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java index 5e62790a9d78a..42834928bafed 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java @@ -260,7 +260,7 @@ private List getAvailableJavaVersions() { private Stream getAvailableJavaInstallationLocationSteam() { return Stream.concat( javaInstallationRegistry.toolchains().stream().map(metadata -> metadata.location), - Stream.of(new InstallationLocation(Jvm.current().getJavaHome(), "Current JVM")) + Stream.of(InstallationLocation.userDefined(Jvm.current().getJavaHome(), "Current JVM")) ); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesPrecommitPlugin.java index 72c08712a1fd9..b1d9cbd1f01d1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesPrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesPrecommitPlugin.java @@ -12,30 +12,23 @@ import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin; import org.gradle.api.Project; import org.gradle.api.Task; -import org.gradle.api.artifacts.Configuration; -import org.gradle.api.artifacts.ProjectDependency; +import org.gradle.api.artifacts.component.ComponentIdentifier; +import org.gradle.api.artifacts.component.ModuleComponentIdentifier; import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.specs.Spec; import org.gradle.api.tasks.TaskProvider; public class DependencyLicensesPrecommitPlugin extends PrecommitPlugin { + private static Spec COMPONENT_FILTER = identifier -> (identifier instanceof ModuleComponentIdentifier) + && ((ModuleComponentIdentifier) identifier).getGroup().startsWith("org.elasticsearch") == false; @Override public TaskProvider createTask(Project project) { project.getPlugins().apply(CompileOnlyResolvePlugin.class); - TaskProvider dependencyLicenses = project.getTasks() - .register("dependencyLicenses", DependencyLicensesTask.class); - - // only require dependency licenses for non-elasticsearch deps - dependencyLicenses.configure(t -> { - Configuration runtimeClasspath = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME); - Configuration compileOnly = project.getConfigurations() - .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME); - t.setDependencies( - runtimeClasspath.fileCollection( - dependency -> dependency instanceof ProjectDependency == false - && dependency.getGroup().startsWith("org.elasticsearch") == false - ).minus(compileOnly) - ); + var dependencyLicenses = project.getTasks().register("dependencyLicenses", DependencyLicensesTask.class, t -> { + var runtimeClasspath = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME); + var compileOnly = project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME); + t.configureDependencies(runtimeClasspath, compileOnly, COMPONENT_FILTER); }); return dependencyLicenses; } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java index f71973c2fb15c..0099a4616f829 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java @@ -11,6 +11,8 @@ import org.gradle.api.DefaultTask; import org.gradle.api.GradleException; import org.gradle.api.InvalidUserDataException; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.component.ComponentIdentifier; import org.gradle.api.file.Directory; import org.gradle.api.file.DirectoryProperty; import org.gradle.api.file.FileCollection; @@ -18,7 +20,9 @@ import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; import org.gradle.api.model.ObjectFactory; +import org.gradle.api.provider.Property; import org.gradle.api.provider.Provider; +import org.gradle.api.specs.Spec; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputDirectory; import org.gradle.api.tasks.InputFiles; @@ -41,6 +45,8 @@ import javax.inject.Inject; +import static org.elasticsearch.gradle.internal.util.DependenciesUtils.createFileCollectionFromNonTransitiveArtifactsView; + /** * A task to check licenses for dependencies. *

@@ -83,7 +89,7 @@ * for the dependency. This artifact will be redistributed by us with the release to * comply with the license terms. */ -public class DependencyLicensesTask extends DefaultTask { +public abstract class DependencyLicensesTask extends DefaultTask { private final Pattern regex = Pattern.compile("-v?\\d+.*"); @@ -181,6 +187,10 @@ public void ignoreFile(String file) { ignoreFiles.add(file); } + @Input + @Optional + public abstract Property> getComponentFilter(); + @TaskAction public void checkDependencies() { if (dependencies == null) { @@ -295,7 +305,6 @@ private String getFileName(String name, Map counters, String type) { // try the other suffix...TODO: get rid of this, just support ending in .txt return fileName + ".txt"; } - return fileName; } @@ -310,4 +319,15 @@ public LinkedHashMap getMappings() { return new LinkedHashMap<>(mappings); } + /** + * Convencience method for configuring dependencies to be checked and ignoring transitive dependencies for now. + * */ + public void configureDependencies( + Configuration plusConfiguration, + Configuration minusConfiguration, + Spec componentFilter + ) { + setDependencies(createFileCollectionFromNonTransitiveArtifactsView(plusConfiguration, componentFilter).minus(minusConfiguration)); + } + } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java index f6d3787a4f686..1fc030be42480 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java @@ -15,11 +15,14 @@ import org.gradle.api.Project; import org.gradle.api.Task; import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.component.ModuleComponentIdentifier; import org.gradle.api.tasks.TaskProvider; import java.io.File; import java.nio.file.Path; +import static org.elasticsearch.gradle.internal.util.DependenciesUtils.createFileCollectionFromNonTransitiveArtifactsView; + public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin { public static final String JDK_JAR_HELL_CONFIG_NAME = "jdkJarHell"; @@ -54,12 +57,14 @@ public TaskProvider createTask(Project project) { Configuration compileOnly = project.getConfigurations() .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME); t.setClasspath(runtimeConfiguration.plus(compileOnly)); - t.getJarsToScan().from(runtimeConfiguration.fileCollection(dep -> { - // These are SelfResolvingDependency, and some of them backed by file collections, like the Gradle API files, - // or dependencies added as `files(...)`, we can't be sure if those are third party or not. - // err on the side of scanning these to make sure we don't miss anything - return dep.getGroup() != null && dep.getGroup().startsWith("org.elasticsearch") == false; - })); + t.getJarsToScan() + .from( + createFileCollectionFromNonTransitiveArtifactsView( + runtimeConfiguration, + identifier -> identifier instanceof ModuleComponentIdentifier + && ((ModuleComponentIdentifier) identifier).getGroup().startsWith("org.elasticsearch") == false + ) + ); t.dependsOn(resourcesTask); if (BuildParams.getIsRuntimeJavaHomeSet()) { t.getJavaHome().set(project.provider(BuildParams::getRuntimeJavaHome).map(File::getPath)); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java new file mode 100644 index 0000000000000..081c28c14fd91 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.util; + +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.ResolvableDependencies; +import org.gradle.api.artifacts.component.ComponentIdentifier; +import org.gradle.api.artifacts.result.ResolvedComponentResult; +import org.gradle.api.artifacts.result.ResolvedDependencyResult; +import org.gradle.api.file.FileCollection; +import org.gradle.api.specs.AndSpec; +import org.gradle.api.specs.Spec; + +import java.util.Set; +import java.util.stream.Collectors; + +public class DependenciesUtils { + + public static FileCollection createFileCollectionFromNonTransitiveArtifactsView( + Configuration configuration, + Spec componentFilter + ) { + ResolvableDependencies incoming = configuration.getIncoming(); + return incoming.artifactView(viewConfiguration -> { + Set firstLevelDependencyComponents = incoming.getResolutionResult() + .getRootComponent() + .map( + rootComponent -> rootComponent.getDependencies() + .stream() + .filter(dependency -> dependency instanceof ResolvedDependencyResult) + .map(dependency -> (ResolvedDependencyResult) dependency) + .filter(dependency -> dependency.getSelected() instanceof ResolvedComponentResult) + .map(dependency -> dependency.getSelected().getId()) + .collect(Collectors.toSet()) + ) + .get(); + viewConfiguration.componentFilter( + new AndSpec<>(identifier -> firstLevelDependencyComponents.contains(identifier), componentFilter) + ); + }).getFiles(); + } + +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/HdfsUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/HdfsUtils.java deleted file mode 100644 index 8b9570d62389e..0000000000000 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/HdfsUtils.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle.internal.util; - -import org.elasticsearch.gradle.OS; -import org.gradle.api.Project; -import org.gradle.api.logging.Logging; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - -public class HdfsUtils { - - public static boolean isHdfsFixtureSupported(Project project) { - String projectPath = project.getProjectDir().getPath(); - if (isLegalHdfsPath(projectPath) == false) { - Logging.getLogger(HdfsUtils.class).warn("hdfs Fixture unsupported since there are spaces in the path: '" + projectPath + "'"); - return false; - } - return (OS.current() != OS.WINDOWS) ? true : isHadoopWindowsInstallationAvailable(); - } - - private static boolean isHadoopWindowsInstallationAvailable() { - // hdfs fixture will not start without hadoop native libraries on windows - String nativePath = System.getenv("HADOOP_HOME"); - if (nativePath != null) { - Path path = Paths.get(nativePath); - if (Files.isDirectory(path) - && Files.exists(path.resolve("bin").resolve("winutils.exe")) - && Files.exists(path.resolve("bin").resolve("hadoop.dll")) - && Files.exists(path.resolve("bin").resolve("hdfs.dll"))) { - return true; - } else { - throw new IllegalStateException( - "HADOOP_HOME: " + path + " is invalid, does not contain hadoop native libraries in \\$HADOOP_HOME\\bin" - ); - } - } - Logging.getLogger(HdfsUtils.class).warn("hdfs Fixture unsupported, please set HADOOP_HOME and put HADOOP_HOME\\bin in PATH"); - - return false; - } - - public static boolean isLegalHdfsPath(String path) { - return path.contains(" ") == false; - - } -} diff --git a/build-tools-internal/src/main/resources/minimumGradleVersion b/build-tools-internal/src/main/resources/minimumGradleVersion index 631c6d36a93a4..83ea3179ddacc 100644 --- a/build-tools-internal/src/main/resources/minimumGradleVersion +++ b/build-tools-internal/src/main/resources/minimumGradleVersion @@ -1 +1 @@ -8.7 \ No newline at end of file +8.8 \ No newline at end of file diff --git a/build-tools/build.gradle b/build-tools/build.gradle index eb5573ac03e0e..7ba5e9f6faa62 100644 --- a/build-tools/build.gradle +++ b/build-tools/build.gradle @@ -6,6 +6,15 @@ * Side Public License, v 1. */ +buildscript { + repositories { + maven { + url 'https://jitpack.io' + } + mavenCentral() + } +} + plugins { id 'java-gradle-plugin' id 'groovy' @@ -107,6 +116,9 @@ configurations { } repositories { + maven { + url 'https://jitpack.io' + } mavenCentral() gradlePluginPortal() } diff --git a/build-tools/settings.gradle b/build-tools/settings.gradle index 63d80efcd505e..7590b8b6b054e 100644 --- a/build-tools/settings.gradle +++ b/build-tools/settings.gradle @@ -17,4 +17,4 @@ dependencyResolutionManagement { from(files("../gradle/build.versions.toml")) } } -} \ No newline at end of file +} diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java index fb8416b24d052..2bc4aa1a1be36 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java @@ -11,11 +11,9 @@ import org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes; import org.elasticsearch.gradle.transform.SymbolicLinkPreservingUntarTransform; import org.elasticsearch.gradle.transform.UnzipTransform; -import org.gradle.api.Action; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Plugin; import org.gradle.api.Project; -import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.dsl.DependencyHandler; import org.gradle.api.artifacts.repositories.IvyArtifactRepository; import org.gradle.api.artifacts.type.ArtifactTypeDefinition; @@ -24,6 +22,7 @@ import org.gradle.api.provider.Provider; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import javax.inject.Inject; @@ -46,6 +45,7 @@ public class DistributionDownloadPlugin implements Plugin { public static final String DISTRO_EXTRACTED_CONFIG_PREFIX = "es_distro_extracted_"; public static final String DISTRO_CONFIG_PREFIX = "es_distro_file_"; + private final ObjectFactory objectFactory; private NamedDomainObjectContainer distributionsContainer; private List distributionsResolutionStrategies; @@ -53,6 +53,7 @@ public class DistributionDownloadPlugin implements Plugin { @Inject public DistributionDownloadPlugin(ObjectFactory objectFactory) { + this.objectFactory = objectFactory; this.dockerAvailability = objectFactory.property(Boolean.class).value(false); } @@ -67,36 +68,92 @@ public void apply(Project project) { transformSpec.getTo().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE); }); - ArtifactTypeDefinition tarArtifactTypeDefinition = project.getDependencies().getArtifactTypes().maybeCreate("tar.gz"); + var tarArtifactTypeDefinition = project.getDependencies().getArtifactTypes().maybeCreate("tar.gz"); project.getDependencies().registerTransform(SymbolicLinkPreservingUntarTransform.class, transformSpec -> { transformSpec.getFrom().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, tarArtifactTypeDefinition.getName()); transformSpec.getTo().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE); }); setupResolutionsContainer(project); - setupDistributionContainer(project, dockerAvailability); + setupDistributionContainer(project); setupDownloadServiceRepo(project); } - private void setupDistributionContainer(Project project, Property dockerAvailable) { - + private void setupDistributionContainer(Project project) { distributionsContainer = project.container(ElasticsearchDistribution.class, name -> { - Configuration fileConfiguration = project.getConfigurations().create(DISTRO_CONFIG_PREFIX + name); - Configuration extractedConfiguration = project.getConfigurations().create(DISTRO_EXTRACTED_CONFIG_PREFIX + name); + var fileConfiguration = project.getConfigurations().create(DISTRO_CONFIG_PREFIX + name); + var extractedConfiguration = project.getConfigurations().create(DISTRO_EXTRACTED_CONFIG_PREFIX + name); extractedConfiguration.getAttributes() .attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE); - return new ElasticsearchDistribution( + + var distribution = new ElasticsearchDistribution( name, - project.getObjects(), + objectFactory, dockerAvailability, - project.getObjects().fileCollection().from(fileConfiguration), - project.getObjects().fileCollection().from(extractedConfiguration), - new FinalizeDistributionAction(distributionsResolutionStrategies, project) + objectFactory.fileCollection().from(fileConfiguration), + objectFactory.fileCollection().from(extractedConfiguration) ); + + registerDistributionDependencies(project, distribution); + return distribution; }); project.getExtensions().add(CONTAINER_NAME, distributionsContainer); } + private void registerDistributionDependencies(Project project, ElasticsearchDistribution distribution) { + project.getConfigurations() + .getByName(DISTRO_CONFIG_PREFIX + distribution.getName()) + .getDependencies() + .addLater( + project.provider(() -> distribution.maybeFreeze()) + .map( + frozenDistro -> project.getDependencies() + .create(resolveDependencyNotation(project, frozenDistro).getDefaultNotation()) + ) + ); + + project.getConfigurations() + .getByName(DISTRO_EXTRACTED_CONFIG_PREFIX + distribution.getName()) + .getDependencies() + .addAllLater( + project.provider(() -> distribution.maybeFreeze()) + .map( + frozenDistro -> distribution.getType().shouldExtract() + ? List.of( + project.getDependencies().create(resolveDependencyNotation(project, frozenDistro).getExtractedNotation()) + ) + : Collections.emptyList() + ) + ); + } + + private DistributionDependency resolveDependencyNotation(Project project, ElasticsearchDistribution distro) { + return distributionsResolutionStrategies.stream() + .map(r -> r.getResolver().resolve(project, distro)) + .filter(d -> d != null) + .findFirst() + .orElseGet(() -> DistributionDependency.of(dependencyNotation(distro))); + } + + /** + * Returns a dependency object representing the given distribution. + *

+ * The returned object is suitable to be passed to {@link DependencyHandler}. + * The concrete type of the object will be a set of maven coordinates as a {@link String}. + * Maven coordinates point to either the integ-test-zip coordinates on maven central, or a set of artificial + * coordinates that resolve to the Elastic download service through an ivy repository. + */ + private static String dependencyNotation(ElasticsearchDistribution distribution) { + if (distribution.getType() == ElasticsearchDistributionTypes.INTEG_TEST_ZIP) { + return "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + distribution.getVersion() + "@zip"; + } + var distroVersion = Version.fromString(distribution.getVersion()); + var extension = distribution.getType().getExtension(distribution.getPlatform()); + var classifier = distribution.getType().getClassifier(distribution.getPlatform(), distroVersion); + var group = distribution.getVersion().endsWith("-SNAPSHOT") ? FAKE_SNAPSHOT_IVY_GROUP : FAKE_IVY_GROUP; + return group + ":elasticsearch" + ":" + distribution.getVersion() + classifier + "@" + extension; + } + private void setupResolutionsContainer(Project project) { distributionsResolutionStrategies = new ArrayList<>(); project.getExtensions().add(RESOLUTION_CONTAINER_NAME, distributionsResolutionStrategies); @@ -133,53 +190,4 @@ private static void setupDownloadServiceRepo(Project project) { addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://snapshots-no-kpi.elastic.co", FAKE_SNAPSHOT_IVY_GROUP); } - private record FinalizeDistributionAction(List resolutionList, Project project) - implements - Action { - @Override - - public void execute(ElasticsearchDistribution distro) { - finalizeDistributionDependencies(project, distro); - } - - private void finalizeDistributionDependencies(Project project, ElasticsearchDistribution distribution) { - // for the distribution as a file, just depend on the artifact directly - DistributionDependency distributionDependency = resolveDependencyNotation(project, distribution); - project.getDependencies().add(DISTRO_CONFIG_PREFIX + distribution.getName(), distributionDependency.getDefaultNotation()); - // no extraction needed for rpm, deb or docker - if (distribution.getType().shouldExtract()) { - // The extracted configuration depends on the artifact directly but has - // an artifact transform registered to resolve it as an unpacked folder. - project.getDependencies() - .add(DISTRO_EXTRACTED_CONFIG_PREFIX + distribution.getName(), distributionDependency.getExtractedNotation()); - } - } - - private DistributionDependency resolveDependencyNotation(Project project, ElasticsearchDistribution distro) { - return resolutionList.stream() - .map(r -> r.getResolver().resolve(project, distro)) - .filter(d -> d != null) - .findFirst() - .orElseGet(() -> DistributionDependency.of(dependencyNotation(distro))); - } - - /** - * Returns a dependency object representing the given distribution. - *

- * The returned object is suitable to be passed to {@link DependencyHandler}. - * The concrete type of the object will be a set of maven coordinates as a {@link String}. - * Maven coordinates point to either the integ-test-zip coordinates on maven central, or a set of artificial - * coordinates that resolve to the Elastic download service through an ivy repository. - */ - private String dependencyNotation(ElasticsearchDistribution distribution) { - if (distribution.getType() == ElasticsearchDistributionTypes.INTEG_TEST_ZIP) { - return "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + distribution.getVersion() + "@zip"; - } - Version distroVersion = Version.fromString(distribution.getVersion()); - String extension = distribution.getType().getExtension(distribution.getPlatform()); - String classifier = distribution.getType().getClassifier(distribution.getPlatform(), distroVersion); - String group = distribution.getVersion().endsWith("-SNAPSHOT") ? FAKE_SNAPSHOT_IVY_GROUP : FAKE_IVY_GROUP; - return group + ":elasticsearch" + ":" + distribution.getVersion() + classifier + "@" + extension; - } - } } diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java b/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java index fab6926008d6c..afb90ba1ca62e 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java @@ -9,7 +9,6 @@ package org.elasticsearch.gradle; import org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes; -import org.gradle.api.Action; import org.gradle.api.Buildable; import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.FileCollection; @@ -55,7 +54,6 @@ public String toString() { private final Property failIfUnavailable; private final Property preferArchive; private final ConfigurableFileCollection extracted; - private transient Action distributionFinalizer; private boolean frozen = false; ElasticsearchDistribution( @@ -63,8 +61,7 @@ public String toString() { ObjectFactory objectFactory, Property dockerAvailability, ConfigurableFileCollection fileConfiguration, - ConfigurableFileCollection extractedConfiguration, - Action distributionFinalizer + ConfigurableFileCollection extractedConfiguration ) { this.name = name; this.dockerAvailability = dockerAvailability; @@ -78,7 +75,6 @@ public String toString() { this.failIfUnavailable = objectFactory.property(Boolean.class).convention(true); this.preferArchive = objectFactory.property(Boolean.class).convention(false); this.extracted = extractedConfiguration; - this.distributionFinalizer = distributionFinalizer; } public String getName() { @@ -172,7 +168,6 @@ public String toString() { public ElasticsearchDistribution maybeFreeze() { if (frozen == false) { finalizeValues(); - distributionFinalizer.execute(this); frozen = true; } return this; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index 999f27a646b1f..d25798ad071bd 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -246,14 +246,12 @@ public void setVersions(List versions) { private void doSetVersion(String version) { String distroName = "testclusters" + path.replace(":", "-") + "-" + this.name + "-" + version; NamedDomainObjectContainer container = DistributionDownloadPlugin.getContainer(project); - if (container.findByName(distroName) == null) { - container.create(distroName); - } - ElasticsearchDistribution distro = container.getByName(distroName); - distro.setVersion(version); - distro.setArchitecture(Architecture.current()); - setDistributionType(distro, testDistribution); - distributions.add(distro); + // TODO Refactor test using register<> for reducing overhead + ElasticsearchDistribution distribution = container.maybeCreate(distroName); + distribution.setVersion(version); + distribution.setArchitecture(Architecture.current()); + setDistributionType(distribution, testDistribution); + distributions.add(distribution); } @Internal diff --git a/build.gradle b/build.gradle index 1d9757f32543d..3869d21b49bfe 100644 --- a/build.gradle +++ b/build.gradle @@ -25,6 +25,16 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure +buildscript { + repositories { + maven { + url 'https://jitpack.io' + } + + mavenCentral() + } +} + plugins { id 'lifecycle-base' id 'elasticsearch.docker-support' @@ -325,7 +335,7 @@ allprojects { integTestTask.mustRunAfter tasks.matching { it.name.equals("test") } } - configurations.matching { it.canBeResolved }.all { Configuration configuration -> +/* configurations.matching { it.canBeResolved }.all { Configuration configuration -> dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep -> Project upstreamProject = dep.dependencyProject if (project.path != upstreamProject?.path) { @@ -336,7 +346,7 @@ allprojects { } } } - } + }*/ } apply plugin: 'elasticsearch.formatting' diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle index 4d7850477dbf5..815ac5d4c2dd8 100644 --- a/distribution/archives/build.gradle +++ b/distribution/archives/build.gradle @@ -18,11 +18,17 @@ CopySpec archiveFiles(String distributionType, String os, String architecture, b with libFiles(os, architecture) } into('config') { - dirMode 0750 - fileMode 0660 + dirPermissions { + unix 0750 + } + filePermissions { + unix 0660 + } with configFiles(distributionType, isTestDistro) from { - dirMode 0750 + dirPermissions { + unix 0750 + } jvmOptionsDir.getParent() } } @@ -36,21 +42,31 @@ CopySpec archiveFiles(String distributionType, String os, String architecture, b } into('') { from { - dirMode 0755 + dirPermissions { + unix 0755 + } logsDir.getParent() } } into('') { from { - dirMode 0755 + dirPermissions { + unix 0755 + } pluginsDir.getParent() } } from(rootProject.projectDir) { + filePermissions { + unix(0644) + } include 'README.asciidoc' } from(rootProject.file('licenses')) { include isTestDistro ? 'SSPL-1.0+ELASTIC-LICENSE-2.0.txt' : 'ELASTIC-LICENSE-2.0.txt' + filePermissions { + unix(0644) + } rename { 'LICENSE.txt' } } diff --git a/distribution/build.gradle b/distribution/build.gradle index c3f9192ecee05..77f1a2d032c73 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -346,9 +346,9 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { if (it.relativePath.segments[-2] == 'bin' || (os == 'darwin' && it.relativePath.segments[-2] == 'MacOS')) { // bin files, wherever they are within modules (eg platform specific) should be executable // and MacOS is an alternative to bin on macOS - it.mode = 0755 + it.permissions.unix(0755) } else { - it.mode = 0644 + it.permissions.unix(0644) } } List excludePlatforms = ['linux-x86_64', 'linux-aarch64', 'windows-x86_64', 'darwin-x86_64', 'darwin-aarch64'] @@ -404,7 +404,11 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { from '../src/bin' exclude '*.exe' exclude '*.bat' - eachFile { it.setMode(0755) } + eachFile { + it.permissions{ + unix(0755) + } + } filter("tokens" : expansionsForDistribution(distributionType, testDistro), ReplaceTokens.class) } // windows files, only for zip @@ -422,7 +426,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { } // module provided bin files with copySpec { - eachFile { it.setMode(0755) } + eachFile { it.permissions.unix(0755) } from(testDistro ? integTestBinFiles : defaultBinFiles) if (distributionType != 'zip') { exclude '*.bat' @@ -437,7 +441,9 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { from buildServerNoticeTaskProvider } else { from (buildDefaultNoticeTaskProvider) { - fileMode = 0644 + filePermissions { + unix(0644) + } } } } @@ -456,7 +462,13 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { } eachFile { FileCopyDetails details -> if (details.relativePath.segments[-2] == 'bin' || details.relativePath.segments[-1] == 'jspawnhelper') { - details.mode = 0755 + details.permissions { + unix(0755) + } + } else { + details.permissions { + unix(0644) + } } if (details.name == 'src.zip') { details.exclude() diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index 6b57f32310c93..6c31bc44017c3 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -42,20 +42,10 @@ import java.util.regex.Pattern * dpkg -c path/to/elasticsearch.deb */ -buildscript { - repositories { - maven { - url 'https://jitpack.io' - } - mavenCentral() - } - dependencies { - classpath "com.github.breskeby:gradle-ospackage-plugin:2da19425133" - } +plugins { + id "com.netflix.nebula.ospackage-base" version "11.9.1" } -apply plugin: "com.netflix.nebula.ospackage-base" - ['deb', 'rpm'].each { type -> String packagingFiles = "build/packaging/${type}" @@ -138,7 +128,9 @@ def commonPackageConfig(String type, String architecture) { } from(rootProject.projectDir) { include 'README.asciidoc' - fileMode 0644 + filePermissions { + unix 0644 + } } into('lib') { with libFiles('linux', architecture) @@ -159,9 +151,13 @@ def commonPackageConfig(String type, String architecture) { directory('/' + segments[0..i].join('/'), 0755) } if (segments[-2] == 'bin' || segments[-1] == 'jspawnhelper') { - fcp.mode = 0755 + fcp.permissions { + unix(0755) + } } else { - fcp.mode = 0644 + fcp.permissions { + unix(0644) + } } } } @@ -171,7 +167,9 @@ def commonPackageConfig(String type, String architecture) { if (type == 'deb') { into("/usr/share/doc/${packageName}") { from "${packagingFiles}/copyright" - fileMode 0644 + filePermissions { + unix(0644) + } } } else { assert type == 'rpm' @@ -180,7 +178,9 @@ def commonPackageConfig(String type, String architecture) { include 'ELASTIC-LICENSE-2.0.txt' rename { 'LICENSE.txt' } } - fileMode 0644 + filePermissions { + unix(0644) + } } } @@ -194,7 +194,9 @@ def commonPackageConfig(String type, String architecture) { configurationFile '/etc/elasticsearch/users' configurationFile '/etc/elasticsearch/users_roles' from("${packagingFiles}") { - dirMode 02750 + dirPermissions { + unix(02750) + } into('/etc') permissionGroup 'elasticsearch' setgid true @@ -205,9 +207,13 @@ def commonPackageConfig(String type, String architecture) { } from("${packagingFiles}/etc/elasticsearch") { into('/etc/elasticsearch') - dirMode 02750 + dirPermissions { + unix(02750) + } setgid = true - fileMode 0660 + filePermissions { + unix(0660) + } permissionGroup 'elasticsearch' includeEmptyDirs true createDirectoryEntry true @@ -218,28 +224,38 @@ def commonPackageConfig(String type, String architecture) { into(new File(envFile).getParent()) { fileType CONFIG | NOREPLACE permissionGroup 'elasticsearch' - fileMode 0660 + filePermissions { + unix(0660) + } from "${packagingFiles}/env/elasticsearch" } // ========= systemd ========= into('/usr/lib/tmpfiles.d') { from "${packagingFiles}/systemd/elasticsearch.conf" - fileMode 0644 + filePermissions { + unix(0644) + } } into('/usr/lib/systemd/system') { fileType CONFIG | NOREPLACE from "${packagingFiles}/systemd/elasticsearch.service" - fileMode 0644 + filePermissions { + unix(0644) + } } into('/usr/lib/sysctl.d') { fileType CONFIG | NOREPLACE from "${packagingFiles}/systemd/sysctl/elasticsearch.conf" - fileMode 0644 + filePermissions { + unix(0644) + } } into('/usr/share/elasticsearch/bin') { from "${packagingFiles}/systemd/systemd-entrypoint" - fileMode 0755 + filePermissions { + unix(0755) + } } // ========= empty dirs ========= @@ -253,7 +269,9 @@ def commonPackageConfig(String type, String architecture) { createDirectoryEntry true user u permissionGroup g - dirMode = mode + dirPermissions { + unix(mode) + } setgid (mode == 02750) } } @@ -322,7 +340,9 @@ Closure commonDebConfig(String architecture) { into('/usr/share/lintian/overrides') { from('src/deb/lintian/elasticsearch') - fileMode 0644 + filePermissions { + unix(0644) + } } } } diff --git a/gradle/build.versions.toml b/gradle/build.versions.toml index 6b5a541d15661..ba81673120569 100644 --- a/gradle/build.versions.toml +++ b/gradle/build.versions.toml @@ -38,7 +38,7 @@ maven-model = "org.apache.maven:maven-model:3.6.2" mockito-core = "org.mockito:mockito-core:1.9.5" nebula-info = "com.netflix.nebula:gradle-info-plugin:11.3.3" reflections = "org.reflections:reflections:0.9.12" -shadow-plugin = "com.github.johnrengelman:shadow:8.1.1" +shadow-plugin = "com.github.breskeby:shadow:3b035f2" spock-core = { group = "org.spockframework", name="spock-core", version.ref="spock" } spock-junit4 = { group = "org.spockframework", name="spock-junit4", version.ref="spock" } spock-platform = { group = "org.spockframework", name="spock-bom", version.ref="spock" } diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 41c3bafde5e33..532112d0138d3 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -376,9 +376,14 @@ - - - + + + + + + + + @@ -851,14 +856,9 @@ - - - - - - - - + + + @@ -4077,6 +4077,11 @@ + + + + + @@ -4182,6 +4187,11 @@ + + + + + diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index fcbbad6dd644c..515ab9d5f1822 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=194717442575a6f96e1c1befa2c30e9a4fc90f701d7aee33eb879b79e7ff05c0 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-all.zip +distributionSha256Sum=f8b4f4772d302c8ff580bc40d0f56e715de69b163546944f787c87abf209c961 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index 1aa94a4269074..b740cf13397ab 100755 --- a/gradlew +++ b/gradlew @@ -55,7 +55,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. diff --git a/plugins/examples/gradle/wrapper/gradle-wrapper.properties b/plugins/examples/gradle/wrapper/gradle-wrapper.properties index fcbbad6dd644c..515ab9d5f1822 100644 --- a/plugins/examples/gradle/wrapper/gradle-wrapper.properties +++ b/plugins/examples/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=194717442575a6f96e1c1befa2c30e9a4fc90f701d7aee33eb879b79e7ff05c0 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-all.zip +distributionSha256Sum=f8b4f4772d302c8ff580bc40d0f56e715de69b163546944f787c87abf209c961 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/server/build.gradle b/server/build.gradle index 03713bc3d2837..5831930421c60 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -254,12 +254,12 @@ tasks.named("thirdPartyAudit").configure { tasks.named("dependencyLicenses").configure { mapping from: /lucene-.*/, to: 'lucene' mapping from: /log4j-.*/, to: 'log4j' - dependencies = project.configurations.runtimeClasspath.fileCollection { - it.group.startsWith('org.elasticsearch') == false || - // keep the following org.elasticsearch jars in - (it.name == 'jna' || - it.name == 'securesm') - } + + configureDependencies( + project.configurations.runtimeClasspath, project.configurations.resolveableCompileOnly, identifier -> { + return identifier instanceof ModuleComponentIdentifier + (identifier.moduleIdentifier.name == 'jna' || identifier.moduleIdentifier.name == 'securesm') + }) } tasks.named("licenseHeaders").configure { diff --git a/settings.gradle b/settings.gradle index 48e3794c9005d..6ed340b27da65 100644 --- a/settings.gradle +++ b/settings.gradle @@ -4,6 +4,9 @@ import org.elasticsearch.gradle.internal.toolchain.AdoptiumJdkToolchainResolver pluginManagement { repositories { + maven { + url 'https://jitpack.io' + } mavenCentral() gradlePluginPortal() }