From 1fa2d0c5abc6be7b8b9779716860f23d76af455c Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:53:14 +0100 Subject: [PATCH 1/4] [8.x][ML] Report deprecation for outdated snapshots (#118939) This PR fixes the constant used to produce deprecated warnings in the Upgrade Assistant in 8.18. It ensures that for all ML model snapshots produced prior to version 8.3 the user can either delete or update them. I marked it as a non-issue since it fixes the logic updated in #118166 and has not been released yet. --- .../elasticsearch/xpack/deprecation/MlDeprecationChecker.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecker.java index c0e1c054f7a13..de73d05a5a37e 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecker.java @@ -26,7 +26,6 @@ import java.util.Map; import java.util.Optional; -import static org.elasticsearch.xpack.core.ml.MachineLearningField.MIN_CHECKED_SUPPORTED_SNAPSHOT_VERSION; import static org.elasticsearch.xpack.core.ml.MachineLearningField.MIN_REPORTED_SUPPORTED_SNAPSHOT_VERSION; public class MlDeprecationChecker implements DeprecationChecker { @@ -69,7 +68,7 @@ static Optional checkDataFeedAggregations(DatafeedConfig dataf } static Optional checkModelSnapshot(ModelSnapshot modelSnapshot) { - if (modelSnapshot.getMinVersion().before(MIN_CHECKED_SUPPORTED_SNAPSHOT_VERSION)) { + if (modelSnapshot.getMinVersion().before(MIN_REPORTED_SUPPORTED_SNAPSHOT_VERSION)) { StringBuilder details = new StringBuilder( String.format( Locale.ROOT, From 5de28aaf92b99d8ba2b1e8b95ba8d9bd7f83e34f Mon Sep 17 00:00:00 2001 From: kosabogi <105062005+kosabogi@users.noreply.github.com> Date: Thu, 19 Dec 2024 12:10:53 +0100 Subject: [PATCH 2/4] [DOCS] Updates SharePoint Online page (#118318) (#119058) --- .../connectors-sharepoint-online.asciidoc | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc b/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc index 02f598c16f63c..2680e3ff840a6 100644 --- a/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc +++ b/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc @@ -133,6 +133,58 @@ The application name will appear in the Title box. ---- +[discrete#es-connectors-sharepoint-online-sites-selected-permissions] +====== Granting `Sites.Selected` permissions + +To configure `Sites.Selected` permissions, follow these steps in the Azure Active Directory portal. These permissions enable precise access control to specific SharePoint sites. + +. Sign in to the https://portal.azure.com/[Azure Active Directory portal^]. +. Navigate to **App registrations** and locate the application created for the connector. +. Under **API permissions**, click **Add permission**. +. Select **Microsoft Graph** > **Application permissions**, then add `Sites.Selected`. +. Click **Grant admin consent** to approve the permission. + +[TIP] +==== +Refer to the official https://learn.microsoft.com/en-us/graph/permissions-reference[Microsoft documentation] for managing permissions in Azure AD. +==== + +To assign access to specific SharePoint sites using `Sites.Selected`: + +. Use Microsoft Graph Explorer or PowerShell to grant access. +. To fetch the site ID, run the following Graph API query: ++ +[source, http] +---- +GET https://graph.microsoft.com/v1.0/sites?select=webUrl,Title,Id&$search="*" +---- ++ +This will return the `id` of the site. + +. Use the `id` to assign read or write access: ++ +[source, http] +---- +POST https://graph.microsoft.com/v1.0/sites//permissions +{ + "roles": ["read"], // or "write" + "grantedToIdentities": [ + { + "application": { + "id": "", + "displayName": "" + } + } + ] +} +---- + +[NOTE] +==== +When using the `Comma-separated list of sites` configuration field, ensure the sites specified match those granted `Sites.Selected` permission in SharePoint. +If the `Comma-separated list of sites` field is set to `*` or the `Enumerate all sites` toggle is enabled, the connector will attempt to access all sites. This requires broader permissions, which are not supported with `Sites.Selected`. +==== + .Graph API permissions **** Microsoft recommends using Graph API for all operations with Sharepoint Online. Graph API is well-documented and more efficient at fetching data, which helps avoid throttling. @@ -594,6 +646,59 @@ The application name will appear in the Title box. ---- +[discrete#es-connectors-sharepoint-online-sites-selected-permissions-self-managed] +====== Granting `Sites.Selected` permissions + +To configure `Sites.Selected` permissions, follow these steps in the Azure Active Directory portal. These permissions enable precise access control to specific SharePoint sites. + +. Sign in to the https://portal.azure.com/[Azure Active Directory portal^]. +. Navigate to **App registrations** and locate the application created for the connector. +. Under **API permissions**, click **Add permission**. +. Select **Microsoft Graph** > **Application permissions**, then add `Sites.Selected`. +. Click **Grant admin consent** to approve the permission. + +[TIP] +==== +Refer to the official https://learn.microsoft.com/en-us/graph/permissions-reference[Microsoft documentation] for managing permissions in Azure AD. +==== + + +To assign access to specific SharePoint sites using `Sites.Selected`: + +. Use Microsoft Graph Explorer or PowerShell to grant access. +. To fetch the site ID, run the following Graph API query: ++ +[source, http] +---- +GET https://graph.microsoft.com/v1.0/sites?select=webUrl,Title,Id&$search="*" +---- ++ +This will return the `id` of the site. + +. Use the `id` to assign read or write access: ++ +[source, http] +---- +POST https://graph.microsoft.com/v1.0/sites//permissions +{ + "roles": ["read"], // or "write" + "grantedToIdentities": [ + { + "application": { + "id": "", + "displayName": "" + } + } + ] +} +---- + +[NOTE] +==== +When using the `Comma-separated list of sites` configuration field, ensure the sites specified match those granted `Sites.Selected` permission in SharePoint. +If the `Comma-separated list of sites` field is set to `*` or the `Enumerate all sites` toggle is enabled, the connector will attempt to access all sites. This requires broader permissions, which are not supported with `Sites.Selected`. +==== + .Graph API permissions **** Microsoft recommends using Graph API for all operations with Sharepoint Online. Graph API is well-documented and more efficient at fetching data, which helps avoid throttling. From 023f73f93edd100476275406a6d8210d115cceef Mon Sep 17 00:00:00 2001 From: Ievgen Degtiarenko Date: Thu, 19 Dec 2024 12:18:52 +0100 Subject: [PATCH 3/4] group dataset files (#118739) (#119045) *.csv files used for creating data and *.csv-spec used to define test scenarios are blending in the resource directory. This change moves all *.csv files to data/*.csv so that it is easier to distinguish between data and specs. This allows to have a quicker overview of existing data when starting a new spec. --- .../java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java | 2 +- .../qa/testFixtures/src/main/resources/{ => data}/addresses.csv | 0 .../esql/qa/testFixtures/src/main/resources/{ => data}/ages.csv | 0 .../src/main/resources/{ => data}/airport_city_boundaries.csv | 0 .../qa/testFixtures/src/main/resources/{ => data}/airports.csv | 0 .../testFixtures/src/main/resources/{ => data}/airports_mp.csv | 0 .../testFixtures/src/main/resources/{ => data}/airports_web.csv | 0 .../qa/testFixtures/src/main/resources/{ => data}/alerts.csv | 0 .../esql/qa/testFixtures/src/main/resources/{ => data}/apps.csv | 0 .../qa/testFixtures/src/main/resources/{ => data}/books.csv | 0 .../src/main/resources/{ => data}/cartesian_multipolygons.csv | 0 .../testFixtures/src/main/resources/{ => data}/client_cidr.csv | 0 .../qa/testFixtures/src/main/resources/{ => data}/clientips.csv | 0 .../src/main/resources/{ => data}/countries_bbox.csv | 0 .../src/main/resources/{ => data}/countries_bbox_web.csv | 0 .../testFixtures/src/main/resources/{ => data}/date_nanos.csv | 0 .../qa/testFixtures/src/main/resources/{ => data}/decades.csv | 0 .../qa/testFixtures/src/main/resources/{ => data}/distances.csv | 0 .../qa/testFixtures/src/main/resources/{ => data}/employees.csv | 0 .../src/main/resources/{ => data}/employees_incompatible.csv | 0 .../qa/testFixtures/src/main/resources/{ => data}/heights.csv | 0 .../qa/testFixtures/src/main/resources/{ => data}/hosts.csv | 0 .../esql/qa/testFixtures/src/main/resources/{ => data}/k8s.csv | 0 .../qa/testFixtures/src/main/resources/{ => data}/languages.csv | 0 .../src/main/resources/{ => data}/languages_non_unique_key.csv | 0 .../src/main/resources/{ => data}/message_types.csv | 0 .../src/main/resources/{ => data}/missing_ip_sample_data.csv | 0 .../src/main/resources/{ => data}/multivalue_geometries.csv | 0 .../src/main/resources/{ => data}/multivalue_points.csv | 0 .../src/main/resources/{ => data}/mv_sample_data.csv | 0 .../testFixtures/src/main/resources/{ => data}/sample_data.csv | 0 .../src/main/resources/{ => data}/sample_data_ts_long.csv | 0 .../src/main/resources/{ => data}/sample_data_ts_nanos.csv | 0 .../src/main/resources/{ => data}/semantic_text.csv | 0 .../qa/testFixtures/src/main/resources/{ => data}/ul_logs.csv | 0 .../src/test/java/org/elasticsearch/xpack/esql/CsvTests.java | 2 +- 36 files changed, 2 insertions(+), 2 deletions(-) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/addresses.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/ages.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/airport_city_boundaries.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/airports.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/airports_mp.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/airports_web.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/alerts.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/apps.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/books.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/cartesian_multipolygons.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/client_cidr.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/clientips.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/countries_bbox.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/countries_bbox_web.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/date_nanos.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/decades.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/distances.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/employees.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/employees_incompatible.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/heights.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/hosts.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/k8s.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/languages.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/languages_non_unique_key.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/message_types.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/missing_ip_sample_data.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/multivalue_geometries.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/multivalue_points.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/mv_sample_data.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/sample_data.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/sample_data_ts_long.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/sample_data_ts_nanos.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/semantic_text.csv (100%) rename x-pack/plugin/esql/qa/testFixtures/src/main/resources/{ => data}/ul_logs.csv (100%) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index fd1d7d3051226..4d9adf06763a7 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -362,7 +362,7 @@ private static void load(RestClient client, TestsDataset dataset, Logger logger, if (mapping == null) { throw new IllegalArgumentException("Cannot find resource " + mappingName); } - final String dataName = "/" + dataset.dataFileName; + final String dataName = "/data/" + dataset.dataFileName; URL data = CsvTestsDataLoader.class.getResource(dataName); if (data == null) { throw new IllegalArgumentException("Cannot find resource " + dataName); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/addresses.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/addresses.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/addresses.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/addresses.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ages.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/ages.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/ages.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/ages.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/airport_city_boundaries.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/airport_city_boundaries.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/airport_city_boundaries.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/airport_city_boundaries.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/airports.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/airports.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/airports.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/airports.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/airports_mp.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/airports_mp.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/airports_mp.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/airports_mp.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/airports_web.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/airports_web.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/airports_web.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/airports_web.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/alerts.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/alerts.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/alerts.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/alerts.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/apps.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/apps.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/apps.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/apps.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/books.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/books.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/books.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/books.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/cartesian_multipolygons.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/cartesian_multipolygons.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/cartesian_multipolygons.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/cartesian_multipolygons.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/client_cidr.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/client_cidr.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/client_cidr.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/client_cidr.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/clientips.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/clientips.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/clientips.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/clientips.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/countries_bbox.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/countries_bbox.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/countries_bbox.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/countries_bbox.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/countries_bbox_web.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/countries_bbox_web.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/countries_bbox_web.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/countries_bbox_web.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/date_nanos.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/date_nanos.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/decades.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/decades.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/decades.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/decades.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/distances.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/distances.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/distances.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/distances.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/employees.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/employees.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/employees_incompatible.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/employees_incompatible.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/heights.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/heights.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/heights.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/heights.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hosts.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/hosts.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/hosts.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/hosts.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/k8s.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/k8s.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/languages.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/languages.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_non_unique_key.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/languages_non_unique_key.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_non_unique_key.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/languages_non_unique_key.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/message_types.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/message_types.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/message_types.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/message_types.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/missing_ip_sample_data.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/missing_ip_sample_data.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/missing_ip_sample_data.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/missing_ip_sample_data.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_geometries.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/multivalue_geometries.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_geometries.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/multivalue_geometries.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_points.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/multivalue_points.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_points.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/multivalue_points.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_sample_data.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/mv_sample_data.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_sample_data.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/mv_sample_data.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/sample_data.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/sample_data.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/sample_data.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/sample_data.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/sample_data_ts_long.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/sample_data_ts_long.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/sample_data_ts_long.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/sample_data_ts_long.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/sample_data_ts_nanos.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/sample_data_ts_nanos.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/sample_data_ts_nanos.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/sample_data_ts_nanos.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/semantic_text.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/semantic_text.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ul_logs.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/ul_logs.csv similarity index 100% rename from x-pack/plugin/esql/qa/testFixtures/src/main/resources/ul_logs.csv rename to x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/ul_logs.csv diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 1b4c7fee9c7dd..bfa56b1896289 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -423,7 +423,7 @@ private static CsvTestsDataLoader.TestsDataset testsDataset(LogicalPlan parsed) } private static TestPhysicalOperationProviders testOperationProviders(CsvTestsDataLoader.TestsDataset dataset) throws Exception { - var testData = loadPageFromCsv(CsvTests.class.getResource("/" + dataset.dataFileName()), dataset.typeMapping()); + var testData = loadPageFromCsv(CsvTests.class.getResource("/data/" + dataset.dataFileName()), dataset.typeMapping()); return new TestPhysicalOperationProviders(testData.v1(), testData.v2()); } From 2ae5911b27a29bc1cb5bcfac188b46fd64441240 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 19 Dec 2024 12:50:26 +0100 Subject: [PATCH 4/4] ESQL: Fix attribute set equals (#118823) (#119048) Also add a test that uses this, for lookup join field attribute ids. --- docs/changelog/118823.yaml | 5 +++ .../esql/core/expression/AttributeSet.java | 8 +++- .../core/expression/AttributeMapTests.java | 2 +- .../core/expression/AttributeSetTests.java | 42 +++++++++++++++++++ .../xpack/esql/analysis/AnalyzerTests.java | 31 ++++++++++++++ 5 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/118823.yaml create mode 100644 x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeSetTests.java diff --git a/docs/changelog/118823.yaml b/docs/changelog/118823.yaml new file mode 100644 index 0000000000000..b1afe1c873c17 --- /dev/null +++ b/docs/changelog/118823.yaml @@ -0,0 +1,5 @@ +pr: 118823 +summary: Fix attribute set equals +area: ES|QL +type: bug +issues: [] diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeSet.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeSet.java index a092e17931237..8a075e8887512 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeSet.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeSet.java @@ -174,8 +174,12 @@ public Stream parallelStream() { } @Override - public boolean equals(Object o) { - return delegate.equals(o); + public boolean equals(Object obj) { + if (obj instanceof AttributeSet as) { + obj = as.delegate; + } + + return delegate.equals(obj); } @Override diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java index 511c7f4b1d2f8..ade79c8168076 100644 --- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java @@ -30,7 +30,7 @@ public class AttributeMapTests extends ESTestCase { - private static Attribute a(String name) { + static Attribute a(String name) { return new UnresolvedAttribute(Source.EMPTY, name); } diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeSetTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeSetTests.java new file mode 100644 index 0000000000000..0e97773fb90d2 --- /dev/null +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeSetTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.core.expression; + +import org.elasticsearch.test.ESTestCase; + +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.expression.AttributeMapTests.a; + +public class AttributeSetTests extends ESTestCase { + + public void testEquals() { + Attribute a1 = a("1"); + Attribute a2 = a("2"); + + AttributeSet first = new AttributeSet(List.of(a1, a2)); + assertEquals(first, first); + + AttributeSet second = new AttributeSet(); + second.add(a1); + second.add(a2); + + assertEquals(first, second); + assertEquals(second, first); + + AttributeSet third = new AttributeSet(); + third.add(a("1")); + third.add(a("2")); + + assertNotEquals(first, third); + assertNotEquals(third, first); + + assertEquals(AttributeSet.EMPTY, AttributeSet.EMPTY); + assertEquals(AttributeSet.EMPTY, first.intersect(third)); + assertEquals(third.intersect(first), AttributeSet.EMPTY); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index f9390d4e2b1d3..6afc4c06f5551 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.AttributeSet; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; @@ -2197,6 +2198,36 @@ public void testLookupJoinUnknownField() { assertThat(e.getMessage(), containsString(errorMessage3 + "right side of join")); } + public void testMultipleLookupJoinsGiveDifferentAttributes() { + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V8.isEnabled()); + + // The field attributes that get contributed by different LOOKUP JOIN commands must have different name ids, + // even if they have the same names. Otherwise, things like dependency analysis - like in PruneColumns - cannot work based on + // name ids and shadowing semantics proliferate into all kinds of optimizer code. + + String query = "FROM test" + + "| EVAL language_code = languages" + + "| LOOKUP JOIN languages_lookup ON language_code" + + "| LOOKUP JOIN languages_lookup ON language_code"; + LogicalPlan analyzedPlan = analyze(query); + + List lookupFields = new ArrayList<>(); + List> lookupFieldNames = new ArrayList<>(); + analyzedPlan.forEachUp(EsRelation.class, esRelation -> { + if (esRelation.indexMode() == IndexMode.LOOKUP) { + lookupFields.add(esRelation.outputSet()); + lookupFieldNames.add(esRelation.outputSet().stream().map(NamedExpression::name).collect(Collectors.toSet())); + } + }); + + assertEquals(lookupFieldNames.size(), 2); + assertEquals(lookupFieldNames.get(0), lookupFieldNames.get(1)); + + assertEquals(lookupFields.size(), 2); + AttributeSet intersection = lookupFields.get(0).intersect(lookupFields.get(1)); + assertEquals(AttributeSet.EMPTY, intersection); + } + public void testLookupJoinIndexMode() { assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V8.isEnabled());