Merge remote-tracking branch 'upstream/8.x' into 118585_backport_8x

docs/changelog/118823.yaml

@@ -0,0 +1,5 @@
+pr: 118823
+summary: Fix attribute set equals
+area: ES|QL
+type: bug
+issues: []

docs/reference/connector/docs/connectors-sharepoint-online.asciidoc

@@ -133,6 +133,58 @@ The application name will appear in the Title box.
 </AppPermissionRequests>
 ----
 
+[discrete#es-connectors-sharepoint-online-sites-selected-permissions]
+====== Granting `Sites.Selected` permissions
+
+To configure `Sites.Selected` permissions, follow these steps in the Azure Active Directory portal. These permissions enable precise access control to specific SharePoint sites.
+
+. Sign in to the https://portal.azure.com/[Azure Active Directory portal^].
+. Navigate to **App registrations** and locate the application created for the connector.
+. Under **API permissions**, click **Add permission**.
+. Select **Microsoft Graph** > **Application permissions**, then add `Sites.Selected`.
+. Click **Grant admin consent** to approve the permission.
+
+[TIP]
+====
+Refer to the official https://learn.microsoft.com/en-us/graph/permissions-reference[Microsoft documentation] for managing permissions in Azure AD.
+====
+
+To assign access to specific SharePoint sites using `Sites.Selected`:
+
+. Use Microsoft Graph Explorer or PowerShell to grant access.
+. To fetch the site ID, run the following Graph API query:
++
+[source, http]
+----
+GET https://graph.microsoft.com/v1.0/sites?select=webUrl,Title,Id&$search="<Name of the site>*"
+----
++
+This will return the `id` of the site.
+
+. Use the `id` to assign read or write access:
++
+[source, http]
+----
+POST https://graph.microsoft.com/v1.0/sites/<siteId>/permissions
+{
+    "roles": ["read"], // or "write"
+    "grantedToIdentities": [
+        {
+            "application": {
+                "id": "<App_Client_ID>",
+                "displayName": "<App_Display_Name>"
+            }
+        }
+    ]
+}
+----
+
+[NOTE]
+====
+When using the `Comma-separated list of sites` configuration field, ensure the sites specified match those granted `Sites.Selected` permission in SharePoint.
+If the `Comma-separated list of sites` field is set to `*` or the `Enumerate all sites` toggle is enabled, the connector will attempt to access all sites. This requires broader permissions, which are not supported with `Sites.Selected`.
+====
+
 .Graph API permissions
 ****
 Microsoft recommends using Graph API for all operations with Sharepoint Online. Graph API is well-documented and more efficient at fetching data, which helps avoid throttling.
@@ -594,6 +646,59 @@ The application name will appear in the Title box.
 </AppPermissionRequests>
 ----
 
+[discrete#es-connectors-sharepoint-online-sites-selected-permissions-self-managed]
+====== Granting `Sites.Selected` permissions
+
+To configure `Sites.Selected` permissions, follow these steps in the Azure Active Directory portal. These permissions enable precise access control to specific SharePoint sites.
+
+. Sign in to the https://portal.azure.com/[Azure Active Directory portal^].
+. Navigate to **App registrations** and locate the application created for the connector.
+. Under **API permissions**, click **Add permission**.
+. Select **Microsoft Graph** > **Application permissions**, then add `Sites.Selected`.
+. Click **Grant admin consent** to approve the permission.
+
+[TIP]
+====
+Refer to the official https://learn.microsoft.com/en-us/graph/permissions-reference[Microsoft documentation] for managing permissions in Azure AD.
+====
+
+
+To assign access to specific SharePoint sites using `Sites.Selected`:
+
+. Use Microsoft Graph Explorer or PowerShell to grant access.
+. To fetch the site ID, run the following Graph API query:
++
+[source, http]
+----
+GET https://graph.microsoft.com/v1.0/sites?select=webUrl,Title,Id&$search="<Name of the site>*"
+----
++
+This will return the `id` of the site.
+
+. Use the `id` to assign read or write access:
++
+[source, http]
+----
+POST https://graph.microsoft.com/v1.0/sites/<siteId>/permissions
+{
+    "roles": ["read"], // or "write"
+    "grantedToIdentities": [
+        {
+            "application": {
+                "id": "<App_Client_ID>",
+                "displayName": "<App_Display_Name>"
+            }
+        }
+    ]
+}
+----
+
+[NOTE]
+====
+When using the `Comma-separated list of sites` configuration field, ensure the sites specified match those granted `Sites.Selected` permission in SharePoint.
+If the `Comma-separated list of sites` field is set to `*` or the `Enumerate all sites` toggle is enabled, the connector will attempt to access all sites. This requires broader permissions, which are not supported with `Sites.Selected`.
+====
+
 .Graph API permissions
 ****
 Microsoft recommends using Graph API for all operations with Sharepoint Online. Graph API is well-documented and more efficient at fetching data, which helps avoid throttling.

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecker.java

@@ -26,7 +26,6 @@
 import java.util.Map;
 import java.util.Optional;
 
-import static org.elasticsearch.xpack.core.ml.MachineLearningField.MIN_CHECKED_SUPPORTED_SNAPSHOT_VERSION;
 import static org.elasticsearch.xpack.core.ml.MachineLearningField.MIN_REPORTED_SUPPORTED_SNAPSHOT_VERSION;
 
 public class MlDeprecationChecker implements DeprecationChecker {
@@ -69,7 +68,7 @@ static Optional<DeprecationIssue> checkDataFeedAggregations(DatafeedConfig dataf
     }
 
     static Optional<DeprecationIssue> checkModelSnapshot(ModelSnapshot modelSnapshot) {
-        if (modelSnapshot.getMinVersion().before(MIN_CHECKED_SUPPORTED_SNAPSHOT_VERSION)) {
+        if (modelSnapshot.getMinVersion().before(MIN_REPORTED_SUPPORTED_SNAPSHOT_VERSION)) {
             StringBuilder details = new StringBuilder(
                 String.format(
                     Locale.ROOT,

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeSet.java

@@ -174,8 +174,12 @@ public Stream<Attribute> parallelStream() {
     }
 
     @Override
-    public boolean equals(Object o) {
-        return delegate.equals(o);
+    public boolean equals(Object obj) {
+        if (obj instanceof AttributeSet as) {
+            obj = as.delegate;
+        }
+
+        return delegate.equals(obj);
     }
 
     @Override

x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java

@@ -30,7 +30,7 @@
 
 public class AttributeMapTests extends ESTestCase {
 
-    private static Attribute a(String name) {
+    static Attribute a(String name) {
         return new UnresolvedAttribute(Source.EMPTY, name);
     }
 

x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeSetTests.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.List;
+
+import static org.elasticsearch.xpack.esql.core.expression.AttributeMapTests.a;
+
+public class AttributeSetTests extends ESTestCase {
+
+    public void testEquals() {
+        Attribute a1 = a("1");
+        Attribute a2 = a("2");
+
+        AttributeSet first = new AttributeSet(List.of(a1, a2));
+        assertEquals(first, first);
+
+        AttributeSet second = new AttributeSet();
+        second.add(a1);
+        second.add(a2);
+
+        assertEquals(first, second);
+        assertEquals(second, first);
+
+        AttributeSet third = new AttributeSet();
+        third.add(a("1"));
+        third.add(a("2"));
+
+        assertNotEquals(first, third);
+        assertNotEquals(third, first);
+
+        assertEquals(AttributeSet.EMPTY, AttributeSet.EMPTY);
+        assertEquals(AttributeSet.EMPTY, first.intersect(third));
+        assertEquals(third.intersect(first), AttributeSet.EMPTY);
+    }
+}

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java

@@ -362,7 +362,7 @@ private static void load(RestClient client, TestsDataset dataset, Logger logger,
         if (mapping == null) {
             throw new IllegalArgumentException("Cannot find resource " + mappingName);
         }
-        final String dataName = "/" + dataset.dataFileName;
+        final String dataName = "/data/" + dataset.dataFileName;
         URL data = CsvTestsDataLoader.class.getResource(dataName);
         if (data == null) {
             throw new IllegalArgumentException("Cannot find resource " + dataName);

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java

@@ -423,7 +423,7 @@ private static CsvTestsDataLoader.TestsDataset testsDataset(LogicalPlan parsed)
     }
 
     private static TestPhysicalOperationProviders testOperationProviders(CsvTestsDataLoader.TestsDataset dataset) throws Exception {
-        var testData = loadPageFromCsv(CsvTests.class.getResource("/" + dataset.dataFileName()), dataset.typeMapping());
+        var testData = loadPageFromCsv(CsvTests.class.getResource("/data/" + dataset.dataFileName()), dataset.typeMapping());
         return new TestPhysicalOperationProviders(testData.v1(), testData.v2());
     }
 

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java

@@ -26,6 +26,7 @@
 import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
@@ -2197,6 +2198,36 @@ public void testLookupJoinUnknownField() {
         assertThat(e.getMessage(), containsString(errorMessage3 + "right side of join"));
     }
 
+    public void testMultipleLookupJoinsGiveDifferentAttributes() {
+        assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V8.isEnabled());
+
+        // The field attributes that get contributed by different LOOKUP JOIN commands must have different name ids,
+        // even if they have the same names. Otherwise, things like dependency analysis - like in PruneColumns - cannot work based on
+        // name ids and shadowing semantics proliferate into all kinds of optimizer code.
+
+        String query = "FROM test"
+            + "| EVAL language_code = languages"
+            + "| LOOKUP JOIN languages_lookup ON language_code"
+            + "| LOOKUP JOIN languages_lookup ON language_code";
+        LogicalPlan analyzedPlan = analyze(query);
+
+        List<AttributeSet> lookupFields = new ArrayList<>();
+        List<Set<String>> lookupFieldNames = new ArrayList<>();
+        analyzedPlan.forEachUp(EsRelation.class, esRelation -> {
+            if (esRelation.indexMode() == IndexMode.LOOKUP) {
+                lookupFields.add(esRelation.outputSet());
+                lookupFieldNames.add(esRelation.outputSet().stream().map(NamedExpression::name).collect(Collectors.toSet()));
+            }
+        });
+
+        assertEquals(lookupFieldNames.size(), 2);
+        assertEquals(lookupFieldNames.get(0), lookupFieldNames.get(1));
+
+        assertEquals(lookupFields.size(), 2);
+        AttributeSet intersection = lookupFields.get(0).intersect(lookupFields.get(1));
+        assertEquals(AttributeSet.EMPTY, intersection);
+    }
+
     public void testLookupJoinIndexMode() {
         assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V8.isEnabled());