Merge remote-tracking branch 'upstream/main' into saml-anony-bug-fix

DarshitChanpura · Mar 26, 2024 · e3384d2 · e3384d2
2 parents 3aecdc6 + a731e62
commit e3384d2
Show file tree

Hide file tree

Showing 38 changed files with 1,311 additions and 110 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -81,7 +81,7 @@ jobs:
         working-directory: downloaded-artifacts
 
       - name: Upload Coverage with retry
-        uses: Wandalen/wretry.action@v1.4.8
+        uses: Wandalen/wretry.action@v2.1.0
         with:
           attempt_limit: 5
           attempt_delay: 2000

diff --git a/.github/workflows/plugin_install.yml b/.github/workflows/plugin_install.yml
@@ -40,7 +40,7 @@ jobs:
         shell: bash
 
       - name: Run Opensearch with A Single Plugin
-        uses: derek-ho/start-opensearch@v2
+        uses: derek-ho/start-opensearch@v3
         with:
           opensearch-version: ${{ env.OPENSEARCH_VERSION }}
           plugins: "file:$(pwd)/${{ env.PLUGIN_NAME }}.zip"

diff --git a/build.gradle b/build.gradle
@@ -26,13 +26,13 @@ buildscript {
 
         common_utils_version = System.getProperty("common_utils.version", '3.0.0.0-SNAPSHOT')
         kafka_version  = '3.7.0'
-        apache_cxf_version = '4.0.3'
-        open_saml_version = '4.3.0'
+        apache_cxf_version = '4.0.4'
+        open_saml_version = '4.3.1'
         one_login_java_saml = '2.9.0'
         jjwt_version = '0.12.5'
         guava_version = '32.1.3-jre'
         jaxb_version = '2.3.9'
-        spring_version = '5.3.32'
+        spring_version = '5.3.33'
 
         if (buildVersionQualifier) {
             opensearch_build += "-${buildVersionQualifier}"
@@ -494,12 +494,12 @@ configurations {
             force "org.apache.commons:commons-lang3:${versions.commonslang}"
 
             // for spotless transitive dependency CVE
-            force "org.eclipse.platform:org.eclipse.core.runtime:3.30.0"
+            force "org.eclipse.platform:org.eclipse.core.runtime:3.31.0"
 
             // For integrationTest
             force "org.apache.httpcomponents:httpclient:4.5.14"
             force "org.apache.httpcomponents:httpcore:4.4.16"
-            force "com.google.errorprone:error_prone_annotations:2.25.0"
+            force "com.google.errorprone:error_prone_annotations:2.26.1"
             force "org.checkerframework:checker-qual:3.42.0"
             force "ch.qos.logback:logback-classic:1.5.3"
         }
@@ -599,7 +599,7 @@ dependencies {
 
     implementation "org.apache.kafka:kafka-clients:${kafka_version}"
 
-    runtimeOnly 'net.minidev:accessors-smart:2.5.0'
+    runtimeOnly 'net.minidev:accessors-smart:2.5.1'
 
     runtimeOnly "org.apache.cxf:cxf-core:${apache_cxf_version}"
     implementation "org.apache.cxf:cxf-rt-rs-json-basic:${apache_cxf_version}"
@@ -609,15 +609,15 @@ dependencies {
     runtimeOnly 'com.eclipsesource.minimal-json:minimal-json:0.9.5'
     runtimeOnly 'commons-codec:commons-codec:1.16.1'
     runtimeOnly 'org.cryptacular:cryptacular:1.2.6'
-    compileOnly 'com.google.errorprone:error_prone_annotations:2.25.0'
+    compileOnly 'com.google.errorprone:error_prone_annotations:2.26.1'
     runtimeOnly 'com.sun.istack:istack-commons-runtime:4.2.0'
     runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.2'
-    runtimeOnly 'org.ow2.asm:asm:9.6'
+    runtimeOnly 'org.ow2.asm:asm:9.7'
 
     testImplementation 'org.apache.camel:camel-xmlsecurity:3.22.1'
 
     //OpenSAML
-    implementation 'net.shibboleth.utilities:java-support:8.4.0'
+    implementation 'net.shibboleth.utilities:java-support:8.4.1'
     implementation "com.onelogin:java-saml:${one_login_java_saml}"
     implementation "com.onelogin:java-saml-core:${one_login_java_saml}"
     implementation "org.opensaml:opensaml-core:${open_saml_version}"
@@ -688,7 +688,7 @@ dependencies {
     testImplementation "org.springframework:spring-beans:${spring_version}"
     testImplementation 'org.junit.jupiter:junit-jupiter:5.10.2'
     testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.2'
-    testImplementation('org.awaitility:awaitility:4.2.0') {
+    testImplementation('org.awaitility:awaitility:4.2.1') {
         exclude(group: 'org.hamcrest', module: 'hamcrest')
     }
     // Only osx-x86_64, osx-aarch_64, linux-x86_64, linux-aarch_64, windows-x86_64 are available
@@ -706,7 +706,7 @@ dependencies {
     testRuntimeOnly 'org.scala-lang:scala-library:2.13.13'
     testRuntimeOnly 'com.yammer.metrics:metrics-core:2.2.0'
     testRuntimeOnly 'com.typesafe.scala-logging:scala-logging_3:3.9.5'
-    testRuntimeOnly('org.apache.zookeeper:zookeeper:3.9.1') {
+    testRuntimeOnly('org.apache.zookeeper:zookeeper:3.9.2') {
         exclude(group:'ch.qos.logback', module: 'logback-classic' )
         exclude(group:'ch.qos.logback', module: 'logback-core' )
     }
@@ -731,7 +731,7 @@ dependencies {
     integrationTestImplementation 'org.hamcrest:hamcrest:2.2'
     integrationTestImplementation "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}"
     integrationTestImplementation "org.bouncycastle:bcutil-jdk18on:${versions.bouncycastle}"
-    integrationTestImplementation('org.awaitility:awaitility:4.2.0') {
+    integrationTestImplementation('org.awaitility:awaitility:4.2.1') {
         exclude(group: 'org.hamcrest', module: 'hamcrest')
     }
     integrationTestImplementation 'com.unboundid:unboundid-ldapsdk:4.0.14'

diff --git a/config/roles.yml b/config/roles.yml
@@ -271,6 +271,15 @@ cross_cluster_search_remote_full_access:
         - 'indices:admin/shards/search_shards'
         - 'indices:data/read/search'
 
+# Allow users to operate query assistant
+ml_query_assistant_access:
+  reserved: true
+  cluster_permissions:
+    - 'cluster:admin/opensearch/ml/execute'
+    - 'cluster:admin/opensearch/ml/memory/conversation/create'
+    - 'cluster:admin/opensearch/ml/memory/interaction/create'
+    - 'cluster:admin/opensearch/ml/predict'
+
 # Allow users to read ML stats/models/tasks
 ml_read_access:
   reserved: true

diff --git a/release-notes/opensearch-security.release-notes-2.13.0.0.md b/release-notes/opensearch-security.release-notes-2.13.0.0.md
@@ -0,0 +1,30 @@
+## 2024-03-19 Version 2.13.0.0
+
+Compatible with OpenSearch 2.13.0
+
+### Enhancements
+* Admin role for Query insights plugin ([#4022](https://github.com/opensearch-project/security/pull/4022))
+* Add query assistant role and new ml system indices ([#4143](https://github.com/opensearch-project/security/pull/4143))
+* Redact sensitive configuration values when retrieving security configuration ([#4028](https://github.com/opensearch-project/security/pull/4028))
+* v2.12 update roles.yml with new API for experimental alerting plugin feature ([#4035](https://github.com/opensearch-project/security/pull/4035))
+* Add deprecate message that TLSv1 and TLSv1.1 support will be removed in the next major version ([#4083](https://github.com/opensearch-project/security/pull/4083))
+* Log password requirement details in demo environment ([#4082](https://github.com/opensearch-project/security/pull/4082))
+* Redact sensitive URL parameters from audit logging ([#4070](https://github.com/opensearch-project/security/pull/4070))
+* Fix unconsumed parameter exception when authenticating with jwtUrlParameter ([#4065](https://github.com/opensearch-project/security/pull/4065))
+* Regenerates root-ca, kirk and esnode certificates to address already expired root ca certificate ([#4066](https://github.com/opensearch-project/security/pull/4066))
+* Add exclude_roles configuration parameter to LDAP authorization backend ([#4043](https://github.com/opensearch-project/security/pull/4043))
+
+### Maintenance
+* Add exlusion for logback-core to resolve CVE-2023-6378 ([#4050](https://github.com/opensearch-project/security/pull/4050))
+* Bump com.netflix.nebula.ospackage from 11.7.0 to 11.8.1 ([#4041](https://github.com/opensearch-project/security/pull/4041), [#4075](https://github.com/opensearch-project/security/pull/4075))
+* Bump Wandalen/wretry.action from 1.3.0 to 1.4.10 ([#4042](https://github.com/opensearch-project/security/pull/4042), [#4092](https://github.com/opensearch-project/security/pull/4092), [#4108](https://github.com/opensearch-project/security/pull/4108), [#4135](https://github.com/opensearch-project/security/pull/4135))
+* Bump spring_version from 5.3.31 to 5.3.33 ([#4058](https://github.com/opensearch-project/security/pull/4058), [#4131](https://github.com/opensearch-project/security/pull/4131))
+* Bump org.scala-lang:scala-library from 2.13.12 to 2.13.13 ([#4076](https://github.com/opensearch-project/security/pull/4076))
+* Bump com.google.googlejavaformat:google-java-format from 1.19.1 to 1.21.0 ([#4078](https://github.com/opensearch-project/security/pull/4078), [#4110](https://github.com/opensearch-project/security/pull/4110))
+* Bump ch.qos.logback:logback-classic from 1.2.13 to 1.5.3 ([#4091](https://github.com/opensearch-project/security/pull/4091), [#4111](https://github.com/opensearch-project/security/pull/4111))
+* Bump com.fasterxml.woodstox:woodstox-core from 6.6.0 to 6.6.1 ([#4093](https://github.com/opensearch-project/security/pull/4093))
+* Bump kafka_version from 3.5.1 to 3.7.0 ([#4095](https://github.com/opensearch-project/security/pull/4095))
+* Bump jakarta.xml.bind:jakarta.xml.bind-api from 4.0.1 to 4.0.2 ([#4109](https://github.com/opensearch-project/security/pull/4109))
+* Bump org.apache.zookeeper:zookeeper from 3.9.1. to 3.9.2 ([#4130](https://github.com/opensearch-project/security/pull/4130))
+* Bump org.awaitility:awaitility from 4.2.0 to 4.2.1 ([#4133](https://github.com/opensearch-project/security/pull/4133))
+* Bump com.google.errorprone:error_prone_annotations from 2.25.0 to 2.26.1 ([#4132](https://github.com/opensearch-project/security/pull/4132))
diff --git a/src/integrationTest/java/org/opensearch/security/DefaultConfigurationTests.java b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationTests.java
@@ -11,17 +11,21 @@
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+import com.fasterxml.jackson.databind.JsonNode;
 import org.apache.commons.io.FileUtils;
 import org.awaitility.Awaitility;
 import org.junit.AfterClass;
 import org.junit.ClassRule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
+import org.opensearch.test.framework.TestSecurityConfig.User;
 import org.opensearch.test.framework.cluster.ClusterManager;
 import org.opensearch.test.framework.cluster.LocalCluster;
 import org.opensearch.test.framework.cluster.TestRestClient;
@@ -32,16 +36,16 @@
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.not;
 
 @RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class)
 @ThreadLeakScope(ThreadLeakScope.Scope.NONE)
 public class DefaultConfigurationTests {
 
     private final static Path configurationFolder = ConfigurationFiles.createConfigurationDirectory();
-    public static final String ADMIN_USER_NAME = "admin";
-    public static final String DEFAULT_PASSWORD = "secret";
-    public static final String NEW_USER = "new-user";
-    public static final String LIMITED_USER = "limited-user";
+    private static final User ADMIN_USER = new User("admin");
+    private static final User NEW_USER = new User("new-user");
+    private static final User LIMITED_USER = new User("limited-user");
 
     @ClassRule
     public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.SINGLENODE)
@@ -64,15 +68,95 @@ public static void cleanConfigurationDirectory() throws IOException {
 
     @Test
     public void shouldLoadDefaultConfiguration() {
-        try (TestRestClient client = cluster.getRestClient(NEW_USER, DEFAULT_PASSWORD)) {
+        try (TestRestClient client = cluster.getRestClient(NEW_USER)) {
             Awaitility.await().alias("Load default configuration").until(() -> client.getAuthInfo().getStatusCode(), equalTo(200));
         }
-        try (TestRestClient client = cluster.getRestClient(ADMIN_USER_NAME, DEFAULT_PASSWORD)) {
-            client.confirmCorrectCredentials(ADMIN_USER_NAME);
+        try (TestRestClient client = cluster.getRestClient(ADMIN_USER)) {
+            client.confirmCorrectCredentials(ADMIN_USER.getName());
             HttpResponse response = client.get("_plugins/_security/api/internalusers");
             response.assertStatusCode(200);
             Map<String, Object> users = response.getBodyAs(Map.class);
-            assertThat(users, allOf(aMapWithSize(3), hasKey(ADMIN_USER_NAME), hasKey(NEW_USER), hasKey(LIMITED_USER)));
+            assertThat(
+                users,
+                allOf(aMapWithSize(3), hasKey(ADMIN_USER.getName()), hasKey(NEW_USER.getName()), hasKey(LIMITED_USER.getName()))
+            );
         }
     }
+
+    @Test
+    public void securityRolesUgrade() throws Exception {
+        try (var client = cluster.getRestClient(ADMIN_USER)) {
+            // Setup: Make sure the config is ready before starting modifications
+            Awaitility.await().alias("Load default configuration").until(() -> client.getAuthInfo().getStatusCode(), equalTo(200));
+
+            // Setup: Collect default roles after cluster start
+            final var expectedRoles = client.get("_plugins/_security/api/roles/");
+            final var expectedRoleNames = extractFieldNames(expectedRoles.getBodyAs(JsonNode.class));
+
+            // Verify: Before any changes, nothing to upgrade
+            final var upgradeCheck = client.get("_plugins/_security/api/_upgrade_check");
+            upgradeCheck.assertStatusCode(200);
+            assertThat(upgradeCheck.getBooleanFromJsonBody("/upgradeAvailable"), equalTo(false));
+
+            // Action: Select a role that is part of the defaults and delete that role
+            final var roleToDelete = "flow_framework_full_access";
+            client.delete("_plugins/_security/api/roles/" + roleToDelete).assertStatusCode(200);
+
+            // Action: Select a role that is part of the defaults and alter that role with removal, edits, and additions
+            final var roleToAlter = "flow_framework_read_access";
+            final var originalRoleConfig = client.get("_plugins/_security/api/roles/" + roleToAlter).getBodyAs(JsonNode.class);
+            final var alteredRoleReponse = client.patch("_plugins/_security/api/roles/" + roleToAlter, "[\n" + //
+                "  {\n" + //
+                "    \"op\": \"replace\",\n" + //
+                "    \"path\": \"/cluster_permissions\",\n" + //
+                "    \"value\": [\"a\", \"b\", \"c\"]\n" + //
+                "  },\n" + //
+                "  {\n" + //
+                "    \"op\": \"add\",\n" + //
+                "    \"path\": \"/index_permissions\",\n" + //
+                "    \"value\": [ {\n" + //
+                "        \"index_patterns\": [\"*\"],\n" + //
+                "        \"allowed_actions\": [\"*\"]\n" + //
+                "      }\n" + //
+                "    ]\n" + //
+                "  }\n" + //
+                "]");
+            alteredRoleReponse.assertStatusCode(200);
+            final var alteredRoleJson = alteredRoleReponse.getBodyAs(JsonNode.class);
+            assertThat(originalRoleConfig, not(equalTo(alteredRoleJson)));
+
+            // Verify: Confirm that the upgrade check detects the changes associated with both role resources
+            final var upgradeCheckAfterChanges = client.get("_plugins/_security/api/_upgrade_check");
+            upgradeCheckAfterChanges.assertStatusCode(200);
+            assertThat(
+                upgradeCheckAfterChanges.getTextArrayFromJsonBody("/upgradeActions/roles/add"),
+                equalTo(List.of("flow_framework_full_access"))
+            );
+            assertThat(
+                upgradeCheckAfterChanges.getTextArrayFromJsonBody("/upgradeActions/roles/modify"),
+                equalTo(List.of("flow_framework_read_access"))
+            );
+
+            // Action: Perform the upgrade to the roles configuration
+            final var performUpgrade = client.post("_plugins/_security/api/_upgrade_perform");
+            performUpgrade.assertStatusCode(200);
+            assertThat(performUpgrade.getTextArrayFromJsonBody("/upgrades/roles/add"), equalTo(List.of("flow_framework_full_access")));
+            assertThat(performUpgrade.getTextArrayFromJsonBody("/upgrades/roles/modify"), equalTo(List.of("flow_framework_read_access")));
+
+            // Verify: Same roles as the original state - the deleted role has been restored
+            final var afterUpgradeRoles = client.get("_plugins/_security/api/roles/");
+            final var afterUpgradeRolesNames = extractFieldNames(afterUpgradeRoles.getBodyAs(JsonNode.class));
+            assertThat(afterUpgradeRolesNames, equalTo(expectedRoleNames));
+
+            // Verify: Altered role was restored to its expected state
+            final var afterUpgradeAlteredRoleConfig = client.get("_plugins/_security/api/roles/" + roleToAlter).getBodyAs(JsonNode.class);
+            assertThat(originalRoleConfig, equalTo(afterUpgradeAlteredRoleConfig));
+        }
+    }
+
+    private Set<String> extractFieldNames(final JsonNode json) {
+        final var set = new HashSet<String>();
+        json.fieldNames().forEachRemaining(set::add);
+        return set;
+    }
 }
diff --git a/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoTest.java b/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoTest.java
@@ -17,6 +17,7 @@
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
+import org.opensearch.security.securityconf.impl.DashboardSignInOption;
 import org.opensearch.test.framework.TestSecurityConfig;
 import org.opensearch.test.framework.TestSecurityConfig.Role;
 import org.opensearch.test.framework.cluster.ClusterManager;
@@ -25,6 +26,7 @@
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
 import static org.opensearch.security.rest.DashboardsInfoAction.DEFAULT_PASSWORD_MESSAGE;
 import static org.opensearch.security.rest.DashboardsInfoAction.DEFAULT_PASSWORD_REGEX;
 import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL;
@@ -53,4 +55,14 @@ public void testDashboardsInfoValidationMessage() throws Exception {
             assertThat(response.getTextFromJsonBody("/password_validation_regex"), equalTo(DEFAULT_PASSWORD_REGEX));
         }
     }
+
+    @Test
+    public void testDashboardsInfoContainsSignInOptions() throws Exception {
+
+        try (TestRestClient client = cluster.getRestClient(DASHBOARDS_USER)) {
+            TestRestClient.HttpResponse response = client.get("_plugins/_security/dashboardsinfo");
+            assertThat(response.getStatusCode(), equalTo(HttpStatus.SC_OK));
+            assertThat(response.getTextArrayFromJsonBody("/sign_in_options").contains(DashboardSignInOption.BASIC.toString()), is(true));
+        }
+    }
 }
diff --git a/src/integrationTest/resources/roles.yml b/src/integrationTest/resources/roles.yml
@@ -17,3 +17,21 @@ user_limited-user__limited-role:
       allowed_actions:
         - "indices:data/read/get"
         - "indices:data/read/search"
+flow_framework_full_access:
+  cluster_permissions:
+    - 'cluster:admin/opensearch/flow_framework/*'
+    - 'cluster_monitor'
+  index_permissions:
+    - index_patterns:
+        - '*'
+      allowed_actions:
+        - 'indices:admin/aliases/get'
+        - 'indices:admin/mappings/get'
+        - 'indices_monitor'
+flow_framework_read_access:
+  cluster_permissions:
+    - 'cluster:admin/opensearch/flow_framework/workflow/get'
+    - 'cluster:admin/opensearch/flow_framework/workflow/search'
+    - 'cluster:admin/opensearch/flow_framework/workflow_state/get'
+    - 'cluster:admin/opensearch/flow_framework/workflow_state/search'
+    - 'cluster:admin/opensearch/flow_framework/workflow_step/get'
diff --git a/src/main/java/org/opensearch/security/auditlog/sink/Log4JSink.java b/src/main/java/org/opensearch/security/auditlog/sink/Log4JSink.java
@@ -27,7 +27,7 @@ public final class Log4JSink extends AuditLogSink {
 
     public Log4JSink(final String name, final Settings settings, final String settingsPrefix, AuditLogSink fallbackSink) {
         super(name, settings, settingsPrefix, fallbackSink);
-        loggerName = settings.get(settingsPrefix + ".log4j.logger_name", "sgaudit");
+        loggerName = settings.get(settingsPrefix + ".log4j.logger_name", "audit");
         auditLogger = LogManager.getLogger(loggerName);
         logLevel = Level.toLevel(settings.get(settingsPrefix + ".log4j.level", "INFO").toUpperCase());
         enabled = auditLogger.isEnabled(logLevel);