Merge branch 'main' into skip-masking-on-cardinality-agg

.github/CODEOWNERS

@@ -1 +1 @@
-* @cliu123 @cwperks @DarshitChanpura @peternied @RyanL1997 @scrawfor99 @reta @willyborankin
+* @cwperks @DarshitChanpura @nibix @peternied @RyanL1997 @stephen-crawford @reta @willyborankin

.github/pull_request_template.md

@@ -7,7 +7,7 @@
 ### Issues Resolved
 [List any issues this PR will resolve]
 
-Is this a backport? If so, please add backport PR # and/or commits #
+Is this a backport? If so, please add backport PR # and/or commits #, and remove `backport-failed` label from the original PR.
 
 Do these changes introduce new permission(s) to be displayed in the static dropdown on the front-end? If so, please open a draft PR in the security dashboards plugin and link the draft PR here
 

.github/workflows/ci.yml

@@ -86,8 +86,9 @@ jobs:
           attempt_limit: 5
           attempt_delay: 2000
           action: codecov/codecov-action@v4
+          env: |
+            CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
           with: |
-            token: ${{ secrets.CODECOV_TOKEN }}
             fail_ci_if_error: true
             verbose: true
 

MAINTAINERS.md

@@ -13,21 +13,22 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 ## Current Maintainers
 
 | Maintainer       | GitHub ID                                             | Affiliation |
-| ---------------- | ----------------------------------------------------- | ----------- |
-| Chang Liu        | [cliu123](https://github.com/cliu123)                 | Amazon      |
+|------------------|-------------------------------------------------------|-------------|
 | Darshit Chanpura | [DarshitChanpura](https://github.com/DarshitChanpura) | Amazon      |
 | Peter Nied       | [peternied](https://github.com/peternied)             | Amazon      |
 | Craig Perkins    | [cwperks](https://github.com/cwperks)                 | Amazon      |
 | Ryan Liang       | [RyanL1997](https://github.com/RyanL1997)             | Amazon      |
-| Stephen Crawford | [scrawfor99](https://github.com/scrawfor99)           | Amazon      |
+| Stephen Crawford | [scrawfor99](https://github.com/stephen-crawford)     | Amazon      |
 | Andriy Redko     | [reta](https://github.com/reta)                       | Aiven       |
 | Andrey Pleskach  | [willyborankin](https://github.com/willyborankin)     | Aiven       |
+| Nils Bandener    | [nibix](https://github.com/nibix)                     | Eliatra     |
 
 ## Emeritus
 
-| Maintainer    | GitHub ID                                           | Affiliation |
-| ------------- | --------------------------------------------------- | ----------- |
-| Dave Lago     | [davidlago](https://github.com/davidlago)           | Contributor |
+| Maintainer | GitHub ID                                 | Affiliation |
+|------------|-------------------------------------------|-------------|
+| Dave Lago  | [davidlago](https://github.com/davidlago) | Contributor |
+| Chang Liu  | [cliu123](https://github.com/cliu123)     | Amazon      |
 
 ## Practices
 

build.gradle

@@ -27,13 +27,12 @@ buildscript {
         common_utils_version = System.getProperty("common_utils.version", '3.0.0.0-SNAPSHOT')
 
         kafka_version  = '3.7.1'
-        apache_cxf_version = '4.0.4'
         open_saml_version = '4.3.2'
         one_login_java_saml = '2.9.0'
         jjwt_version = '0.12.6'
         guava_version = '32.1.3-jre'
         jaxb_version = '2.3.9'
-        spring_version = '5.3.37'
+        spring_version = '5.3.39'
 
         if (buildVersionQualifier) {
             opensearch_build += "-${buildVersionQualifier}"
@@ -65,8 +64,8 @@ plugins {
     id 'maven-publish'
     id 'com.diffplug.spotless' version '6.25.0'
     id 'checkstyle'
-    id 'com.netflix.nebula.ospackage' version "11.9.1"
-    id "org.gradle.test-retry" version "1.5.9"
+    id 'com.netflix.nebula.ospackage' version "11.10.0"
+    id "org.gradle.test-retry" version "1.5.10"
     id 'eclipse'
     id "com.github.spotbugs" version "5.2.5"
     id "com.google.osdetector" version "1.7.3"
@@ -470,7 +469,7 @@ bundlePlugin {
 configurations {
     all {
         resolutionStrategy {
-            force 'commons-codec:commons-codec:1.17.0'
+            force 'commons-codec:commons-codec:1.17.1'
             force 'org.slf4j:slf4j-api:1.7.36'
             force 'org.scala-lang:scala-library:2.13.14'
             force "com.fasterxml.jackson:jackson-bom:${versions.jackson}"
@@ -483,7 +482,7 @@ configurations {
             force "io.netty:netty-transport:${versions.netty}"
             force "io.netty:netty-transport-native-unix-common:${versions.netty}"
             force "com.github.luben:zstd-jni:${versions.zstd}"
-            force "org.xerial.snappy:snappy-java:1.1.10.5"
+            force "org.xerial.snappy:snappy-java:1.1.10.6"
             force "com.google.guava:guava:${guava_version}"
 
             // for spotbugs dependency conflict
@@ -496,9 +495,9 @@ configurations {
             // For integrationTest
             force "org.apache.httpcomponents:httpclient:4.5.14"
             force "org.apache.httpcomponents:httpcore:4.4.16"
-            force "com.google.errorprone:error_prone_annotations:2.28.0"
-            force "org.checkerframework:checker-qual:3.45.0"
-            force "ch.qos.logback:logback-classic:1.5.6"
+            force "com.google.errorprone:error_prone_annotations:2.30.0"
+            force "org.checkerframework:checker-qual:3.46.0"
+            force "ch.qos.logback:logback-classic:1.5.7"
         }
     }
 
@@ -578,7 +577,7 @@ dependencies {
     implementation "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
     implementation "com.google.guava:guava:${guava_version}"
     implementation 'org.greenrobot:eventbus-java:3.3.1'
-    implementation 'commons-cli:commons-cli:1.8.0'
+    implementation 'commons-cli:commons-cli:1.9.0'
     implementation "org.bouncycastle:bcprov-jdk18on:${versions.bouncycastle}"
     implementation 'org.ldaptive:ldaptive:1.2.3'
     implementation 'com.nimbusds:nimbus-jose-jwt:9.40'
@@ -593,21 +592,17 @@ dependencies {
     implementation 'org.apache.commons:commons-collections4:4.4'
 
     //Password generation
-    implementation 'org.passay:passay:1.6.4'
+    implementation 'org.passay:passay:1.6.5'
 
     implementation "org.apache.kafka:kafka-clients:${kafka_version}"
 
     runtimeOnly 'net.minidev:accessors-smart:2.5.1'
 
-    runtimeOnly "org.apache.cxf:cxf-core:${apache_cxf_version}"
-    implementation "org.apache.cxf:cxf-rt-rs-json-basic:${apache_cxf_version}"
-    runtimeOnly "org.apache.cxf:cxf-rt-security:${apache_cxf_version}"
-
     runtimeOnly 'com.sun.activation:jakarta.activation:1.2.2'
     runtimeOnly 'com.eclipsesource.minimal-json:minimal-json:0.9.5'
-    runtimeOnly 'commons-codec:commons-codec:1.17.0'
-    runtimeOnly 'org.cryptacular:cryptacular:1.2.6'
-    compileOnly 'com.google.errorprone:error_prone_annotations:2.28.0'
+    runtimeOnly 'commons-codec:commons-codec:1.17.1'
+    runtimeOnly 'org.cryptacular:cryptacular:1.2.7'
+    compileOnly 'com.google.errorprone:error_prone_annotations:2.30.0'
     runtimeOnly 'com.sun.istack:istack-commons-runtime:4.2.0'
     runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.2'
     runtimeOnly 'org.ow2.asm:asm:9.7'
@@ -616,7 +611,7 @@ dependencies {
 
     //OpenSAML
     implementation 'net.shibboleth.utilities:java-support:8.4.2'
-    runtimeOnly "io.dropwizard.metrics:metrics-core:4.2.26"
+    runtimeOnly "io.dropwizard.metrics:metrics-core:4.2.27"
     implementation "com.onelogin:java-saml:${one_login_java_saml}"
     implementation "com.onelogin:java-saml-core:${one_login_java_saml}"
     implementation "org.opensaml:opensaml-core:${open_saml_version}"
@@ -644,14 +639,14 @@ dependencies {
     runtimeOnly 'org.lz4:lz4-java:1.8.0'
     runtimeOnly 'org.slf4j:slf4j-api:1.7.36'
     runtimeOnly "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
-    runtimeOnly 'org.xerial.snappy:snappy-java:1.1.10.5'
+    runtimeOnly 'org.xerial.snappy:snappy-java:1.1.10.6'
     runtimeOnly 'org.codehaus.woodstox:stax2-api:4.2.2'
     runtimeOnly "org.glassfish.jaxb:txw2:${jaxb_version}"
     runtimeOnly 'com.fasterxml.woodstox:woodstox-core:6.7.0'
     runtimeOnly 'org.apache.ws.xmlschema:xmlschema-core:2.3.1'
     runtimeOnly 'org.apache.santuario:xmlsec:2.3.4'
     runtimeOnly "com.github.luben:zstd-jni:${versions.zstd}"
-    runtimeOnly 'org.checkerframework:checker-qual:3.45.0'
+    runtimeOnly 'org.checkerframework:checker-qual:3.46.0'
     runtimeOnly "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}"
     runtimeOnly 'org.scala-lang.modules:scala-java8-compat_3:1.0.2'
 
@@ -684,9 +679,9 @@ dependencies {
     testImplementation 'commons-validator:commons-validator:1.9.0'
     testImplementation 'org.springframework.kafka:spring-kafka-test:2.9.13'
     testImplementation "org.springframework:spring-beans:${spring_version}"
-    testImplementation 'org.junit.jupiter:junit-jupiter:5.10.3'
-    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.3'
-    testImplementation('org.awaitility:awaitility:4.2.1') {
+    testImplementation 'org.junit.jupiter:junit-jupiter:5.11.0'
+    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.11.0'
+    testImplementation('org.awaitility:awaitility:4.2.2') {
         exclude(group: 'org.hamcrest', module: 'hamcrest')
     }
     // Only osx-x86_64, osx-aarch_64, linux-x86_64, linux-aarch_64, windows-x86_64 are available
@@ -729,7 +724,7 @@ dependencies {
     integrationTestImplementation 'org.hamcrest:hamcrest:2.2'
     integrationTestImplementation "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}"
     integrationTestImplementation "org.bouncycastle:bcutil-jdk18on:${versions.bouncycastle}"
-    integrationTestImplementation('org.awaitility:awaitility:4.2.1') {
+    integrationTestImplementation('org.awaitility:awaitility:4.2.2') {
         exclude(group: 'org.hamcrest', module: 'hamcrest')
     }
     integrationTestImplementation 'com.unboundid:unboundid-ldapsdk:4.0.14'
@@ -741,7 +736,7 @@ dependencies {
     integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5"
 
     //spotless
-    implementation('com.google.googlejavaformat:google-java-format:1.22.0') {
+    implementation('com.google.googlejavaformat:google-java-format:1.23.0') {
         exclude group: 'com.google.guava'
     }
 }

gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=a4b4158601f8636cdeeab09bd76afb640030bb5b144aafe261a5e8af027dc612
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-bin.zip
+distributionSha256Sum=5b9c5eb3f9fc2c94abaea57d90bd78747ca117ddbbf96c859d3741181a12bf2a
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

gradlew

@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# SPDX-License-Identifier: Apache-2.0
+#
 
 ##############################################################################
 #
@@ -55,7 +57,7 @@
 #       Darwin, MinGW, and NonStop.
 #
 #   (3) This script is generated from the Groovy template
-#       https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
 #       within the Gradle project.
 #
 #       You can find Gradle at https://github.com/gradle/gradle/.
@@ -84,7 +86,8 @@ done
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
 # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
-APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
+APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
+' "$PWD" ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum

gradlew.bat

@@ -13,6 +13,8 @@
 @rem See the License for the specific language governing permissions and
 @rem limitations under the License.
 @rem
+@rem SPDX-License-Identifier: Apache-2.0
+@rem
 
 @if "%DEBUG%"=="" @echo off
 @rem ##########################################################################

release-notes/opensearch-security.release-notes-1.3.18.0.md

@@ -5,3 +5,4 @@ Compatible with OpenSearch 1.3.18
 ### Maintenance
 
 * Bump bouncycastle to 1.78.1 and kafka to 3.7.0 ([#4437](https://github.com/opensearch-project/security/pull/4437))
+* Bump jose to address CVE ([#4549](https://github.com/opensearch-project/security/pull/4549))

release-notes/opensearch-security.release-notes-2.16.0.0.md

@@ -0,0 +1,41 @@
+## Version 2.16.0 Release Notes
+
+Compatible with OpenSearch and OpenSearch Dashboards version 2.16.0
+
+### Enhancements
+* Add support for PBKDF2 for password hashing & add support for configuring BCrypt and PBKDF2 ([#4524](https://github.com/opensearch-project/security/pull/4524))
+* Use SystemIndexRegistry from core to determine if request contains system indices ([#4471](https://github.com/opensearch-project/security/pull/4471))
+* Separated DLS/FLS privilege evaluation from action privilege evaluation ([#4490](https://github.com/opensearch-project/security/pull/4490))
+* Update PULL_REQUEST_TEMPLATE to include an API spec change in the checklist. ([#4533](https://github.com/opensearch-project/security/pull/4533))
+* Update PATCH API to fail validation if nothing changes ([#4530](https://github.com/opensearch-project/security/pull/4530))
+* Refactor InternalUsers REST API test ([#4481](https://github.com/opensearch-project/security/pull/4481))
+* Refactor Role Mappings REST API test ([#4450](https://github.com/opensearch-project/security/pull/4450))
+* Remove special handling for do_not_fail_on_forbidden on cluster actions ([#4486](https://github.com/opensearch-project/security/pull/4486))
+* Add Tenants REST API test and partial fix ([#4166](https://github.com/opensearch-project/security/pull/4166))
+* Refactor Roles REST API test and partial fix #4166 ([#4433](https://github.com/opensearch-project/security/pull/4433))
+* New algorithm for resolving action groups ([#4448](https://github.com/opensearch-project/security/pull/4448))
+* Check block request only if system index ([#4430](https://github.com/opensearch-project/security/pull/4430))
+* Replaced uses of SecurityRoles by Set<String> mappedRoles where the SecurityRoles functionality is not needed ([#4432](https://github.com/opensearch-project/security/pull/4432))
+
+### Bug Fixes
+* Fixed test failures in FlsAndFieldMaskingTests ([#4548](https://github.com/opensearch-project/security/pull/4548))
+* Typo in securityadmin.sh hint ([#4526](https://github.com/opensearch-project/security/pull/4526))
+* Fix NPE getting metaFields from mapperService on a close index request ([#4497](https://github.com/opensearch-project/security/pull/4497))
+* Fixes flaky integration tests ([#4452](https://github.com/opensearch-project/security/pull/4452))
+
+### Maintenance
+* Remove unused dependancy Apache CXF ([#4580](https://github.com/opensearch-project/security/pull/4580))
+* Remove unnecessary return statements ([#4558](https://github.com/opensearch-project/security/pull/4558))
+* Pass set to SystemIndexRegistry.matchesSystemIndexPattern ([#4569](https://github.com/opensearch-project/security/pull/4569))
+* Refactor and update existing ml roles ([#4151](https://github.com/opensearch-project/security/pull/4151))
+* Replace JUnit assertEquals() with Hamcrest matchers assertThat() ([#4544](https://github.com/opensearch-project/security/pull/4544))
+* Update Gradle to 8.9 ([#4553](https://github.com/opensearch-project/security/pull/4553))
+* Bump org.checkerframework:checker-qual from 3.44.0 to 3.45.0 ([#4531](https://github.com/opensearch-project/security/pull/4531))
+* Add security analytics threat intel action  ([#4498](https://github.com/opensearch-project/security/pull/4498))
+* Bump kafka_version from 3.7.0 to 3.7.1 ([#4501](https://github.com/opensearch-project/security/pull/4501))
+* Bump org.junit.jupiter:junit-jupiter from 5.10.2 to 5.10.3 ([#4503](https://github.com/opensearch-project/security/pull/4503))
+* Bump com.fasterxml.woodstox:woodstox-core from 6.6.2 to 6.7.0 ([#4483](https://github.com/opensearch-project/security/pull/4483))
+* Bump jjwt_version from 0.12.5 to 0.12.6 ([#4484](https://github.com/opensearch-project/security/pull/4484))
+* Bump org.eclipse.platform:org.eclipse.core.runtime from 3.31.0 to 3.3.1.100 ([#4467](https://github.com/opensearch-project/security/pull/4467))
+* Bump spring_version from 5.3.36 to 5.3.37 ([#4466](https://github.com/opensearch-project/security/pull/4466))
+* Update to Gradle 8.8 ([#4459](https://github.com/opensearch-project/security/pull/4459))