From ea2f9b21a90bec2678ad31c113c847872c955ea8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 08:05:20 -0500 Subject: [PATCH 001/204] Bump org.junit.jupiter:junit-jupiter-api from 5.10.0 to 5.10.1 (#3681) Bumps [org.junit.jupiter:junit-jupiter-api](https://github.com/junit-team/junit5) from 5.10.0 to 5.10.1.
Release notes

Sourced from org.junit.jupiter:junit-jupiter-api's releases.

JUnit 5.10.1 = Platform 1.10.1 + Jupiter 5.10.1 + Vintage 5.10.1

See Release Notes.

Full Changelog: https://github.com/junit-team/junit5/compare/r5.10.0...r5.10.1

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.junit.jupiter:junit-jupiter-api&package-manager=gradle&previous-version=5.10.0&new-version=5.10.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index ef4a33edb0..96a5fbf33c 100644 --- a/build.gradle +++ b/build.gradle @@ -681,7 +681,7 @@ dependencies { testImplementation 'org.springframework.kafka:spring-kafka-test:2.9.13' testImplementation 'org.springframework:spring-beans:5.3.30' testImplementation 'org.junit.jupiter:junit-jupiter:5.10.0' - testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.0' + testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.1' // Only osx-x86_64, osx-aarch_64, linux-x86_64, linux-aarch_64, windows-x86_64 are available if (osdetector.classifier in ["osx-x86_64", "osx-aarch_64", "linux-x86_64", "linux-aarch_64", "windows-x86_64"]) { testImplementation "io.netty:netty-tcnative-classes:2.0.61.Final" From af149372b8b59259811625dcccadb402c5c32bd5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 08:06:57 -0500 Subject: [PATCH 002/204] Bump com.google.googlejavaformat:google-java-format from 1.17.0 to 1.18.1 (#3684) Bumps [com.google.googlejavaformat:google-java-format](https://github.com/google/google-java-format) from 1.17.0 to 1.18.1.
Release notes

Sourced from com.google.googlejavaformat:google-java-format's releases.

v1.18.1

Changes

  • Fixed version number for Eclipse plugin (#744)

Full Changelog: https://github.com/google/google-java-format/compare/v1.18.0...v1.18.1

v1.18.0

Changes

Full Changelog: https://github.com/google/google-java-format/compare/v1.17.0...v1.18.0

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.googlejavaformat:google-java-format&package-manager=gradle&previous-version=1.17.0&new-version=1.18.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 96a5fbf33c..b455721ebe 100644 --- a/build.gradle +++ b/build.gradle @@ -732,7 +732,7 @@ dependencies { integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5" //spotless - implementation('com.google.googlejavaformat:google-java-format:1.17.0') { + implementation('com.google.googlejavaformat:google-java-format:1.18.1') { exclude group: 'com.google.guava' } } From 20d196ba46996ecaafacb638f85856a2d0de313f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 13:23:56 +0000 Subject: [PATCH 003/204] Bump com.nimbusds:nimbus-jose-jwt from 9.37 to 9.37.1 (#3682) Bumps [com.nimbusds:nimbus-jose-jwt](https://bitbucket.org/connect2id/nimbus-jose-jwt) from 9.37 to 9.37.1.
Changelog

Sourced from com.nimbusds:nimbus-jose-jwt's changelog.

version 1.0 (2012-03-01)

  • First version based on the OpenInfoCard JWT, JWS and JWE code base.

version 1.1 (2012-03-06)

  • Introduces type-safe enumeration of the JSON Web Algorithms (JWA).
  • Refactors the JWT class.

version 1.2 (2012-03-08)

  • Moves JWS and JWE code into separate classes.

version 1.3 (2012-03-09)

  • Switches to Apache Commons Codec for Base64URL encoding and decoding
  • Consolidates the crypto utilities within the package.
  • Introduces a JWT content serialiser class.

version 1.4 (2012-03-09)

  • Refactoring of JWT class and JUnit tests.

version 1.5 (2012-03-18)

  • Switches to JSON Smart for JSON serialisation and parsing.
  • Introduces claims set class with JSON objects, string, Base64URL and byte array views.

version 1.6 (2012-03-20)

  • Creates class for representing, serialising and parsing JSON Web Keys (JWK).
  • Introduces separate class for representing JWT headers.

version 1.7 (2012-04-01)

  • Introduces separate classes for plain, JWS and JWE headers.
  • Introduces separate classes for plain, signed and encrypted JWTs.
  • Removes the JWTContent class.
  • Removes password-based (PE820) encryption support.

version 1.8 (2012-04-03)

  • Adds support for the ZIP JWE header parameter.
  • Removes unsupported algorithms from the JWA enumeration.

version 1.9 (2012-04-03)

  • Renames JWEHeader.{get|set}EncryptionAlgorithm() to JWEHeader.{get|set}EncryptionMethod().

version 1.9.1 (2012-04-03)

  • Upgrades JSON Smart JAR to 1.1.1.

version 1.10 (2012-04-14)

  • Introduces serialize() method to base abstract JWT class.

version 1.11 (2012-05-13)

  • JWT.serialize() throws checked JWTException instead of

... (truncated)

Commits
  • 60caa26 [maven-release-plugin] prepare for next development iteration
  • 82a03c7 Fixes README.md MD list formatting
  • ac64737 Updates README.md formatting
  • 75960bf Updates README.md OpenID Federation 1.0 ref
  • 8d9e6f1 Fixes Payload JavaDoc
  • 02aacf0 Expands JWTClaimsSet tests
  • dbd5de4 exclude GSON's module-info.class from shaded jar
  • 61f93de Merged in master (pull request #110)
  • cf557b0 Merge branch 'master' of ssh://bitbucket.org/connect2id/nimbus-jose-jwt
  • a682550 Adds change log entry for iss #496 fix (PR)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.nimbusds:nimbus-jose-jwt&package-manager=gradle&previous-version=9.37&new-version=9.37.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index b455721ebe..a23d5aad88 100644 --- a/build.gradle +++ b/build.gradle @@ -572,7 +572,7 @@ dependencies { implementation 'commons-cli:commons-cli:1.6.0' implementation "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}" implementation 'org.ldaptive:ldaptive:1.2.3' - implementation 'com.nimbusds:nimbus-jose-jwt:9.37' + implementation 'com.nimbusds:nimbus-jose-jwt:9.37.1' //JWT implementation "io.jsonwebtoken:jjwt-api:${jjwt_version}" From 6f0f4d0c34c40c3abc45ecd4126c2819bbd077cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 15:25:27 +0000 Subject: [PATCH 004/204] Bump org.junit.jupiter:junit-jupiter from 5.10.0 to 5.10.1 (#3683) Bumps [org.junit.jupiter:junit-jupiter](https://github.com/junit-team/junit5) from 5.10.0 to 5.10.1.
Release notes

Sourced from org.junit.jupiter:junit-jupiter's releases.

JUnit 5.10.1 = Platform 1.10.1 + Jupiter 5.10.1 + Vintage 5.10.1

See Release Notes.

Full Changelog: https://github.com/junit-team/junit5/compare/r5.10.0...r5.10.1

Commits
  • e5f50d8 Release 5.10.1
  • ac86d18 Fix typo in AfterAll documentation
  • 388c5be Harmonize application of method and field filters in search algorithms
  • f82dd1e Apply field predicate before searching type hierarchy
  • 1d1eb85 Polishing
  • 5ce280e Update picocli to 4.7.5 and enable help width computation
  • fea05c3 Fix ConsoleLauncherTests and StandaloneTests
  • c556735 Use same expected files for all JDK versions
  • 808493a Run StandaloneTests for Java 8 under Java 8
  • 9ec5766 Unify messages about exit codes in StandaloneTests
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.junit.jupiter:junit-jupiter&package-manager=gradle&previous-version=5.10.0&new-version=5.10.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a23d5aad88..dcc8171263 100644 --- a/build.gradle +++ b/build.gradle @@ -680,7 +680,7 @@ dependencies { testImplementation 'commons-validator:commons-validator:1.7' testImplementation 'org.springframework.kafka:spring-kafka-test:2.9.13' testImplementation 'org.springframework:spring-beans:5.3.30' - testImplementation 'org.junit.jupiter:junit-jupiter:5.10.0' + testImplementation 'org.junit.jupiter:junit-jupiter:5.10.1' testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.1' // Only osx-x86_64, osx-aarch_64, linux-x86_64, linux-aarch_64, windows-x86_64 are available if (osdetector.classifier in ["osx-x86_64", "osx-aarch_64", "linux-x86_64", "linux-aarch_64", "windows-x86_64"]) { From 7588593db3d314558c0bd26654df43e113462cc8 Mon Sep 17 00:00:00 2001 From: Andrey Pleskach Date: Mon, 13 Nov 2023 21:41:00 +0100 Subject: [PATCH 005/204] Remove duplicate permissions (#3690) ### Description Permission: `permission java.util.PropertyPermission "*", "read,write";` was declared twice. Observed here: https://github.com/opensearch-project/security/pull/3671 I will backport it in my PR. ### Testing [Please provide details of testing done: unit testing, integration testing and manual testing] ### Check List - [ ] New functionality includes testing - [ ] New functionality has been documented - [ ] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). Signed-off-by: Andrey Pleskach --- plugin-security.policy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin-security.policy b/plugin-security.policy index 65b6b22fee..2969e47b04 100644 --- a/plugin-security.policy +++ b/plugin-security.policy @@ -34,6 +34,8 @@ grant { permission javax.security.auth.AuthPermission "modifyPrivateCredentials"; permission javax.security.auth.AuthPermission "doAs"; permission javax.security.auth.kerberos.ServicePermission "*","accept"; + + //SAML and internal plugin policy permission java.util.PropertyPermission "*","read,write"; //Enable when we switch to UnboundID LDAP SDK @@ -74,8 +76,6 @@ grant { //Enable this permission to debug unauthorized de-serialization attempt //permission java.io.SerializablePermission "enableSubstitution"; - //SAML policy - permission java.util.PropertyPermission "*", "read,write"; }; grant codeBase "${codebase.netty-common}" { From ee66199dead52754caff9fe7793b2362aa3a26d3 Mon Sep 17 00:00:00 2001 From: David Osorno <48450162+davidosorno@users.noreply.github.com> Date: Tue, 14 Nov 2023 07:21:21 -0800 Subject: [PATCH 006/204] Allow adding URL to the cluster (#3574) ### Description Allow adding URLs to the OpenSearch cluster with support for both http:// and https:// for external logs. --------- Signed-off-by: David Osorno --- .../security/httpclient/HttpClient.java | 32 +++++++++++++------ .../security/httpclient/HttpClientTest.java | 7 +++- .../endpoints/routing/configuration_valid.yml | 18 ++++++++++- 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/opensearch/security/httpclient/HttpClient.java b/src/main/java/org/opensearch/security/httpclient/HttpClient.java index 466dac2a82..8c31a5f9c9 100644 --- a/src/main/java/org/opensearch/security/httpclient/HttpClient.java +++ b/src/main/java/org/opensearch/security/httpclient/HttpClient.java @@ -13,6 +13,8 @@ import java.io.Closeable; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.security.KeyManagementException; import java.security.KeyStore; @@ -31,7 +33,6 @@ import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLParameters; -import com.google.common.collect.Lists; import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder; import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; @@ -51,7 +52,6 @@ import org.apache.hc.core5.ssl.SSLContexts; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; - import org.opensearch.action.index.IndexRequest; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.WriteRequest.RefreshPolicy; @@ -62,6 +62,8 @@ import org.opensearch.client.RestHighLevelClient; import org.opensearch.common.xcontent.XContentType; +import com.google.common.collect.Lists; + public class HttpClient implements Closeable { public static class HttpClientBuilder { @@ -176,14 +178,8 @@ private HttpClient( this.supportedCipherSuites = supportedCipherSuites; this.keystoreAlias = keystoreAlias; - HttpHost[] hosts = Arrays.stream(servers) - .map(s -> s.split(":")) - .map(s -> new HttpHost(ssl ? "https" : "http", s[0], Integer.parseInt(s[1]))) - .collect(Collectors.toList()) - .toArray(new HttpHost[0]); - + HttpHost[] hosts = createHosts(servers); RestClientBuilder builder = RestClient.builder(hosts); - // builder.setMaxRetryTimeoutMillis(10000); builder.setFailureListener(new RestClient.FailureListener() { @Override @@ -208,6 +204,24 @@ public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpCli rclient = new RestHighLevelClient(builder); } + private HttpHost[] createHosts(String[] servers) { + return Arrays.stream(servers).map(server -> { + try { + server = addSchemeBasedOnSSL(server); + URI uri = new URI(server); + return new HttpHost(uri.getScheme(), uri.getHost(), uri.getPort()); + } catch (URISyntaxException e) { + return null; + } + }).filter(Objects::nonNull).collect(Collectors.toList()).toArray(HttpHost[]::new); + } + + private String addSchemeBasedOnSSL(String server) { + server = server.replaceAll("https://|http://", ""); + String protocol = ssl ? "https://" : "http://"; + return protocol.concat(server); + } + public boolean index(final String content, final String index, final String type, final boolean refresh) { try { diff --git a/src/test/java/org/opensearch/security/httpclient/HttpClientTest.java b/src/test/java/org/opensearch/security/httpclient/HttpClientTest.java index 3da6ad3d7f..6a8db35d14 100644 --- a/src/test/java/org/opensearch/security/httpclient/HttpClientTest.java +++ b/src/test/java/org/opensearch/security/httpclient/HttpClientTest.java @@ -30,7 +30,10 @@ protected String getResourceFolder() { @Test public void testPlainConnection() throws Exception { - final Settings settings = Settings.builder().put("plugins.security.ssl.http.enabled", false).build(); + final Settings settings = Settings.builder() + .put("plugins.security.ssl.http.enabled", false) + .loadFromPath(FileHelper.getAbsoluteFilePathFromClassPath("auditlog/endpoints/routing/configuration_valid.yml")) + .build(); setup(Settings.EMPTY, new DynamicSecurityConfig(), settings); @@ -84,6 +87,7 @@ public void testSslConnection() throws Exception { .put(SSLConfigConstants.SECURITY_SSL_HTTP_KEYSTORE_ALIAS, "node-0") .put("plugins.security.ssl.http.keystore_filepath", FileHelper.getAbsoluteFilePathFromClassPath("auditlog/node-0-keystore.jks")) .put("plugins.security.ssl.http.truststore_filepath", FileHelper.getAbsoluteFilePathFromClassPath("auditlog/truststore.jks")) + .loadFromPath(FileHelper.getAbsoluteFilePathFromClassPath("auditlog/endpoints/routing/configuration_valid.yml")) .build(); setup(Settings.EMPTY, new DynamicSecurityConfig(), settings); @@ -123,6 +127,7 @@ public void testSslConnectionPKIAuth() throws Exception { .put(SSLConfigConstants.SECURITY_SSL_HTTP_KEYSTORE_ALIAS, "node-0") .put("plugins.security.ssl.http.keystore_filepath", FileHelper.getAbsoluteFilePathFromClassPath("auditlog/node-0-keystore.jks")) .put("plugins.security.ssl.http.truststore_filepath", FileHelper.getAbsoluteFilePathFromClassPath("auditlog/truststore.jks")) + .loadFromPath(FileHelper.getAbsoluteFilePathFromClassPath("auditlog/endpoints/routing/configuration_valid.yml")) .build(); setup(Settings.EMPTY, new DynamicSecurityConfig(), settings); diff --git a/src/test/resources/auditlog/endpoints/routing/configuration_valid.yml b/src/test/resources/auditlog/endpoints/routing/configuration_valid.yml index 046e4d6ee5..027ee6869e 100644 --- a/src/test/resources/auditlog/endpoints/routing/configuration_valid.yml +++ b/src/test/resources/auditlog/endpoints/routing/configuration_valid.yml @@ -15,7 +15,23 @@ plugins.security: endpoint2: type: external_opensearch config: - http_endpoints: ['localhost:9200','localhost:9201','localhost:9202'] + http_endpoints: [ + 'localhost', + 'localhost:9200', + 'localhost:9201', + 'localhost:9202', + 'localhost:9202/opensearch', + '127.0.0.1', + '127.0.0.1:9200', + '127.0.0.1:9200/opensearch', + 'my-opensearch-cluster.company.com:9200', + 'my-opensearch-cluster.company.com:9200/opensearch', + 'http://my-opensearch-cluster.company.com', + 'https://my-opensearch-cluster.company.com:9200', + 'https://my-opensearch-cluster.company.com:9200/opensearch', + '[::1]:9200', + '[::1]:9200/opensearch', + ] index: auditlog username: auditloguser password: auditlogpassword From 5b7531941cca59673aa3e809b0afe3c20dae86d9 Mon Sep 17 00:00:00 2001 From: Darshit Chanpura <35282393+DarshitChanpura@users.noreply.github.com> Date: Tue, 14 Nov 2023 15:14:18 -0500 Subject: [PATCH 007/204] Ensures that jacoco test report is produced after integration test task completes (#3662) Signed-off-by: Darshit Chanpura --- build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build.gradle b/build.gradle index dcc8171263..7717034434 100644 --- a/build.gradle +++ b/build.gradle @@ -557,6 +557,8 @@ task integrationTest(type: Test) { } } +tasks.integrationTest.finalizedBy(jacocoTestReport) // report is always generated after integration tests run + //run the integrationTest task before the check task check.dependsOn integrationTest From b7f47b1f516bee03a7dfaeec3a9133e4eafa3a73 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 16 Nov 2023 10:09:52 -0500 Subject: [PATCH 008/204] Use BytesRestResponse constructor with contentType in asRestResponse (#3717) ### Description Modifies `SecurityResponse.asRestResponse` to use the corresponding constructor for `BytesRestResponse` if the `SecurityResponse` contains the content-type header. This adds a test that fails before the change and demonstrates how using the constructor of BytesRestResponse without content-type defaults to text/plain --------- Signed-off-by: Craig Perkins Signed-off-by: Stephen Crawford Signed-off-by: Craig Perkins Co-authored-by: Stephen Crawford Co-authored-by: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> --- .../kerberos/HTTPSpnegoAuthenticator.java | 12 +- .../http/saml/AuthTokenProcessorHandler.java | 4 +- .../security/auth/BackendRegistry.java | 12 +- .../security/filter/SecurityResponse.java | 58 ++++++- .../security/filter/SecurityRestFilter.java | 4 +- .../security/httpclient/HttpClient.java | 4 +- .../impl/AllowlistingSettings.java | 3 +- .../impl/WhitelistingSettings.java | 3 +- .../http/saml/HTTPSamlAuthenticatorTest.java | 2 +- .../filter/SecurityResponseTests.java | 155 ++++++++++++++++++ 10 files changed, 232 insertions(+), 25 deletions(-) create mode 100644 src/test/java/org/opensearch/security/filter/SecurityResponseTests.java diff --git a/src/main/java/com/amazon/dlic/auth/http/kerberos/HTTPSpnegoAuthenticator.java b/src/main/java/com/amazon/dlic/auth/http/kerberos/HTTPSpnegoAuthenticator.java index 44bff5c73e..125bbed073 100644 --- a/src/main/java/com/amazon/dlic/auth/http/kerberos/HTTPSpnegoAuthenticator.java +++ b/src/main/java/com/amazon/dlic/auth/http/kerberos/HTTPSpnegoAuthenticator.java @@ -39,6 +39,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.env.Environment; import org.opensearch.security.auth.HTTPAuthenticator; @@ -284,10 +285,12 @@ public GSSCredential run() throws GSSException { public Optional reRequestAuthentication(final SecurityRequest request, AuthCredentials creds) { final Map headers = new HashMap<>(); String responseBody = ""; + String contentType = null; + SecurityResponse response; final String negotiateResponseBody = getNegotiateResponseBody(); if (negotiateResponseBody != null) { responseBody = negotiateResponseBody; - headers.putAll(SecurityResponse.CONTENT_TYPE_APP_JSON); + contentType = XContentType.JSON.mediaType(); } if (creds == null || creds.getNativeCredentials() == null) { @@ -296,7 +299,12 @@ public Optional reRequestAuthentication(final SecurityRequest headers.put("WWW-Authenticate", "Negotiate " + Base64.getEncoder().encodeToString((byte[]) creds.getNativeCredentials())); } - return Optional.of(new SecurityResponse(SC_UNAUTHORIZED, headers, responseBody)); + if (contentType != null) { + response = new SecurityResponse(SC_UNAUTHORIZED, headers, responseBody, contentType); + } else { + response = new SecurityResponse(SC_UNAUTHORIZED, headers, responseBody); + } + return Optional.of(response); } @Override diff --git a/src/main/java/com/amazon/dlic/auth/http/saml/AuthTokenProcessorHandler.java b/src/main/java/com/amazon/dlic/auth/http/saml/AuthTokenProcessorHandler.java index 41e9305ba6..32e01b9e2f 100644 --- a/src/main/java/com/amazon/dlic/auth/http/saml/AuthTokenProcessorHandler.java +++ b/src/main/java/com/amazon/dlic/auth/http/saml/AuthTokenProcessorHandler.java @@ -226,10 +226,10 @@ private Optional handleLowLevel(RestRequest restRequest) throw String responseBodyString = DefaultObjectMapper.objectMapper.writeValueAsString(responseBody); - return Optional.of(new SecurityResponse(HttpStatus.SC_OK, SecurityResponse.CONTENT_TYPE_APP_JSON, responseBodyString)); + return Optional.of(new SecurityResponse(HttpStatus.SC_OK, null, responseBodyString, XContentType.JSON.mediaType())); } catch (JsonProcessingException e) { log.warn("Error while parsing JSON for /_opendistro/_security/api/authtoken", e); - return Optional.of(new SecurityResponse(HttpStatus.SC_BAD_REQUEST, new Exception("JSON could not be parsed"))); + return Optional.of(new SecurityResponse(HttpStatus.SC_BAD_REQUEST, "JSON could not be parsed")); } } diff --git a/src/main/java/org/opensearch/security/auth/BackendRegistry.java b/src/main/java/org/opensearch/security/auth/BackendRegistry.java index 3f6aae0720..3ab9a2afc9 100644 --- a/src/main/java/org/opensearch/security/auth/BackendRegistry.java +++ b/src/main/java/org/opensearch/security/auth/BackendRegistry.java @@ -202,7 +202,7 @@ public boolean authenticate(final SecurityRequestChannel request) { log.debug("Rejecting REST request because of blocked address: {}", request.getRemoteAddress().orElse(null)); } - request.queueForSending(new SecurityResponse(SC_UNAUTHORIZED, new Exception("Authentication finally failed"))); + request.queueForSending(new SecurityResponse(SC_UNAUTHORIZED, "Authentication finally failed")); return false; } @@ -224,7 +224,7 @@ public boolean authenticate(final SecurityRequestChannel request) { if (!isInitialized()) { log.error("Not yet initialized (you may need to run securityadmin)"); - request.queueForSending(new SecurityResponse(SC_SERVICE_UNAVAILABLE, new Exception("OpenSearch Security not initialized."))); + request.queueForSending(new SecurityResponse(SC_SERVICE_UNAVAILABLE, "OpenSearch Security not initialized.")); return false; } @@ -354,11 +354,7 @@ public boolean authenticate(final SecurityRequestChannel request) { log.error("Cannot authenticate rest user because admin user is not permitted to login via HTTP"); auditLog.logFailedLogin(authenticatedUser.getName(), true, null, request); request.queueForSending( - new SecurityResponse( - SC_FORBIDDEN, - null, - "Cannot authenticate user because admin user is not permitted to login via HTTP" - ) + new SecurityResponse(SC_FORBIDDEN, "Cannot authenticate user because admin user is not permitted to login via HTTP") ); return false; } @@ -429,7 +425,7 @@ public boolean authenticate(final SecurityRequestChannel request) { notifyIpAuthFailureListeners(request, authCredentials); request.queueForSending( - challengeResponse.orElseGet(() -> new SecurityResponse(SC_UNAUTHORIZED, null, "Authentication finally failed")) + challengeResponse.orElseGet(() -> new SecurityResponse(SC_UNAUTHORIZED, "Authentication finally failed")) ); return false; } diff --git a/src/main/java/org/opensearch/security/filter/SecurityResponse.java b/src/main/java/org/opensearch/security/filter/SecurityResponse.java index 0dc833a440..5041936d2e 100644 --- a/src/main/java/org/opensearch/security/filter/SecurityResponse.java +++ b/src/main/java/org/opensearch/security/filter/SecurityResponse.java @@ -12,11 +12,15 @@ package org.opensearch.security.filter; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.http.HttpHeaders; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestResponse; @@ -26,26 +30,63 @@ public class SecurityResponse { public static final Map CONTENT_TYPE_APP_JSON = Map.of(HttpHeaders.CONTENT_TYPE, "application/json"); private final int status; - private final Map headers; + private Map> headers; private final String body; + private final String contentType; public SecurityResponse(final int status, final Exception e) { this.status = status; - this.headers = CONTENT_TYPE_APP_JSON; + populateHeaders(CONTENT_TYPE_APP_JSON); this.body = generateFailureMessage(e); + this.contentType = XContentType.JSON.mediaType(); + } + + public SecurityResponse(final int status, String body) { + this.status = status; + this.body = body; + this.contentType = null; } public SecurityResponse(final int status, final Map headers, final String body) { this.status = status; - this.headers = headers; + populateHeaders(headers); + this.body = body; + this.contentType = null; + } + + public SecurityResponse(final int status, final Map headers, final String body, String contentType) { + this.status = status; this.body = body; + this.contentType = contentType; + populateHeaders(headers); + } + + private void populateHeaders(Map headers) { + if (headers != null) { + headers.entrySet().forEach(entry -> addHeader(entry.getKey(), entry.getValue())); + } + } + + /** + * Add a custom header. + */ + public void addHeader(String name, String value) { + if (headers == null) { + headers = new HashMap<>(2); + } + List header = headers.get(name); + if (header == null) { + header = new ArrayList<>(); + headers.put(name, header); + } + header.add(value); } public int getStatus() { return status; } - public Map getHeaders() { + public Map> getHeaders() { return headers; } @@ -54,9 +95,14 @@ public String getBody() { } public RestResponse asRestResponse() { - final RestResponse restResponse = new BytesRestResponse(RestStatus.fromCode(getStatus()), getBody()); + final RestResponse restResponse; + if (this.contentType != null) { + restResponse = new BytesRestResponse(RestStatus.fromCode(getStatus()), this.contentType, getBody()); + } else { + restResponse = new BytesRestResponse(RestStatus.fromCode(getStatus()), getBody()); + } if (getHeaders() != null) { - getHeaders().forEach(restResponse::addHeader); + getHeaders().entrySet().forEach(entry -> { entry.getValue().forEach(value -> restResponse.addHeader(entry.getKey(), value)); }); } return restResponse; } diff --git a/src/main/java/org/opensearch/security/filter/SecurityRestFilter.java b/src/main/java/org/opensearch/security/filter/SecurityRestFilter.java index e4d087cfe3..d52c5109fc 100644 --- a/src/main/java/org/opensearch/security/filter/SecurityRestFilter.java +++ b/src/main/java/org/opensearch/security/filter/SecurityRestFilter.java @@ -245,7 +245,7 @@ void authorizeRequest(RestHandler original, SecurityRequestChannel request, User } log.debug(err); - request.queueForSending(new SecurityResponse(HttpStatus.SC_UNAUTHORIZED, null, err)); + request.queueForSending(new SecurityResponse(HttpStatus.SC_UNAUTHORIZED, err)); return; } } @@ -288,7 +288,7 @@ public void checkAndAuthenticateRequest(SecurityRequestChannel requestChannel) t } catch (SSLPeerUnverifiedException e) { log.error("No ssl info", e); auditLog.logSSLException(requestChannel, e); - requestChannel.queueForSending(new SecurityResponse(HttpStatus.SC_FORBIDDEN, new Exception("No ssl info"))); + requestChannel.queueForSending(new SecurityResponse(HttpStatus.SC_FORBIDDEN, e)); return; } diff --git a/src/main/java/org/opensearch/security/httpclient/HttpClient.java b/src/main/java/org/opensearch/security/httpclient/HttpClient.java index 8c31a5f9c9..43b5107b70 100644 --- a/src/main/java/org/opensearch/security/httpclient/HttpClient.java +++ b/src/main/java/org/opensearch/security/httpclient/HttpClient.java @@ -33,6 +33,7 @@ import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLParameters; +import com.google.common.collect.Lists; import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder; import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; @@ -52,6 +53,7 @@ import org.apache.hc.core5.ssl.SSLContexts; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; + import org.opensearch.action.index.IndexRequest; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.WriteRequest.RefreshPolicy; @@ -62,8 +64,6 @@ import org.opensearch.client.RestHighLevelClient; import org.opensearch.common.xcontent.XContentType; -import com.google.common.collect.Lists; - public class HttpClient implements Closeable { public static class HttpClientBuilder { diff --git a/src/main/java/org/opensearch/security/securityconf/impl/AllowlistingSettings.java b/src/main/java/org/opensearch/security/securityconf/impl/AllowlistingSettings.java index 63d9186e1f..2a25ad8795 100644 --- a/src/main/java/org/opensearch/security/securityconf/impl/AllowlistingSettings.java +++ b/src/main/java/org/opensearch/security/securityconf/impl/AllowlistingSettings.java @@ -20,6 +20,7 @@ import org.apache.http.HttpStatus; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.rest.RestStatus; import org.opensearch.security.filter.SecurityRequest; import org.opensearch.security.filter.SecurityResponse; @@ -113,7 +114,7 @@ public Optional checkRequestIsAllowed(final SecurityRequest re // if allowlisting is enabled but the request is not allowlisted, then return false, otherwise true. if (this.enabled && !requestIsAllowlisted(request)) { return Optional.of( - new SecurityResponse(HttpStatus.SC_FORBIDDEN, SecurityResponse.CONTENT_TYPE_APP_JSON, generateFailureMessage(request)) + new SecurityResponse(HttpStatus.SC_FORBIDDEN, null, generateFailureMessage(request), XContentType.JSON.mediaType()) ); } return Optional.empty(); diff --git a/src/main/java/org/opensearch/security/securityconf/impl/WhitelistingSettings.java b/src/main/java/org/opensearch/security/securityconf/impl/WhitelistingSettings.java index ce643477c2..4cc16a7f00 100644 --- a/src/main/java/org/opensearch/security/securityconf/impl/WhitelistingSettings.java +++ b/src/main/java/org/opensearch/security/securityconf/impl/WhitelistingSettings.java @@ -18,6 +18,7 @@ import org.apache.http.HttpStatus; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.security.filter.SecurityRequest; import org.opensearch.security.filter.SecurityResponse; @@ -111,7 +112,7 @@ public Optional checkRequestIsAllowed(final SecurityRequest re // if whitelisting is enabled but the request is not whitelisted, then return false, otherwise true. if (this.enabled && !requestIsWhitelisted(request)) { return Optional.of( - new SecurityResponse(HttpStatus.SC_FORBIDDEN, SecurityResponse.CONTENT_TYPE_APP_JSON, generateFailureMessage(request)) + new SecurityResponse(HttpStatus.SC_FORBIDDEN, null, generateFailureMessage(request), XContentType.JSON.mediaType()) ); } return Optional.empty(); diff --git a/src/test/java/com/amazon/dlic/auth/http/saml/HTTPSamlAuthenticatorTest.java b/src/test/java/com/amazon/dlic/auth/http/saml/HTTPSamlAuthenticatorTest.java index c76a1b546d..bba2ee8b5c 100644 --- a/src/test/java/com/amazon/dlic/auth/http/saml/HTTPSamlAuthenticatorTest.java +++ b/src/test/java/com/amazon/dlic/auth/http/saml/HTTPSamlAuthenticatorTest.java @@ -887,7 +887,7 @@ private AuthenticateHeaders getAutenticateHeaders(HTTPSamlAuthenticator samlAuth RestRequest restRequest = new FakeRestRequest(ImmutableMap.of(), new HashMap()); SecurityResponse response = sendToAuthenticator(samlAuthenticator, restRequest).orElseThrow(); - String wwwAuthenticateHeader = response.getHeaders().get("WWW-Authenticate"); + String wwwAuthenticateHeader = response.getHeaders().get("WWW-Authenticate").get(0); Assert.assertNotNull(wwwAuthenticateHeader); diff --git a/src/test/java/org/opensearch/security/filter/SecurityResponseTests.java b/src/test/java/org/opensearch/security/filter/SecurityResponseTests.java new file mode 100644 index 0000000000..7735a8a7cd --- /dev/null +++ b/src/test/java/org/opensearch/security/filter/SecurityResponseTests.java @@ -0,0 +1,155 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.filter; + +import java.util.List; +import java.util.Map; + +import org.apache.hc.core5.http.HttpHeaders; +import org.apache.http.HttpStatus; +import org.junit.Test; + +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestResponse; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +public class SecurityResponseTests { + + /** + * This test should check whether a basic constructor with the JSON content type is successfully converted to RestResponse + */ + @Test + public void testSecurityResponseHasSingleContentType() { + final SecurityResponse response = new SecurityResponse(HttpStatus.SC_OK, null, "foo bar", XContentType.JSON.mediaType()); + final RestResponse restResponse = response.asRestResponse(); + assertThat(restResponse.status(), equalTo(RestStatus.OK)); + assertThat(restResponse.contentType(), equalTo(XContentType.JSON.mediaType())); + } + + /** + * This test should check whether adding a new HTTP Header for the content type takes the argument or the added header (should take arg.) + */ + @Test + public void testSecurityResponseMultipleContentTypesUsesPassed() { + final SecurityResponse response = new SecurityResponse(HttpStatus.SC_OK, null, "foo bar", XContentType.JSON.mediaType()); + response.addHeader(HttpHeaders.CONTENT_TYPE, BytesRestResponse.TEXT_CONTENT_TYPE); + assertThat(response.getHeaders().get("Content-Type"), equalTo(List.of(BytesRestResponse.TEXT_CONTENT_TYPE))); + final RestResponse restResponse = response.asRestResponse(); + assertThat(restResponse.contentType(), equalTo(XContentType.JSON.mediaType())); + assertThat(restResponse.status(), equalTo(RestStatus.OK)); + } + + /** + * This test should check whether specifying no content type correctly uses plain text + */ + @Test + public void testSecurityResponseDefaultContentTypeIsText() { + final SecurityResponse response = new SecurityResponse(HttpStatus.SC_OK, null, "foo bar"); + final RestResponse restResponse = response.asRestResponse(); + assertThat(restResponse.contentType(), equalTo(BytesRestResponse.TEXT_CONTENT_TYPE)); + assertThat(restResponse.status(), equalTo(RestStatus.OK)); + } + + /** + * This test checks whether adding a new ContentType header actually changes the converted content type header (it should not) + */ + @Test + public void testSecurityResponseSetHeaderContentTypeDoesNothing() { + final SecurityResponse response = new SecurityResponse(HttpStatus.SC_OK, null, "foo bar"); + response.addHeader(HttpHeaders.CONTENT_TYPE, XContentType.JSON.mediaType()); + final RestResponse restResponse = response.asRestResponse(); + assertThat(restResponse.contentType(), equalTo(BytesRestResponse.TEXT_CONTENT_TYPE)); + assertThat(restResponse.status(), equalTo(RestStatus.OK)); + } + + /** + * This test should check whether adding a multiple new HTTP Headers for the content type takes the argument or the added header (should take arg.) + */ + @Test + public void testSecurityResponseAddMultipleContentTypeHeaders() { + final SecurityResponse response = new SecurityResponse(HttpStatus.SC_OK, null, "foo bar", XContentType.JSON.mediaType()); + response.addHeader(HttpHeaders.CONTENT_TYPE, BytesRestResponse.TEXT_CONTENT_TYPE); + assertThat(response.getHeaders().get("Content-Type"), equalTo(List.of(BytesRestResponse.TEXT_CONTENT_TYPE))); + response.addHeader(HttpHeaders.CONTENT_TYPE, "newContentType"); + assertThat(response.getHeaders().get("Content-Type"), equalTo(List.of(BytesRestResponse.TEXT_CONTENT_TYPE, "newContentType"))); + final RestResponse restResponse = response.asRestResponse(); + assertThat(restResponse.status(), equalTo(RestStatus.OK)); + } + + /** + * This test confirms that fake content types work for conversion + */ + @Test + public void testSecurityResponseFakeContentTypeArgumentPasses() { + final SecurityResponse response = new SecurityResponse(HttpStatus.SC_OK, null, "foo bar", "testType"); + final RestResponse restResponse = response.asRestResponse(); + assertThat(restResponse.contentType(), equalTo("testType")); + assertThat(restResponse.status(), equalTo(RestStatus.OK)); + } + + /** + * This test checks that types passed as part of the Headers parameter in the argument do not overwrite actual Content Type + */ + @Test + public void testSecurityResponseContentTypeInConstructorHeader() { + final SecurityResponse response = new SecurityResponse(HttpStatus.SC_OK, Map.of("Content-Type", "testType"), "foo bar"); + assertThat(response.getHeaders().get("Content-Type"), equalTo(List.of("testType"))); + final RestResponse restResponse = response.asRestResponse(); + assertThat(restResponse.contentType(), equalTo(BytesRestResponse.TEXT_CONTENT_TYPE)); + assertThat(restResponse.status(), equalTo(RestStatus.OK)); + } + + /** + * This test confirms the same as above but with a conflicting content type arg + */ + @Test + public void testSecurityResponseContentTypeInConstructorHeaderConflicts() { + final SecurityResponse response = new SecurityResponse( + HttpStatus.SC_OK, + Map.of("Content-Type", "testType"), + "foo bar", + XContentType.JSON.mediaType() + ); + assertThat(response.getHeaders().get("Content-Type"), equalTo(List.of("testType"))); + final RestResponse restResponse = response.asRestResponse(); + assertThat(restResponse.contentType(), equalTo(XContentType.JSON.mediaType())); + assertThat(restResponse.status(), equalTo(RestStatus.OK)); + } + + /** + * This test should check whether unauthorized requests are converted properly + */ + @Test + public void testSecurityResponseUnauthorizedRequestWithPlainTextContentType() { + final SecurityResponse response = new SecurityResponse(HttpStatus.SC_UNAUTHORIZED, null, "foo bar"); + response.addHeader(HttpHeaders.CONTENT_TYPE, "application/json"); + final RestResponse restResponse = response.asRestResponse(); + assertThat(restResponse.contentType(), equalTo(BytesRestResponse.TEXT_CONTENT_TYPE)); + assertThat(restResponse.status(), equalTo(RestStatus.UNAUTHORIZED)); + } + + /** + * This test should check whether forbidden requests are converted properly + */ + @Test + public void testSecurityResponseForbiddenRequestWithPlainTextContentType() { + final SecurityResponse response = new SecurityResponse(HttpStatus.SC_FORBIDDEN, null, "foo bar"); + response.addHeader(HttpHeaders.CONTENT_TYPE, "application/json"); + final RestResponse restResponse = response.asRestResponse(); + assertThat(restResponse.contentType(), equalTo(BytesRestResponse.TEXT_CONTENT_TYPE)); + assertThat(restResponse.status(), equalTo(RestStatus.FORBIDDEN)); + } +} From deff84265cd22badf9cca02a3240aeb000acb439 Mon Sep 17 00:00:00 2001 From: Prabhas Kurapati <66924475+prabhask5@users.noreply.github.com> Date: Thu, 16 Nov 2023 10:46:22 -0800 Subject: [PATCH 009/204] [Feature-2188] AuditMessage validation in tests should be more useful / removed (#3596) Changed validateMsgs in AbstractAuditlogUnitTest.java to throw exceptions with descriptions instead of just returning true or false. Signed-off-by: Prabhas Kurapati --- .../auditlog/AbstractAuditlogiUnitTest.java | 30 ++++++--------- .../compliance/ComplianceAuditlogTest.java | 10 ++--- .../RestApiComplianceAuditlogTest.java | 10 ++--- .../integration/BasicAuditlogTest.java | 38 +++++++++---------- 4 files changed, 41 insertions(+), 47 deletions(-) diff --git a/src/test/java/org/opensearch/security/auditlog/AbstractAuditlogiUnitTest.java b/src/test/java/org/opensearch/security/auditlog/AbstractAuditlogiUnitTest.java index 3d814231cf..b671378ad4 100644 --- a/src/test/java/org/opensearch/security/auditlog/AbstractAuditlogiUnitTest.java +++ b/src/test/java/org/opensearch/security/auditlog/AbstractAuditlogiUnitTest.java @@ -84,34 +84,28 @@ protected void setupStarfleetIndex() { rh.keystore = keystore; } - protected boolean validateMsgs(final Collection msgs) { - boolean valid = true; + protected void validateMsgs(final Collection msgs) throws Exception { for (AuditMessage msg : msgs) { - valid = validateMsg(msg) && valid; + validateMsg(msg); } - return valid; - } - protected boolean validateMsg(final AuditMessage msg) { - return validateJson(msg.toJson()) && validateJson(msg.toPrettyString()); } - protected boolean validateJson(final String json) { + protected void validateMsg(final AuditMessage msg) throws Exception { + validateJson(msg.toJson()); + validateJson(msg.toPrettyString()); + } + protected void validateJson(final String json) throws Exception { // this function can throw either IllegalArgumentException, + // JsonMappingException if (json == null || json.isEmpty()) { - return false; + throw new IllegalArgumentException("json is either null or empty"); } - try { - JsonNode node = DefaultObjectMapper.objectMapper.readTree(json); - - if (node.get("audit_request_body") != null) { - DefaultObjectMapper.objectMapper.readTree(node.get("audit_request_body").asText()); - } + JsonNode node = DefaultObjectMapper.objectMapper.readTree(json); - return true; - } catch (Exception e) { - return false; + if (node.get("audit_request_body") != null) { + DefaultObjectMapper.objectMapper.readTree(node.get("audit_request_body").asText()); } } diff --git a/src/test/java/org/opensearch/security/auditlog/compliance/ComplianceAuditlogTest.java b/src/test/java/org/opensearch/security/auditlog/compliance/ComplianceAuditlogTest.java index 773180bd1b..f9d4474c3e 100644 --- a/src/test/java/org/opensearch/security/auditlog/compliance/ComplianceAuditlogTest.java +++ b/src/test/java/org/opensearch/security/auditlog/compliance/ComplianceAuditlogTest.java @@ -97,7 +97,7 @@ public void testSourceFilter() throws Exception { assertThat(message.getRequestBody(), not(containsString("Salary"))); assertThat(message.getRequestBody(), containsString("Gender")); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } @Test @@ -245,7 +245,7 @@ public void testSourceFilterMsearch() throws Exception { assertThat(genderMsg.getRequestBody(), containsString("Gender")); assertThat(genderMsg.getRequestBody(), not(containsString("Salary"))); - Assert.assertTrue(validateMsgs(messages)); + validateMsgs(messages); } @Test @@ -302,7 +302,7 @@ public void testInternalConfig() throws Exception { ); }); - Assert.assertTrue(validateMsgs(messages)); + validateMsgs(messages); } @Test @@ -346,7 +346,7 @@ public void testExternalConfig() throws Exception { assertThat(messages.get(1).getNodeId(), not(equalTo(messages.get(2).getNodeId()))); assertThat(messages.get(2).getNodeId(), not(equalTo(messages.get(3).getNodeId()))); - Assert.assertTrue(validateMsgs(messages)); + validateMsgs(messages); } @Test @@ -399,7 +399,7 @@ public void testUpdate() throws Exception { assertThat(ex2.getMissingCount(), equalTo(1)); Assert.assertTrue(TestAuditlogImpl.messages.isEmpty()); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } @Test diff --git a/src/test/java/org/opensearch/security/auditlog/compliance/RestApiComplianceAuditlogTest.java b/src/test/java/org/opensearch/security/auditlog/compliance/RestApiComplianceAuditlogTest.java index cf06726ea1..dfabe50431 100644 --- a/src/test/java/org/opensearch/security/auditlog/compliance/RestApiComplianceAuditlogTest.java +++ b/src/test/java/org/opensearch/security/auditlog/compliance/RestApiComplianceAuditlogTest.java @@ -55,7 +55,7 @@ public void testRestApiRolesEnabled() throws Exception { Assert.assertFalse(TestAuditlogImpl.sb.toString().contains("COMPLIANCE_INTERNAL_CONFIG_READ")); Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("COMPLIANCE_INTERNAL_CONFIG_WRITE")); Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("UPDATE")); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } @Test @@ -89,7 +89,7 @@ public void testRestApiRolesDisabled() throws Exception { Assert.assertFalse(TestAuditlogImpl.sb.toString().contains("COMPLIANCE_INTERNAL_CONFIG_READ")); Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("COMPLIANCE_INTERNAL_CONFIG_WRITE")); Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("UPDATE")); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } @Test @@ -122,7 +122,7 @@ public void testRestApiRolesDisabledGet() throws Exception { Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("COMPLIANCE_INTERNAL_CONFIG_READ")); Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("COMPLIANCE_INTERNAL_CONFIG_WRITE")); Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("UPDATE")); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } @Test @@ -147,7 +147,7 @@ public void testAutoInit() throws Exception { Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("audit_request_effective_user")); Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("COMPLIANCE_INTERNAL_CONFIG_WRITE")); Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("COMPLIANCE_EXTERNAL_CONFIG")); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } @Test @@ -208,7 +208,7 @@ public void testRestInternalConfigRead() throws Exception { Assert.assertTrue(auditLogImpl.contains("COMPLIANCE_INTERNAL_CONFIG_READ")); Assert.assertFalse(auditLogImpl.contains("COMPLIANCE_INTERNAL_CONFIG_WRITE")); Assert.assertFalse(auditLogImpl.contains("UPDATE")); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } @Test diff --git a/src/test/java/org/opensearch/security/auditlog/integration/BasicAuditlogTest.java b/src/test/java/org/opensearch/security/auditlog/integration/BasicAuditlogTest.java index 6c1812c32b..89b8426120 100644 --- a/src/test/java/org/opensearch/security/auditlog/integration/BasicAuditlogTest.java +++ b/src/test/java/org/opensearch/security/auditlog/integration/BasicAuditlogTest.java @@ -149,7 +149,7 @@ public void testSSLPlainText() throws Exception { Assert.assertEquals(AuditCategory.SSL_EXCEPTION, message.getCategory()); Assert.assertTrue(message.getExceptionStackTrace().contains("not an SSL/TLS record")); }); - Assert.assertTrue(validateMsgs(messages)); + validateMsgs(messages); } @Test @@ -182,7 +182,7 @@ public void testTaskId() throws Exception { TestAuditlogImpl.messages.get(1).getAsMap().get(AuditMessage.TASK_ID), TestAuditlogImpl.messages.get(1).getAsMap().get(AuditMessage.TASK_ID) ); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } @Test @@ -211,7 +211,7 @@ public void testDefaultsRest() throws Exception { Assert.assertTrue(auditLogImpl.contains("\"audit_request_effective_user\" : \"admin\"")); Assert.assertTrue(auditLogImpl.contains("REST")); Assert.assertFalse(auditLogImpl.toLowerCase().contains("authorization")); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } @Test @@ -317,7 +317,7 @@ public void testWrongUser() throws Exception { Assert.assertTrue(TestAuditlogImpl.sb.toString(), TestAuditlogImpl.sb.toString().contains(AuditMessage.UTC_TIMESTAMP)); Assert.assertFalse(TestAuditlogImpl.sb.toString(), TestAuditlogImpl.sb.toString().contains("AUTHENTICATED")); Assert.assertEquals(1, TestAuditlogImpl.messages.size()); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } public void testUnknownAuthorization() throws Exception { @@ -329,7 +329,7 @@ public void testUnknownAuthorization() throws Exception { Assert.assertTrue(TestAuditlogImpl.sb.toString().contains(AuditMessage.UTC_TIMESTAMP)); Assert.assertFalse(TestAuditlogImpl.sb.toString().contains("AUTHENTICATED")); Assert.assertEquals(1, TestAuditlogImpl.messages.size()); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } public void testUnauthenticated() throws Exception { @@ -345,7 +345,7 @@ public void testUnauthenticated() throws Exception { Assert.assertTrue(auditLogImpl.contains("/_search")); Assert.assertTrue(auditLogImpl.contains(AuditMessage.UTC_TIMESTAMP)); Assert.assertFalse(auditLogImpl.contains("AUTHENTICATED")); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } @@ -353,7 +353,7 @@ public void testJustAuthenticated() throws Exception { HttpResponse response = rh.executeGetRequest("", encodeBasicHeader("admin", "admin")); Assert.assertEquals(HttpStatus.SC_OK, response.getStatusCode()); Assert.assertEquals(0, TestAuditlogImpl.messages.size()); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } public void testSecurityIndexAttempt() throws Exception { @@ -366,7 +366,7 @@ public void testSecurityIndexAttempt() throws Exception { Assert.assertTrue(TestAuditlogImpl.sb.toString().contains(AuditMessage.UTC_TIMESTAMP)); Assert.assertFalse(TestAuditlogImpl.sb.toString().contains("AUTHENTICATED")); Assert.assertEquals(2, TestAuditlogImpl.messages.size()); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } public void testBadHeader() throws Exception { @@ -381,7 +381,7 @@ public void testBadHeader() throws Exception { Assert.assertTrue(TestAuditlogImpl.sb.toString(), TestAuditlogImpl.sb.toString().contains("BAD_HEADERS")); Assert.assertTrue(TestAuditlogImpl.sb.toString(), TestAuditlogImpl.sb.toString().contains("_opendistro_security_bad")); Assert.assertEquals(TestAuditlogImpl.sb.toString(), 1, TestAuditlogImpl.messages.size()); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } public void testMissingPriv() throws Exception { @@ -395,7 +395,7 @@ public void testMissingPriv() throws Exception { Assert.assertTrue(TestAuditlogImpl.sb.toString().contains(AuditMessage.UTC_TIMESTAMP)); Assert.assertFalse(TestAuditlogImpl.sb.toString().contains("AUTHENTICATED")); Assert.assertEquals(1, TestAuditlogImpl.messages.size()); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } public void testMsearch() throws Exception { @@ -419,7 +419,7 @@ public void testMsearch() throws Exception { Assert.assertTrue(auditLogImpl.contains("audit_trace_task_id")); Assert.assertEquals(auditLogImpl, 4, TestAuditlogImpl.messages.size()); Assert.assertFalse(auditLogImpl.toLowerCase().contains("authorization")); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } public void testBulkAuth() throws Exception { @@ -458,7 +458,7 @@ public void testBulkAuth() throws Exception { Assert.assertTrue(auditLogImpl.contains("audit_trace_task_id")); // may vary because we log shardrequests which are not predictable here Assert.assertTrue(TestAuditlogImpl.messages.size() >= 17); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } public void testBulkNonAuth() throws Exception { @@ -496,7 +496,7 @@ public void testBulkNonAuth() throws Exception { Assert.assertTrue(auditLogImpl.contains("IndexRequest")); // may vary because we log shardrequests which are not predictable here Assert.assertTrue(TestAuditlogImpl.messages.size() >= 7); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } public void testUpdateSettings() throws Exception { @@ -521,7 +521,7 @@ public void testUpdateSettings() throws Exception { Assert.assertTrue(auditLogImpl.contains(expectedRequestBodyLog)); // may vary because we log may hit cluster manager directly or not Assert.assertTrue(TestAuditlogImpl.messages.size() > 1); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } @Test @@ -620,7 +620,7 @@ public void testAliases() throws Exception { Assert.assertTrue(auditLogImpl.contains("starfleet")); Assert.assertTrue(auditLogImpl.contains("sf")); Assert.assertEquals(2, TestAuditlogImpl.messages.size()); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } @Test @@ -682,7 +682,7 @@ public void testScroll() throws Exception { Assert.assertTrue(auditLogImpl.contains("InternalScrollSearchRequest")); Assert.assertTrue(auditLogImpl.contains("MISSING_PRIVILEGES")); Assert.assertTrue(TestAuditlogImpl.messages.size() > 2); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } @Test @@ -718,7 +718,7 @@ public void testAliasResolution() throws Exception { Assert.assertTrue(auditLogImpl.contains("audit_trace_resolved_indices")); Assert.assertTrue(auditLogImpl.contains("vulcangov")); Assert.assertEquals(1, TestAuditlogImpl.messages.size()); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); TestAuditlogImpl.clear(); } @@ -747,7 +747,7 @@ public void testAliasBadHeaders() throws Exception { Assert.assertTrue(auditLogImpl.contains("BAD_HEADERS")); Assert.assertTrue(auditLogImpl.contains("xxx")); Assert.assertEquals(1, TestAuditlogImpl.messages.size()); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); TestAuditlogImpl.clear(); } @@ -780,7 +780,7 @@ public void testIndexCloseDelete() throws Exception { Assert.assertTrue(auditLogImpl.contains("indices:admin/close")); Assert.assertTrue(auditLogImpl.contains("indices:admin/delete")); Assert.assertTrue(auditLogImpl, TestAuditlogImpl.messages.size() >= 2); - Assert.assertTrue(validateMsgs(TestAuditlogImpl.messages)); + validateMsgs(TestAuditlogImpl.messages); } @Test From 17748b9e7edb143ff108d45483e2333f70ab15df Mon Sep 17 00:00:00 2001 From: Darshit Chanpura <35282393+DarshitChanpura@users.noreply.github.com> Date: Thu, 16 Nov 2023 21:58:30 -0500 Subject: [PATCH 010/204] Extracts demo configuration script to a java tool (#3669) This change extracts the core demo configuration install script to java tool to allow re-usability while leveraging that OpenSearch itself is JAVA process. All other tools shipped by security are java tools, and now with this PR, demo configuration setup will also be a java tool. --------- Signed-off-by: Darshit Chanpura --- .github/workflows/plugin_install.yml | 4 +- .../democonfig/CertificateGenerator.java | 49 ++ .../tools/democonfig/Certificates.java | 174 +++++++ .../democonfig/ExecutionEnvironment.java | 9 + .../security/tools/democonfig/Installer.java | 366 ++++++++++++++ .../SecuritySettingsConfigurer.java | 319 ++++++++++++ tools/install_demo_configuration.bat | 416 +--------------- tools/install_demo_configuration.sh | 463 +----------------- 8 files changed, 937 insertions(+), 863 deletions(-) create mode 100644 src/main/java/org/opensearch/security/tools/democonfig/CertificateGenerator.java create mode 100644 src/main/java/org/opensearch/security/tools/democonfig/Certificates.java create mode 100644 src/main/java/org/opensearch/security/tools/democonfig/ExecutionEnvironment.java create mode 100644 src/main/java/org/opensearch/security/tools/democonfig/Installer.java create mode 100644 src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java diff --git a/.github/workflows/plugin_install.yml b/.github/workflows/plugin_install.yml index ae570a9df8..92d923bb0d 100644 --- a/.github/workflows/plugin_install.yml +++ b/.github/workflows/plugin_install.yml @@ -44,14 +44,14 @@ jobs: run: | cat > setup.sh <<'EOF' chmod +x ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/plugins/${{ env.PLUGIN_NAME }}/tools/install_demo_configuration.sh - /bin/bash -c "yes | ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/plugins/${{ env.PLUGIN_NAME }}/tools/install_demo_configuration.sh" + /bin/bash -c "yes | ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/plugins/${{ env.PLUGIN_NAME }}/tools/install_demo_configuration.sh -t" EOF - name: Create Setup Script if: ${{ runner.os == 'Windows' }} run: | New-Item .\setup.bat -type file - Set-Content .\setup.bat -Value "powershell.exe .\opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT\plugins\${{ env.PLUGIN_NAME }}\tools\install_demo_configuration.bat -i -c -y" + Set-Content .\setup.bat -Value "powershell.exe .\opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT\plugins\${{ env.PLUGIN_NAME }}\tools\install_demo_configuration.bat -i -c -y -t" Get-Content .\setup.bat - name: Run Opensearch with A Single Plugin diff --git a/src/main/java/org/opensearch/security/tools/democonfig/CertificateGenerator.java b/src/main/java/org/opensearch/security/tools/democonfig/CertificateGenerator.java new file mode 100644 index 0000000000..a7b39c226e --- /dev/null +++ b/src/main/java/org/opensearch/security/tools/democonfig/CertificateGenerator.java @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.tools.democonfig; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +/** + * This class creates demo certificate files + */ +public class CertificateGenerator extends Installer { + + /** + * Creates demo super-admin, node and root certificates by iterating through Certificates enum + */ + public void createDemoCertificates() { + for (Certificates cert : Certificates.values()) { + String filePath = OPENSEARCH_CONF_DIR + File.separator + cert.getFileName(); + writeCertificateToFile(filePath, cert.getContent()); + } + } + + /** + * Helper method to write the certificates to their own file + * @param filePath the file which needs to be written + * @param content the content which needs to be written to this file + */ + static void writeCertificateToFile(String filePath, String content) { + try { + FileWriter fileWriter = new FileWriter(filePath, StandardCharsets.UTF_8); + fileWriter.write(content); + fileWriter.close(); + } catch (IOException e) { + System.err.println("Error writing certificate file: " + filePath); + System.exit(-1); + } + } +} diff --git a/src/main/java/org/opensearch/security/tools/democonfig/Certificates.java b/src/main/java/org/opensearch/security/tools/democonfig/Certificates.java new file mode 100644 index 0000000000..6821147e8c --- /dev/null +++ b/src/main/java/org/opensearch/security/tools/democonfig/Certificates.java @@ -0,0 +1,174 @@ +package org.opensearch.security.tools.democonfig; + +/** + * Enum for demo certificates + */ +public enum Certificates { + ADMIN_CERT( + "kirk.pem", + "-----BEGIN CERTIFICATE-----\n" + + "MIIEmDCCA4CgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iYwDQYJKoZIhvcNAQEL\n" + + "BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt\n" + + "cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl\n" + + "IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v\n" + + "dCBDQTAeFw0yMzA4MjkyMDA2MzdaFw0zMzA4MjYyMDA2MzdaME0xCzAJBgNVBAYT\n" + + "AmRlMQ0wCwYDVQQHDAR0ZXN0MQ8wDQYDVQQKDAZjbGllbnQxDzANBgNVBAsMBmNs\n" + + "aWVudDENMAsGA1UEAwwEa2lyazCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC\n" + + "ggEBAJVcOAQlCiuB9emCljROAXnlsPbG7PE3kNz2sN+BbGuw686Wgyl3uToVHvVs\n" + + "paMmLUqm1KYz9wMSWTIBZgpJ9hYaIbGxD4RBb7qTAJ8Q4ddCV2f7T4lxao/6ixI+\n" + + "O0l/BG9E3mRGo/r0w+jtTQ3aR2p6eoxaOYbVyEMYtFI4QZTkcgGIPGxm05y8xonx\n" + + "vV5pbSW9L7qAVDzQC8EYGQMMI4ccu0NcHKWtmTYJA/wDPE2JwhngHwbcIbc4cDz6\n" + + "cG0S3FmgiKGuuSqUy35v/k3y7zMHQSdx7DSR2tzhH/bBL/9qGvpT71KKrxPtaxS0\n" + + "bAqPcEkKWDo7IMlGGW7LaAWfGg8CAwEAAaOCASswggEnMAwGA1UdEwEB/wQCMAAw\n" + + "DgYDVR0PAQH/BAQDAgXgMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMCMIHPBgNVHSME\n" + + "gccwgcSAFBeH36Ba62YSp9XQ+LoSRTy3KwCcoYGVpIGSMIGPMRMwEQYKCZImiZPy\n" + + "LGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQRXhh\n" + + "bXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENB\n" + + "MSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0GCFHfkrz782p+T9k0G\n" + + "xGeM4+BrehWKMB0GA1UdDgQWBBSjMS8tgguX/V7KSGLoGg7K6XMzIDANBgkqhkiG\n" + + "9w0BAQsFAAOCAQEANMwD1JYlwAh82yG1gU3WSdh/tb6gqaSzZK7R6I0L7slaXN9m\n" + + "y2ErUljpTyaHrdiBFmPhU/2Kj2r+fIUXtXdDXzizx/JdmueT0nG9hOixLqzfoC9p\n" + + "fAhZxM62RgtyZoaczQN82k1/geMSwRpEndFe3OH7arkS/HSbIFxQhAIy229eWe5d\n" + + "1bUzP59iu7f3r567I4ob8Vy7PP+Ov35p7Vv4oDHHwgsdRzX6pvL6mmwVrQ3BfVec\n" + + "h9Dqprr+ukYmjho76g6k5cQuRaB6MxqldzUg+2E7IHQP8MCF+co51uZq2nl33mtp\n" + + "RGr6JbdHXc96zsLTL3saJQ8AWEfu1gbTVrwyRA==\n" + + "-----END CERTIFICATE-----" + ), + ADMIN_CERT_KEY( + "kirk-key.pem", + "-----BEGIN PRIVATE KEY-----\n" + + "MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCVXDgEJQorgfXp\n" + + "gpY0TgF55bD2xuzxN5Dc9rDfgWxrsOvOloMpd7k6FR71bKWjJi1KptSmM/cDElky\n" + + "AWYKSfYWGiGxsQ+EQW+6kwCfEOHXQldn+0+JcWqP+osSPjtJfwRvRN5kRqP69MPo\n" + + "7U0N2kdqenqMWjmG1chDGLRSOEGU5HIBiDxsZtOcvMaJ8b1eaW0lvS+6gFQ80AvB\n" + + "GBkDDCOHHLtDXBylrZk2CQP8AzxNicIZ4B8G3CG3OHA8+nBtEtxZoIihrrkqlMt+\n" + + "b/5N8u8zB0Encew0kdrc4R/2wS//ahr6U+9Siq8T7WsUtGwKj3BJClg6OyDJRhlu\n" + + "y2gFnxoPAgMBAAECggEAP5TOycDkx+megAWVoHV2fmgvgZXkBrlzQwUG/VZQi7V4\n" + + "ZGzBMBVltdqI38wc5MtbK3TCgHANnnKgor9iq02Z4wXDwytPIiti/ycV9CDRKvv0\n" + + "TnD2hllQFjN/IUh5n4thHWbRTxmdM7cfcNgX3aZGkYbLBVVhOMtn4VwyYu/Mxy8j\n" + + "xClZT2xKOHkxqwmWPmdDTbAeZIbSv7RkIGfrKuQyUGUaWhrPslvYzFkYZ0umaDgQ\n" + + "OAthZew5Bz3OfUGOMPLH61SVPuJZh9zN1hTWOvT65WFWfsPd2yStI+WD/5PU1Doo\n" + + "1RyeHJO7s3ug8JPbtNJmaJwHe9nXBb/HXFdqb976yQKBgQDNYhpu+MYSYupaYqjs\n" + + "9YFmHQNKpNZqgZ4ceRFZ6cMJoqpI5dpEMqToFH7tpor72Lturct2U9nc2WR0HeEs\n" + + "/6tiptyMPTFEiMFb1opQlXF2ae7LeJllntDGN0Q6vxKnQV+7VMcXA0Y8F7tvGDy3\n" + + "qJu5lfvB1mNM2I6y/eMxjBuQhwKBgQC6K41DXMFro0UnoO879pOQYMydCErJRmjG\n" + + "/tZSy3Wj4KA/QJsDSViwGfvdPuHZRaG9WtxdL6kn0w1exM9Rb0bBKl36lvi7o7xv\n" + + "M+Lw9eyXMkww8/F5d7YYH77gIhGo+RITkKI3+5BxeBaUnrGvmHrpmpgRXWmINqr0\n" + + "0jsnN3u0OQKBgCf45vIgItSjQb8zonLz2SpZjTFy4XQ7I92gxnq8X0Q5z3B+o7tQ\n" + + "K/4rNwTju/sGFHyXAJlX+nfcK4vZ4OBUJjP+C8CTjEotX4yTNbo3S6zjMyGQqDI5\n" + + "9aIOUY4pb+TzeUFJX7If5gR+DfGyQubvvtcg1K3GHu9u2l8FwLj87sRzAoGAflQF\n" + + "RHuRiG+/AngTPnZAhc0Zq0kwLkpH2Rid6IrFZhGLy8AUL/O6aa0IGoaMDLpSWUJp\n" + + "nBY2S57MSM11/MVslrEgGmYNnI4r1K25xlaqV6K6ztEJv6n69327MS4NG8L/gCU5\n" + + "3pEm38hkUi8pVYU7in7rx4TCkrq94OkzWJYurAkCgYATQCL/rJLQAlJIGulp8s6h\n" + + "mQGwy8vIqMjAdHGLrCS35sVYBXG13knS52LJHvbVee39AbD5/LlWvjJGlQMzCLrw\n" + + "F7oILW5kXxhb8S73GWcuMbuQMFVHFONbZAZgn+C9FW4l7XyRdkrbR1MRZ2km8YMs\n" + + "/AHmo368d4PSNRMMzLHw8Q==\n" + + "-----END PRIVATE KEY-----" + ), + NODE_CERT( + "esnode.pem", + "-----BEGIN CERTIFICATE-----\n" + + "MIIEPDCCAySgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iIwDQYJKoZIhvcNAQEL\n" + + "BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt\n" + + "cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl\n" + + "IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v\n" + + "dCBDQTAeFw0yMzA4MjkwNDIzMTJaFw0zMzA4MjYwNDIzMTJaMFcxCzAJBgNVBAYT\n" + + "AmRlMQ0wCwYDVQQHDAR0ZXN0MQ0wCwYDVQQKDARub2RlMQ0wCwYDVQQLDARub2Rl\n" + + "MRswGQYDVQQDDBJub2RlLTAuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA\n" + + "A4IBDwAwggEKAoIBAQCm93kXteDQHMAvbUPNPW5pyRHKDD42XGWSgq0k1D29C/Ud\n" + + "yL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0\n" + + "HGkn47XVu3EwbfrTENg3jFu+Oem6a/501SzITzJWtS0cn2dIFOBimTVpT/4Zv5qr\n" + + "XA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8n\n" + + "dibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b6l+KLo3IKpfTbAIJXIO+M67FLtWKtttD\n" + + "ao94B069skzKk6FPgW/OZh6PRCD0oxOavV+ld2SjAgMBAAGjgcYwgcMwRwYDVR0R\n" + + "BEAwPogFKgMEBQWCEm5vZGUtMC5leGFtcGxlLmNvbYIJbG9jYWxob3N0hxAAAAAA\n" + + "AAAAAAAAAAAAAAABhwR/AAABMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEF\n" + + "BQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU0/qDQaY10jIo\n" + + "wCjLUpz/HfQXyt8wHwYDVR0jBBgwFoAUF4ffoFrrZhKn1dD4uhJFPLcrAJwwDQYJ\n" + + "KoZIhvcNAQELBQADggEBAD2hkndVih6TWxoe/oOW0i2Bq7ScNO/n7/yHWL04HJmR\n" + + "MaHv/Xjc8zLFLgHuHaRvC02ikWIJyQf5xJt0Oqu2GVbqXH9PBGKuEP2kCsRRyU27\n" + + "zTclAzfQhqmKBTYQ/3lJ3GhRQvXIdYTe+t4aq78TCawp1nSN+vdH/1geG6QjMn5N\n" + + "1FU8tovDd4x8Ib/0dv8RJx+n9gytI8n/giIaDCEbfLLpe4EkV5e5UNpOnRgJjjuy\n" + + "vtZutc81TQnzBtkS9XuulovDE0qI+jQrKkKu8xgGLhgH0zxnPkKtUg2I3Aq6zl1L\n" + + "zYkEOUF8Y25J6WeY88Yfnc0iigI+Pnz5NK8R9GL7TYo=\n" + + "-----END CERTIFICATE-----" + ), + NODE_KEY( + "esnode-key.pem", + "-----BEGIN PRIVATE KEY-----\n" + + "MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCm93kXteDQHMAv\n" + + "bUPNPW5pyRHKDD42XGWSgq0k1D29C/UdyL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0\n" + + "o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0HGkn47XVu3EwbfrTENg3jFu+Oem6a/50\n" + + "1SzITzJWtS0cn2dIFOBimTVpT/4Zv5qrXA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1\n" + + "MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8ndibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b\n" + + "6l+KLo3IKpfTbAIJXIO+M67FLtWKtttDao94B069skzKk6FPgW/OZh6PRCD0oxOa\n" + + "vV+ld2SjAgMBAAECggEAQK1+uAOZeaSZggW2jQut+MaN4JHLi61RH2cFgU3COLgo\n" + + "FIiNjFn8f2KKU3gpkt1It8PjlmprpYut4wHI7r6UQfuv7ZrmncRiPWHm9PB82+ZQ\n" + + "5MXYqj4YUxoQJ62Cyz4sM6BobZDrjG6HHGTzuwiKvHHkbsEE9jQ4E5m7yfbVvM0O\n" + + "zvwrSOM1tkZihKSTpR0j2+taji914tjBssbn12TMZQL5ItGnhR3luY8mEwT9MNkZ\n" + + "xg0VcREoAH+pu9FE0vPUgLVzhJ3be7qZTTSRqv08bmW+y1plu80GbppePcgYhEow\n" + + "dlW4l6XPJaHVSn1lSFHE6QAx6sqiAnBz0NoTPIaLyQKBgQDZqDOlhCRciMRicSXn\n" + + "7yid9rhEmdMkySJHTVFOidFWwlBcp0fGxxn8UNSBcXdSy7GLlUtH41W9PWl8tp9U\n" + + "hQiiXORxOJ7ZcB80uNKXF01hpPj2DpFPWyHFxpDkWiTAYpZl68rOlYujxZUjJIej\n" + + "VvcykBC2BlEOG9uZv2kxcqLyJwKBgQDEYULTxaTuLIa17wU3nAhaainKB3vHxw9B\n" + + "Ksy5p3ND43UNEKkQm7K/WENx0q47TA1mKD9i+BhaLod98mu0YZ+BCUNgWKcBHK8c\n" + + "uXpauvM/pLhFLXZ2jvEJVpFY3J79FSRK8bwE9RgKfVKMMgEk4zOyZowS8WScOqiy\n" + + "hnQn1vKTJQKBgElhYuAnl9a2qXcC7KOwRsJS3rcKIVxijzL4xzOyVShp5IwIPbOv\n" + + "hnxBiBOH/JGmaNpFYBcBdvORE9JfA4KMQ2fx53agfzWRjoPI1/7mdUk5RFI4gRb/\n" + + "A3jZRBoopgFSe6ArCbnyQxzYzToG48/Wzwp19ZxYrtUR4UyJct6f5n27AoGBAJDh\n" + + "KIpQQDOvCdtjcbfrF4aM2DPCfaGPzENJriwxy6oEPzDaX8Bu/dqI5Ykt43i/zQrX\n" + + "GpyLaHvv4+oZVTiI5UIvcVO9U8hQPyiz9f7F+fu0LHZs6f7hyhYXlbe3XFxeop3f\n" + + "5dTKdWgXuTTRF2L9dABkA2deS9mutRKwezWBMQk5AoGBALPtX0FrT1zIosibmlud\n" + + "tu49A/0KZu4PBjrFMYTSEWGNJez3Fb2VsJwylVl6HivwbP61FhlYfyksCzQQFU71\n" + + "+x7Nmybp7PmpEBECr3deoZKQ/acNHn0iwb0It+YqV5+TquQebqgwK6WCLsMuiYKT\n" + + "bg/ch9Rhxbq22yrVgWHh6epp\n" + + "-----END PRIVATE KEY-----" + ), + ROOT_CA( + "root-ca.pem", + "-----BEGIN CERTIFICATE-----\n" + + "MIIExjCCA66gAwIBAgIUd+SvPvzan5P2TQbEZ4zj4Gt6FYowDQYJKoZIhvcNAQEL\n" + + "BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt\n" + + "cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl\n" + + "IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v\n" + + "dCBDQTAeFw0yMzA4MjkwNDIwMDNaFw0yMzA5MjgwNDIwMDNaMIGPMRMwEQYKCZIm\n" + + "iZPyLGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQ\n" + + "RXhhbXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290\n" + + "IENBMSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0EwggEiMA0GCSqG\n" + + "SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEPyN7J9VGPyJcQmCBl5TGwfSzvVdWwoQU\n" + + "j9aEsdfFJ6pBCDQSsj8Lv4RqL0dZra7h7SpZLLX/YZcnjikrYC+rP5OwsI9xEE/4\n" + + "U98CsTBPhIMgqFK6SzNE5494BsAk4cL72dOOc8tX19oDS/PvBULbNkthQ0aAF1dg\n" + + "vbrHvu7hq7LisB5ZRGHVE1k/AbCs2PaaKkn2jCw/b+U0Ml9qPuuEgz2mAqJDGYoA\n" + + "WSR4YXrOcrmPuRqbws464YZbJW898/0Pn/U300ed+4YHiNYLLJp51AMkR4YEw969\n" + + "VRPbWIvLrd0PQBooC/eLrL6rvud/GpYhdQEUx8qcNCKd4bz3OaQ5AgMBAAGjggEW\n" + + "MIIBEjAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQU\n" + + "F4ffoFrrZhKn1dD4uhJFPLcrAJwwgc8GA1UdIwSBxzCBxIAUF4ffoFrrZhKn1dD4\n" + + "uhJFPLcrAJyhgZWkgZIwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJ\n" + + "k/IsZAEZFgdleGFtcGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYD\n" + + "VQQLDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUg\n" + + "Q29tIEluYy4gUm9vdCBDQYIUd+SvPvzan5P2TQbEZ4zj4Gt6FYowDQYJKoZIhvcN\n" + + "AQELBQADggEBAIopqco/k9RSjouTeKP4z0EVUxdD4qnNh1GLSRqyAVe0aChyKF5f\n" + + "qt1Bd1XCY8D16RgekkKGHDpJhGCpel+vtIoXPBxUaGQNYxmJCf5OzLMODlcrZk5i\n" + + "jHIcv/FMeK02NBcz/WQ3mbWHVwXLhmwqa2zBsF4FmPCJAbFLchLhkAv1HJifHbnD\n" + + "jQzlKyl5jxam/wtjWxSm0iyso0z2TgyzY+MESqjEqB1hZkCFzD1xtUOCxbXgtKae\n" + + "dgfHVFuovr3fNLV3GvQk0s9okDwDUcqV7DSH61e5bUMfE84o3of8YA7+HUoPV5Du\n" + + "8sTOKRf7ncGXdDRA8aofW268pTCuIu3+g/Y=\n" + + "-----END CERTIFICATE-----" + ); + + private final String fileName; + private final String content; + + Certificates(String fileName, String content) { + this.fileName = fileName; + this.content = content; + } + + public String getFileName() { + return fileName; + } + + public String getContent() { + return content; + } +} diff --git a/src/main/java/org/opensearch/security/tools/democonfig/ExecutionEnvironment.java b/src/main/java/org/opensearch/security/tools/democonfig/ExecutionEnvironment.java new file mode 100644 index 0000000000..9f901c4487 --- /dev/null +++ b/src/main/java/org/opensearch/security/tools/democonfig/ExecutionEnvironment.java @@ -0,0 +1,9 @@ +package org.opensearch.security.tools.democonfig; + +/** + * The environment in which the demo config installation script is being executed + */ +public enum ExecutionEnvironment { + DEMO, // default value + TEST // to be used only for tests +} diff --git a/src/main/java/org/opensearch/security/tools/democonfig/Installer.java b/src/main/java/org/opensearch/security/tools/democonfig/Installer.java new file mode 100644 index 0000000000..0b166ad580 --- /dev/null +++ b/src/main/java/org/opensearch/security/tools/democonfig/Installer.java @@ -0,0 +1,366 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.tools.democonfig; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.PosixFilePermission; +import java.util.HashSet; +import java.util.Scanner; +import java.util.Set; + +/** + * This class installs demo configuration for security plugin + */ +public class Installer { + + static boolean assumeyes = false; + static boolean initsecurity = false; + static boolean cluster_mode = false; + static int skip_updates = -1; + static String SCRIPT_DIR; + static String BASE_DIR; + static String OPENSEARCH_CONF_FILE; + static String OPENSEARCH_BIN_DIR; + static String OPENSEARCH_PLUGINS_DIR; + static String OPENSEARCH_LIB_PATH; + static String OPENSEARCH_INSTALL_TYPE; + static String OPENSEARCH_CONF_DIR; + static String OPENSEARCH_VERSION; + static String SECURITY_VERSION; + + static ExecutionEnvironment environment = ExecutionEnvironment.DEMO; + + static final String OS = System.getProperty("os.name") + " " + System.getProperty("os.version") + " " + System.getProperty("os.arch"); + + static final String FILE_EXTENSION = OS.toLowerCase().contains("win") ? ".bat" : ".sh"; + + static final String SYSTEM_INDICES = ".plugins-ml-config, .plugins-ml-connector, .plugins-ml-model-group, .plugins-ml-model, " + + ".plugins-ml-task, .plugins-ml-conversation-meta, .plugins-ml-conversation-interactions, .opendistro-alerting-config, .opendistro-alerting-alert*, " + + ".opendistro-anomaly-results*, .opendistro-anomaly-detector*, .opendistro-anomaly-checkpoints, .opendistro-anomaly-detection-state, " + + ".opendistro-reports-*, .opensearch-notifications-*, .opensearch-notebooks, .opensearch-observability, .ql-datasources, " + + ".opendistro-asynchronous-search-response*, .replication-metadata-store, .opensearch-knn-models, .geospatial-ip2geo-data*"; + + static SecuritySettingsConfigurer securitySettingsConfigurer; + static CertificateGenerator certificateGenerator; + + public static void main(String[] options) { + securitySettingsConfigurer = new SecuritySettingsConfigurer(); + certificateGenerator = new CertificateGenerator(); + + printScriptHeaders(); + readOptions(options); + gatherUserInputs(); + initializeVariables(); + printVariables(); + securitySettingsConfigurer.configureSecuritySettings(); + certificateGenerator.createDemoCertificates(); + finishScriptExecution(); + } + + /** + * Prints headers that indicate the start of script execution + */ + static void printScriptHeaders() { + System.out.println("### OpenSearch Security Demo Installer"); + System.out.println("### ** Warning: Do not use on production or public reachable systems **"); + } + + /** + * Reads the options passed to the script + * @param options an array of strings containing options passed to the script + */ + static void readOptions(String[] options) { + // set script execution dir + SCRIPT_DIR = options[0]; + + for (int i = 1; i < options.length; i++) { + switch (options[i]) { + case "-y": + assumeyes = true; + break; + case "-i": + initsecurity = true; + break; + case "-c": + cluster_mode = true; + break; + case "-s": + skip_updates = 0; + break; + case "-t": + environment = ExecutionEnvironment.TEST; + break; + case "-h": + case "-?": + showHelp(); + return; + default: + System.out.println("Invalid option: " + options[i]); + } + } + } + + /** + * Prints the help menu when -h option is passed + */ + static void showHelp() { + System.out.println("install_demo_configuration.sh [-y] [-i] [-c]"); + System.out.println(" -h show help"); + System.out.println(" -y confirm all installation dialogues automatically"); + System.out.println(" -i initialize Security plugin with default configuration (default is to ask if -y is not given)"); + System.out.println(" -c enable cluster mode by binding to all network interfaces (default is to ask if -y is not given)"); + System.out.println(" -s skip updates if config is already applied to opensearch.yml"); + System.out.println( + " -t set the execution environment to `test` to skip password validation. Should be used only for testing. (default is set to `demo`)" + ); + System.exit(0); + } + + /** + * Prompt the user and collect user inputs + * Input collection will be skipped if -y option was passed + */ + static void gatherUserInputs() { + if (!assumeyes) { + try (Scanner scanner = new Scanner(System.in, StandardCharsets.UTF_8)) { + + if (!confirmAction(scanner, "Install demo certificates?")) { + System.exit(0); + } + + if (!initsecurity) { + initsecurity = confirmAction(scanner, "Initialize Security Modules?"); + } + + if (!cluster_mode) { + System.out.println("Cluster mode requires additional setup of:"); + System.out.println(" - Virtual memory (vm.max_map_count)\n"); + cluster_mode = confirmAction(scanner, "Enable cluster mode?"); + } + } + } else { + initsecurity = true; + cluster_mode = true; + } + } + + /** + * Helper method to scan user inputs. + * @param scanner object to be used for scanning user input + * @param message prompt question + * @return true or false based on user input + */ + static boolean confirmAction(Scanner scanner, String message) { + System.out.print(message + " [y/N] "); + String response = scanner.nextLine(); + return response.equalsIgnoreCase("yes") || response.equalsIgnoreCase("y"); + } + + /** + * Initialize all class level variables required + */ + static void initializeVariables() { + setBaseDir(); + setOpenSearchVariables(); + setSecurityVariables(); + } + + /** + * Sets the base directory to be used by the script + */ + static void setBaseDir() { + File baseDirFile = new File(SCRIPT_DIR).getParentFile().getParentFile().getParentFile(); + BASE_DIR = baseDirFile != null ? baseDirFile.getAbsolutePath() : null; + + if (BASE_DIR == null || !new File(BASE_DIR).isDirectory()) { + System.out.println("DEBUG: basedir does not exist"); + System.exit(-1); + } + + BASE_DIR += File.separator; + } + + /** + * Sets the variables for items at OpenSearch level + */ + static void setOpenSearchVariables() { + OPENSEARCH_CONF_FILE = BASE_DIR + "config" + File.separator + "opensearch.yml"; + OPENSEARCH_BIN_DIR = BASE_DIR + "bin" + File.separator; + OPENSEARCH_PLUGINS_DIR = BASE_DIR + "plugins" + File.separator; + OPENSEARCH_LIB_PATH = BASE_DIR + "lib" + File.separator; + OPENSEARCH_INSTALL_TYPE = determineInstallType(); + + if (!(new File(OPENSEARCH_CONF_FILE).exists())) { + System.out.println("Unable to determine OpenSearch config directory. Quit."); + System.exit(-1); + } + + if (!(new File(OPENSEARCH_BIN_DIR).exists())) { + System.out.println("Unable to determine OpenSearch bin directory. Quit."); + System.exit(-1); + } + + if (!(new File(OPENSEARCH_PLUGINS_DIR).exists())) { + System.out.println("Unable to determine OpenSearch plugins directory. Quit."); + System.exit(-1); + } + + if (!(new File(OPENSEARCH_LIB_PATH).exists())) { + System.out.println("Unable to determine OpenSearch lib directory. Quit."); + System.exit(-1); + } + + OPENSEARCH_CONF_DIR = new File(OPENSEARCH_CONF_FILE).getParent(); + OPENSEARCH_CONF_DIR = new File(OPENSEARCH_CONF_DIR).getAbsolutePath() + File.separator; + } + + /** + * Returns the installation type based on the underlying operating system + * @return will be one of `.zip`, `.tar.gz` or `rpm/deb` + */ + static String determineInstallType() { + // windows (.bat execution) + if (OS.toLowerCase().contains("win")) { + return ".zip"; + } + + // other OS (.sh execution) + if (new File("/usr/share/opensearch").equals(new File(BASE_DIR))) { + OPENSEARCH_CONF_FILE = "/usr/share/opensearch/config/opensearch.yml"; + if (!new File(OPENSEARCH_CONF_FILE).exists()) { + OPENSEARCH_CONF_FILE = "/etc/opensearch/opensearch.yml"; + } + return "rpm/deb"; + } + return ".tar.gz"; + } + + /** + * Sets the path variables for items at OpenSearch security plugin level + */ + static void setSecurityVariables() { + if (!(new File(OPENSEARCH_PLUGINS_DIR + "opensearch-security").exists())) { + System.out.println("OpenSearch Security plugin not installed. Quit."); + System.exit(-1); + } + + // Extract OpenSearch version and Security version + File[] opensearchLibFiles = new File(OPENSEARCH_LIB_PATH).listFiles( + pathname -> pathname.getName().startsWith("opensearch-") && pathname.getName().endsWith(".jar") + ); + + if (opensearchLibFiles != null && opensearchLibFiles.length > 0) { + OPENSEARCH_VERSION = opensearchLibFiles[0].getName().replaceAll("opensearch-(.*).jar", "$1"); + } + + File[] securityFiles = new File(OPENSEARCH_PLUGINS_DIR + "opensearch-security").listFiles( + pathname -> pathname.getName().startsWith("opensearch-security-") && pathname.getName().endsWith(".jar") + ); + + if (securityFiles != null && securityFiles.length > 0) { + SECURITY_VERSION = securityFiles[0].getName().replaceAll("opensearch-security-(.*).jar", "$1"); + } + } + + /** + * Prints the initialized variables + */ + static void printVariables() { + System.out.println("OpenSearch install type: " + OPENSEARCH_INSTALL_TYPE + " on " + OS); + System.out.println("OpenSearch config dir: " + OPENSEARCH_CONF_DIR); + System.out.println("OpenSearch config file: " + OPENSEARCH_CONF_FILE); + System.out.println("OpenSearch bin dir: " + OPENSEARCH_BIN_DIR); + System.out.println("OpenSearch plugins dir: " + OPENSEARCH_PLUGINS_DIR); + System.out.println("OpenSearch lib dir: " + OPENSEARCH_LIB_PATH); + System.out.println("Detected OpenSearch Version: " + OPENSEARCH_VERSION); + System.out.println("Detected OpenSearch Security Version: " + SECURITY_VERSION); + } + + /** + * Prints end of script execution message and creates security admin demo file. + */ + static void finishScriptExecution() { + System.out.println("### Success"); + System.out.println("### Execute this script now on all your nodes and then start all nodes"); + + try { + String securityAdminScriptPath = OPENSEARCH_PLUGINS_DIR + + "opensearch-security" + + File.separator + + "tools" + + File.separator + + "securityadmin" + + FILE_EXTENSION; + String securityAdminDemoScriptPath = OPENSEARCH_CONF_DIR + "securityadmin_demo" + FILE_EXTENSION; + + securitySettingsConfigurer.createSecurityAdminDemoScript(securityAdminScriptPath, securityAdminDemoScriptPath); + + // Make securityadmin_demo script executable + // not needed for windows + if (!OS.toLowerCase().contains("win")) { + Path file = Paths.get(securityAdminDemoScriptPath); + Set perms = new HashSet<>(); + // Add the execute permission for owner, group, and others + perms.add(PosixFilePermission.OWNER_READ); + perms.add(PosixFilePermission.OWNER_EXECUTE); + perms.add(PosixFilePermission.GROUP_EXECUTE); + perms.add(PosixFilePermission.OTHERS_EXECUTE); + Files.setPosixFilePermissions(file, perms); + } + + // Read the last line of the security-admin script + String lastLine = ""; + try (BufferedReader reader = new BufferedReader(new FileReader(securityAdminDemoScriptPath, StandardCharsets.UTF_8))) { + String currentLine; + while ((currentLine = reader.readLine()) != null) { + lastLine = currentLine; + } + } + + if (!initsecurity) { + System.out.println("### After the whole cluster is up execute: "); + System.out.println(lastLine); + System.out.println("### or run ." + File.separator + "securityadmin_demo" + FILE_EXTENSION); + System.out.println("### After that you can also use the Security Plugin ConfigurationGUI"); + } else { + System.out.println("### OpenSearch Security will be automatically initialized."); + System.out.println("### If you like to change the runtime configuration "); + System.out.println( + "### change the files in .." + + File.separator + + ".." + + File.separator + + ".." + + File.separator + + "config" + + File.separator + + "opensearch-security and execute: " + ); + System.out.println(lastLine); + System.out.println("### or run ." + File.separator + "securityadmin_demo" + FILE_EXTENSION); + System.out.println("### To use the Security Plugin ConfigurationGUI"); + } + + System.out.println("### To access your secured cluster open https://: and log in with admin/admin."); + System.out.println("### (Ignore the SSL certificate warning because we installed self-signed demo certificates)"); + + } catch (Exception e) { + System.out.println(e.getMessage()); + } + } +} diff --git a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java new file mode 100644 index 0000000000..1e318c38b9 --- /dev/null +++ b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java @@ -0,0 +1,319 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.tools.democonfig; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.opensearch.common.settings.Settings; +import org.opensearch.security.dlic.rest.validation.PasswordValidator; +import org.opensearch.security.dlic.rest.validation.RequestContentValidator; +import org.opensearch.security.tools.Hasher; + +import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_PASSWORD_MIN_LENGTH; +import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_PASSWORD_VALIDATION_REGEX; +import static org.opensearch.security.user.UserService.generatePassword; + +/** + * This class updates the security related configuration, as needed. + */ +public class SecuritySettingsConfigurer extends Installer { + + /** + * Configures security related changes to the opensearch configuration + * 1. Checks if plugins is already configuration. If yes, exit + * 2. Sets the custom admin password (Generates one if none is provided) + * 3. Write the security config to opensearch.yml + */ + public void configureSecuritySettings() { + checkIfSecurityPluginIsAlreadyConfigured(); + updateAdminPassword(); + writeSecurityConfigToOpenSearchYML(); + } + + /** + * Replaces the admin password in internal_users.yml with the custom or generated password + */ + static void updateAdminPassword() { + String ADMIN_PASSWORD = ""; + String initialAdminPassword = System.getenv("initialAdminPassword"); + String ADMIN_PASSWORD_FILE_PATH = OPENSEARCH_CONF_DIR + "initialAdminPassword.txt"; + String INTERNAL_USERS_FILE_PATH = OPENSEARCH_CONF_DIR + "opensearch-security" + File.separator + "internal_users.yml"; + boolean shouldValidatePassword = environment.equals(ExecutionEnvironment.DEMO); + try { + final PasswordValidator passwordValidator = PasswordValidator.of( + Settings.builder() + .put(SECURITY_RESTAPI_PASSWORD_VALIDATION_REGEX, "(?=.*[A-Z])(?=.*[^a-zA-Z\\\\d])(?=.*[0-9])(?=.*[a-z]).{8,}") + .put(SECURITY_RESTAPI_PASSWORD_MIN_LENGTH, 8) + .build() + ); + + // Read custom password + if (initialAdminPassword != null && !initialAdminPassword.isEmpty()) { + ADMIN_PASSWORD = initialAdminPassword; + } else { + File adminPasswordFile = new File(ADMIN_PASSWORD_FILE_PATH); + if (adminPasswordFile.exists() && adminPasswordFile.length() > 0) { + try (BufferedReader br = new BufferedReader(new FileReader(ADMIN_PASSWORD_FILE_PATH, StandardCharsets.UTF_8))) { + ADMIN_PASSWORD = br.readLine(); + } catch (IOException e) { + System.out.println("Error reading admin password from initialAdminPassword.txt."); + System.exit(-1); + } + } + } + + // If script execution environment is set to demo, validate custom password, else if set to test, skip validation + if (shouldValidatePassword + && !ADMIN_PASSWORD.isEmpty() + && passwordValidator.validate("admin", ADMIN_PASSWORD) != RequestContentValidator.ValidationError.NONE) { + System.out.println("Password " + ADMIN_PASSWORD + " is weak. Please re-try with a stronger password."); + System.exit(-1); + } + + // if ADMIN_PASSWORD is still an empty string, it implies no custom password was provided. We proceed with generating a new one. + if (ADMIN_PASSWORD.isEmpty()) { + System.out.println("No custom admin password found. Generating a new password now."); + // generate a new random password + // We always validate a generated password + while (passwordValidator.validate("admin", ADMIN_PASSWORD) != RequestContentValidator.ValidationError.NONE) { + ADMIN_PASSWORD = generatePassword(); + } + } + + // print the password to the logs + System.out.println("\t***************************************************"); + System.out.println("\t\tADMIN PASSWORD SET TO: " + ADMIN_PASSWORD); + System.out.println("\t***************************************************"); + + writePasswordToInternalUsersFile(ADMIN_PASSWORD, INTERNAL_USERS_FILE_PATH); + + } catch (IOException e) { + System.out.println("Exception: " + e.getMessage()); + System.exit(-1); + } + } + + /** + * Generate password hash and update it in the internal_users.yml file + * @param adminPassword the password to be hashed and updated + * @param internalUsersFile the file path string to internal_users.yml file + * @throws IOException while reading, writing to files + */ + static void writePasswordToInternalUsersFile(String adminPassword, String internalUsersFile) throws IOException { + String hashedAdminPassword = Hasher.hash(adminPassword.toCharArray()); + + if (hashedAdminPassword.isEmpty()) { + System.out.println("Hash the admin password failure, see console for details"); + System.exit(-1); + } + + Path tempFilePath = Paths.get(internalUsersFile + ".tmp"); + Path internalUsersPath = Paths.get(internalUsersFile); + + try ( + BufferedReader reader = new BufferedReader(new FileReader(internalUsersFile, StandardCharsets.UTF_8)); + BufferedWriter writer = new BufferedWriter(new FileWriter(tempFilePath.toFile(), StandardCharsets.UTF_8)) + ) { + String line; + while ((line = reader.readLine()) != null) { + if (line.matches(" *hash: *\"\\$2a\\$12\\$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG\"")) { + line = line.replace( + "\"$2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG\"", + "\"" + hashedAdminPassword + "\"" + ); + } + writer.write(line + System.lineSeparator()); + } + } catch (IOException e) { + throw new IOException("Unable to update the internal users file with the hashed password."); + } + Files.move(tempFilePath, internalUsersPath, java.nio.file.StandardCopyOption.REPLACE_EXISTING); + } + + /** + * Checks if security plugin is already configured. If so, the script execution will not continue. + */ + static void checkIfSecurityPluginIsAlreadyConfigured() { + // Check if the configuration file contains the 'plugins.security' string + if (OPENSEARCH_CONF_FILE != null && new File(OPENSEARCH_CONF_FILE).exists()) { + try (BufferedReader br = new BufferedReader(new FileReader(OPENSEARCH_CONF_FILE, StandardCharsets.UTF_8))) { + String line; + while ((line = br.readLine()) != null) { + if (line.toLowerCase().contains("plugins.security")) { + System.out.println(OPENSEARCH_CONF_FILE + " seems to be already configured for Security. Quit."); + System.exit(skip_updates); + } + } + } catch (IOException e) { + System.err.println("Error reading configuration file."); + System.exit(-1); + } + } else { + System.err.println("OpenSearch configuration file does not exist. Quit."); + System.exit(-1); + } + } + + /** + * Update opensearch.yml with security configuration information + */ + static void writeSecurityConfigToOpenSearchYML() { + String securityConfig = buildSecurityConfigString(); + + try (FileWriter writer = new FileWriter(OPENSEARCH_CONF_FILE, StandardCharsets.UTF_8, true)) { + writer.write(securityConfig); + } catch (IOException e) { + System.err.println("Exception writing security configuration to opensearch.yml."); + System.exit(-1); + } + } + + /** + * Helper method to build security configuration to append to opensearch.yml + * @return the configuration string to be written to opensearch.yml + */ + static String buildSecurityConfigString() { + StringBuilder securityConfigLines = new StringBuilder(); + + securityConfigLines.append("\n") + .append("######## Start OpenSearch Security Demo Configuration ########\n") + .append("# WARNING: revise all the lines below before you go into production\n") + .append("plugins.security.ssl.transport.pemcert_filepath: esnode.pem\n") + .append("plugins.security.ssl.transport.pemkey_filepath: esnode-key.pem\n") + .append("plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem\n") + .append("plugins.security.ssl.transport.enforce_hostname_verification: false\n") + .append("plugins.security.ssl.http.enabled: true\n") + .append("plugins.security.ssl.http.pemcert_filepath: esnode.pem\n") + .append("plugins.security.ssl.http.pemkey_filepath: esnode-key.pem\n") + .append("plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem\n") + .append("plugins.security.allow_unsafe_democertificates: true\n"); + + if (initsecurity) { + securityConfigLines.append("plugins.security.allow_default_init_securityindex: true\n"); + } + + securityConfigLines.append("plugins.security.authcz.admin_dn:\n - CN=kirk,OU=client,O=client,L=test, C=de\n\n"); + + securityConfigLines.append("plugins.security.system_indices.enabled: true\n" + "plugins.security.system_indices.indices: [") + .append(SYSTEM_INDICES) + .append("]\n"); + + if (!isNetworkHostAlreadyPresent(OPENSEARCH_CONF_FILE)) { + if (cluster_mode) { + securityConfigLines.append("network.host: 0.0.0.0\n"); + securityConfigLines.append("node.name: smoketestnode\n"); + securityConfigLines.append("cluster.initial_cluster_manager_nodes: smoketestnode\n"); + } + } + + if (!isNodeMaxLocalStorageNodesAlreadyPresent(OPENSEARCH_CONF_FILE)) { + securityConfigLines.append("node.max_local_storage_nodes: 3\n"); + } + + securityConfigLines.append("######## End OpenSearch Security Demo Configuration ########\n"); + + return securityConfigLines.toString(); + } + + /** + * Helper method to check if network.host config is present + * @param filePath path to opensearch.yml + * @return true is present, false otherwise + */ + static boolean isNetworkHostAlreadyPresent(String filePath) { + try { + String searchString = "^network.host"; + return isStringAlreadyPresentInFile(filePath, searchString); + } catch (IOException e) { + return false; + } + } + + /** + * Helper method to check if node.max_local_storage_nodes config is present + * @param filePath path to opensearch.yml + * @return true if present, false otherwise + */ + static boolean isNodeMaxLocalStorageNodesAlreadyPresent(String filePath) { + try { + String searchString = "^node.max_local_storage_nodes"; + return isStringAlreadyPresentInFile(filePath, searchString); + } catch (IOException e) { + return false; + } + } + + /** + * Checks if given string is already present in the file + * @param filePath path to file in which given string should be searched + * @param searchString the string to be searched for + * @return true if string is present, false otherwise + * @throws IOException if there was exception reading the file + */ + static boolean isStringAlreadyPresentInFile(String filePath, String searchString) throws IOException { + try (BufferedReader reader = new BufferedReader(new FileReader(filePath, StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + if (line.matches(searchString)) { + return true; + } + } + } + return false; + } + + /** + * Helper method to create security_admin_demo.(sh|bat) + * @param securityAdminScriptPath path to original script + * @param securityAdminDemoScriptPath path to security admin demo script + * @throws IOException if there was error reading/writing the file + */ + void createSecurityAdminDemoScript(String securityAdminScriptPath, String securityAdminDemoScriptPath) throws IOException { + String[] securityAdminCommands; + + String securityAdminExecutionPath = securityAdminScriptPath + + "\" -cd \"" + + OPENSEARCH_CONF_DIR + + "opensearch-security\" -icl -key \"" + + OPENSEARCH_CONF_DIR + + Certificates.ADMIN_CERT_KEY.getFileName() + + "\" -cert \"" + + OPENSEARCH_CONF_DIR + + Certificates.ADMIN_CERT.getFileName() + + "\" -cacert \"" + + OPENSEARCH_CONF_DIR + + Certificates.ROOT_CA.getFileName() + + "\" -nhnv"; + + if (OS.toLowerCase().contains("win")) { + securityAdminCommands = new String[] { "@echo off", "call \"" + securityAdminExecutionPath }; + } else { + securityAdminCommands = new String[] { "#!/bin/bash", "sudo" + " \"" + securityAdminExecutionPath }; + } + + // Write securityadmin_demo script + FileWriter writer = new FileWriter(securityAdminDemoScriptPath, StandardCharsets.UTF_8); + for (String command : securityAdminCommands) { + writer.write(command + "\n"); + } + writer.close(); + } +} diff --git a/tools/install_demo_configuration.bat b/tools/install_demo_configuration.bat index d9d30fea2b..5767166b26 100755 --- a/tools/install_demo_configuration.bat +++ b/tools/install_demo_configuration.bat @@ -1,414 +1,14 @@ @echo off -setlocal enableDelayedExpansion -set "SCRIPT_DIR=%~dp0" +set DIR=%~dp0 -echo ************************************************************************** -echo ** This tool will be deprecated in the next major release of OpenSearch ** -echo ** https://github.com/opensearch-project/security/issues/1755 ** -echo ************************************************************************** - -echo. -echo OpenSearch Security Demo Installer -echo ** Warning: Do not use on production or public reachable systems ** - -echo. - -set "assumeyes=0" -set "initsecurity=0" -set "cluster_mode=0" -set "skip_updates=-1" - -goto :GETOPTS - -:show_help -echo install_demo_configuration.bat [-y] [-i] [-c] -echo -h show help -echo -y confirm all installation dialogues automatically -echo -i initialize Security plugin with default configuration (default is to ask if -y is not given) -echo -c enable cluster mode by binding to all network interfaces (default is to ask if -y is not given) -echo -s skip updates if config is already applied to opensearch.yml -EXIT /B 0 - -:GETOPTS -if /I "%1" == "-h" call :show_help & exit /b 0 -if /I "%1" == "-y" set "assumeyes=1" -if /I "%1" == "-i" set "initsecurity=1" -if /I "%1" == "-c" set "cluster_mode=1" -if /I "%1" == "-s" set "skip_updates=0" -shift -if not "%1" == "" goto :GETOPTS - -if "%1" == "--" shift - -if %assumeyes% == 0 ( - set /p "response=Install demo certificates? [y/N] " - if /I "!response!" neq "Y" exit /b 0 -) - -if %initsecurity% == 0 ( - if %assumeyes% == 0 ( - set /p "response=Initialize Security Modules? [y/N] " - if /I "!response!" == "Y" (set "initsecurity=1") ELSE (set "initsecurity=0") - ) -) - -if %cluster_mode% == 0 ( - if %assumeyes% == 0 ( - echo Cluster mode requires maybe additional setup of: - echo - Virtual memory [vm.max_map_count] - echo. - set /p "response=Enable cluster mode? [y/N] " - if /I "!response!" == "Y" (set "cluster_mode=1") ELSE (set "cluster_mode=0") - ) -) - -set BASE_DIR=%SCRIPT_DIR%\..\..\..\ -if not exist %BASE_DIR% ( - echo "basedir does not exist" - exit /b 1 -) - -set "CUR=%cd%" -cd %BASE_DIR% -set "BASE_DIR=%cd%\" -cd %CUR% -echo Basedir: %BASE_DIR% - -set "OPENSEARCH_CONF_FILE=%BASE_DIR%config\opensearch.yml" -set "INTERNAL_USERS_FILE"=%BASE_DIR%config\opensearch-security\internal_users.yml" -set "OPENSEARCH_CONF_DIR=%BASE_DIR%config\" -set "OPENSEARCH_BIN_DIR=%BASE_DIR%bin\" -set "OPENSEARCH_PLUGINS_DIR=%BASE_DIR%plugins\" -set "OPENSEARCH_MODULES_DIR=%BASE_DIR%modules\" -set "OPENSEARCH_LIB_PATH=%BASE_DIR%lib\" -set "OPENSEARCH_INSTALL_TYPE=.zip" - -if not exist %OPENSEARCH_CONF_FILE% ( - echo Unable to determine OpenSearch config file. Quit. - exit /b 1 -) - -if not exist %OPENSEARCH_BIN_DIR% ( - echo Unable to determine OpenSearch bin directory. Quit. - exit /b 1 -) - -if not exist %OPENSEARCH_PLUGINS_DIR% ( - echo Unable to determine OpenSearch plugins directory. Quit. - exit /b 1 -) - -if not exist %OPENSEARCH_MODULES_DIR% ( - echo Unable to determine OpenSearch modules directory. Quit. - exit /b 1 -) - -if not exist %OPENSEARCH_LIB_PATH% ( - echo Unable to determine OpenSearch lib directory. Quit. - exit /b 1 -) - -if not exist %OPENSEARCH_PLUGINS_DIR%\opensearch-security\ ( - echo OpenSearch Security plugin not installed. Quit. - exit /b 1 -) - -set "OPENSEARCH_VERSION=" -for %%F in ("%OPENSEARCH_LIB_PATH%opensearch-*.jar") do set "OPENSEARCH_VERSION=%%~nxF" & goto :opensearch_version -:opensearch_version -set "OPENSEARCH_JAR_VERSION=" -for /f "tokens=2 delims=[-]" %%a in ("%OPENSEARCH_VERSION%") do set "OPENSEARCH_JAR_VERSION=%%a" - -set "SECURITY_VERSION=" -for %%F in ("%OPENSEARCH_PLUGINS_DIR%\opensearch-security\opensearch-security-*.jar") do set "SECURITY_VERSION=%%~nxF" -set "SECURITY_JAR_VERSION=" -for /f "tokens=3 delims=[-]" %%a in ("%SECURITY_VERSION%") do set "SECURITY_JAR_VERSION=%%a" - -for /f "tokens=4-7 delims=[.] " %%i in ('ver') do (if %%i==Version (set "OS=%%j.%%k") else (set v="%%i.%%j")) -echo OpenSearch install type: %OPENSEARCH_INSTALL_TYPE% on %OS% -echo OpenSearch config dir: %OPENSEARCH_CONF_DIR% -echo OpenSearch config file: %OPENSEARCH_CONF_FILE% -echo OpenSearch bin dir: %OPENSEARCH_BIN_DIR% -echo OpenSearch plugins dir: %OPENSEARCH_PLUGINS_DIR% -echo OpenSearch lib dir: %OPENSEARCH_LIB_PATH% -echo Detected OpenSearch Version: %OPENSEARCH_JAR_VERSION% -echo Detected OpenSearch Security Version: %SECURITY_JAR_VERSION% - ->nul findstr /c:"plugins.security" "%OPENSEARCH_CONF_FILE%" && ( - echo %OPENSEARCH_CONF_FILE% seems to be already configured for Security. Quit. - exit /b %skip_updates% -) - -set LF=^ - - -:: two empty line required after LF -set ADMIN_CERT=-----BEGIN CERTIFICATE-----!LF!^ -MIIEmDCCA4CgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iYwDQYJKoZIhvcNAQEL!LF!^ -BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt!LF!^ -cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl!LF!^ -IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v!LF!^ -dCBDQTAeFw0yMzA4MjkyMDA2MzdaFw0zMzA4MjYyMDA2MzdaME0xCzAJBgNVBAYT!LF!^ -AmRlMQ0wCwYDVQQHDAR0ZXN0MQ8wDQYDVQQKDAZjbGllbnQxDzANBgNVBAsMBmNs!LF!^ -aWVudDENMAsGA1UEAwwEa2lyazCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC!LF!^ -ggEBAJVcOAQlCiuB9emCljROAXnlsPbG7PE3kNz2sN+BbGuw686Wgyl3uToVHvVs!LF!^ -paMmLUqm1KYz9wMSWTIBZgpJ9hYaIbGxD4RBb7qTAJ8Q4ddCV2f7T4lxao/6ixI+!LF!^ -O0l/BG9E3mRGo/r0w+jtTQ3aR2p6eoxaOYbVyEMYtFI4QZTkcgGIPGxm05y8xonx!LF!^ -vV5pbSW9L7qAVDzQC8EYGQMMI4ccu0NcHKWtmTYJA/wDPE2JwhngHwbcIbc4cDz6!LF!^ -cG0S3FmgiKGuuSqUy35v/k3y7zMHQSdx7DSR2tzhH/bBL/9qGvpT71KKrxPtaxS0!LF!^ -bAqPcEkKWDo7IMlGGW7LaAWfGg8CAwEAAaOCASswggEnMAwGA1UdEwEB/wQCMAAw!LF!^ -DgYDVR0PAQH/BAQDAgXgMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMCMIHPBgNVHSME!LF!^ -gccwgcSAFBeH36Ba62YSp9XQ+LoSRTy3KwCcoYGVpIGSMIGPMRMwEQYKCZImiZPy!LF!^ -LGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQRXhh!LF!^ -bXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENB!LF!^ -MSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0GCFHfkrz782p+T9k0G!LF!^ -xGeM4+BrehWKMB0GA1UdDgQWBBSjMS8tgguX/V7KSGLoGg7K6XMzIDANBgkqhkiG!LF!^ -9w0BAQsFAAOCAQEANMwD1JYlwAh82yG1gU3WSdh/tb6gqaSzZK7R6I0L7slaXN9m!LF!^ -y2ErUljpTyaHrdiBFmPhU/2Kj2r+fIUXtXdDXzizx/JdmueT0nG9hOixLqzfoC9p!LF!^ -fAhZxM62RgtyZoaczQN82k1/geMSwRpEndFe3OH7arkS/HSbIFxQhAIy229eWe5d!LF!^ -1bUzP59iu7f3r567I4ob8Vy7PP+Ov35p7Vv4oDHHwgsdRzX6pvL6mmwVrQ3BfVec!LF!^ -h9Dqprr+ukYmjho76g6k5cQuRaB6MxqldzUg+2E7IHQP8MCF+co51uZq2nl33mtp!LF!^ -RGr6JbdHXc96zsLTL3saJQ8AWEfu1gbTVrwyRA==!LF!^ ------END CERTIFICATE-----!LF! - - -set ADMIN_CERT_KEY=-----BEGIN PRIVATE KEY-----!LF!^ -MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCVXDgEJQorgfXp!LF!^ -gpY0TgF55bD2xuzxN5Dc9rDfgWxrsOvOloMpd7k6FR71bKWjJi1KptSmM/cDElky!LF!^ -AWYKSfYWGiGxsQ+EQW+6kwCfEOHXQldn+0+JcWqP+osSPjtJfwRvRN5kRqP69MPo!LF!^ -7U0N2kdqenqMWjmG1chDGLRSOEGU5HIBiDxsZtOcvMaJ8b1eaW0lvS+6gFQ80AvB!LF!^ -GBkDDCOHHLtDXBylrZk2CQP8AzxNicIZ4B8G3CG3OHA8+nBtEtxZoIihrrkqlMt+!LF!^ -b/5N8u8zB0Encew0kdrc4R/2wS//ahr6U+9Siq8T7WsUtGwKj3BJClg6OyDJRhlu!LF!^ -y2gFnxoPAgMBAAECggEAP5TOycDkx+megAWVoHV2fmgvgZXkBrlzQwUG/VZQi7V4!LF!^ -ZGzBMBVltdqI38wc5MtbK3TCgHANnnKgor9iq02Z4wXDwytPIiti/ycV9CDRKvv0!LF!^ -TnD2hllQFjN/IUh5n4thHWbRTxmdM7cfcNgX3aZGkYbLBVVhOMtn4VwyYu/Mxy8j!LF!^ -xClZT2xKOHkxqwmWPmdDTbAeZIbSv7RkIGfrKuQyUGUaWhrPslvYzFkYZ0umaDgQ!LF!^ -OAthZew5Bz3OfUGOMPLH61SVPuJZh9zN1hTWOvT65WFWfsPd2yStI+WD/5PU1Doo!LF!^ -1RyeHJO7s3ug8JPbtNJmaJwHe9nXBb/HXFdqb976yQKBgQDNYhpu+MYSYupaYqjs!LF!^ -9YFmHQNKpNZqgZ4ceRFZ6cMJoqpI5dpEMqToFH7tpor72Lturct2U9nc2WR0HeEs!LF!^ -/6tiptyMPTFEiMFb1opQlXF2ae7LeJllntDGN0Q6vxKnQV+7VMcXA0Y8F7tvGDy3!LF!^ -qJu5lfvB1mNM2I6y/eMxjBuQhwKBgQC6K41DXMFro0UnoO879pOQYMydCErJRmjG!LF!^ -/tZSy3Wj4KA/QJsDSViwGfvdPuHZRaG9WtxdL6kn0w1exM9Rb0bBKl36lvi7o7xv!LF!^ -M+Lw9eyXMkww8/F5d7YYH77gIhGo+RITkKI3+5BxeBaUnrGvmHrpmpgRXWmINqr0!LF!^ -0jsnN3u0OQKBgCf45vIgItSjQb8zonLz2SpZjTFy4XQ7I92gxnq8X0Q5z3B+o7tQ!LF!^ -K/4rNwTju/sGFHyXAJlX+nfcK4vZ4OBUJjP+C8CTjEotX4yTNbo3S6zjMyGQqDI5!LF!^ -9aIOUY4pb+TzeUFJX7If5gR+DfGyQubvvtcg1K3GHu9u2l8FwLj87sRzAoGAflQF!LF!^ -RHuRiG+/AngTPnZAhc0Zq0kwLkpH2Rid6IrFZhGLy8AUL/O6aa0IGoaMDLpSWUJp!LF!^ -nBY2S57MSM11/MVslrEgGmYNnI4r1K25xlaqV6K6ztEJv6n69327MS4NG8L/gCU5!LF!^ -3pEm38hkUi8pVYU7in7rx4TCkrq94OkzWJYurAkCgYATQCL/rJLQAlJIGulp8s6h!LF!^ -mQGwy8vIqMjAdHGLrCS35sVYBXG13knS52LJHvbVee39AbD5/LlWvjJGlQMzCLrw!LF!^ -F7oILW5kXxhb8S73GWcuMbuQMFVHFONbZAZgn+C9FW4l7XyRdkrbR1MRZ2km8YMs!LF!^ -/AHmo368d4PSNRMMzLHw8Q==!LF!^ ------END PRIVATE KEY-----!LF! - - -set NODE_CERT=-----BEGIN CERTIFICATE-----!LF!^ -MIIEPDCCAySgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iIwDQYJKoZIhvcNAQEL!LF!^ -BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt!LF!^ -cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl!LF!^ -IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v!LF!^ -dCBDQTAeFw0yMzA4MjkwNDIzMTJaFw0zMzA4MjYwNDIzMTJaMFcxCzAJBgNVBAYT!LF!^ -AmRlMQ0wCwYDVQQHDAR0ZXN0MQ0wCwYDVQQKDARub2RlMQ0wCwYDVQQLDARub2Rl!LF!^ -MRswGQYDVQQDDBJub2RlLTAuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA!LF!^ -A4IBDwAwggEKAoIBAQCm93kXteDQHMAvbUPNPW5pyRHKDD42XGWSgq0k1D29C/Ud!LF!^ -yL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0!LF!^ -HGkn47XVu3EwbfrTENg3jFu+Oem6a/501SzITzJWtS0cn2dIFOBimTVpT/4Zv5qr!LF!^ -XA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8n!LF!^ -dibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b6l+KLo3IKpfTbAIJXIO+M67FLtWKtttD!LF!^ -ao94B069skzKk6FPgW/OZh6PRCD0oxOavV+ld2SjAgMBAAGjgcYwgcMwRwYDVR0R!LF!^ -BEAwPogFKgMEBQWCEm5vZGUtMC5leGFtcGxlLmNvbYIJbG9jYWxob3N0hxAAAAAA!LF!^ -AAAAAAAAAAAAAAABhwR/AAABMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEF!LF!^ -BQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU0/qDQaY10jIo!LF!^ -wCjLUpz/HfQXyt8wHwYDVR0jBBgwFoAUF4ffoFrrZhKn1dD4uhJFPLcrAJwwDQYJ!LF!^ -KoZIhvcNAQELBQADggEBAD2hkndVih6TWxoe/oOW0i2Bq7ScNO/n7/yHWL04HJmR!LF!^ -MaHv/Xjc8zLFLgHuHaRvC02ikWIJyQf5xJt0Oqu2GVbqXH9PBGKuEP2kCsRRyU27!LF!^ -zTclAzfQhqmKBTYQ/3lJ3GhRQvXIdYTe+t4aq78TCawp1nSN+vdH/1geG6QjMn5N!LF!^ -1FU8tovDd4x8Ib/0dv8RJx+n9gytI8n/giIaDCEbfLLpe4EkV5e5UNpOnRgJjjuy!LF!^ -vtZutc81TQnzBtkS9XuulovDE0qI+jQrKkKu8xgGLhgH0zxnPkKtUg2I3Aq6zl1L!LF!^ -zYkEOUF8Y25J6WeY88Yfnc0iigI+Pnz5NK8R9GL7TYo=!LF!^ ------END CERTIFICATE-----!LF! - - -set NODE_KEY=-----BEGIN PRIVATE KEY-----!LF!^ -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCm93kXteDQHMAv!LF!^ -bUPNPW5pyRHKDD42XGWSgq0k1D29C/UdyL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0!LF!^ -o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0HGkn47XVu3EwbfrTENg3jFu+Oem6a/50!LF!^ -1SzITzJWtS0cn2dIFOBimTVpT/4Zv5qrXA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1!LF!^ -MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8ndibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b!LF!^ -6l+KLo3IKpfTbAIJXIO+M67FLtWKtttDao94B069skzKk6FPgW/OZh6PRCD0oxOa!LF!^ -vV+ld2SjAgMBAAECggEAQK1+uAOZeaSZggW2jQut+MaN4JHLi61RH2cFgU3COLgo!LF!^ -FIiNjFn8f2KKU3gpkt1It8PjlmprpYut4wHI7r6UQfuv7ZrmncRiPWHm9PB82+ZQ!LF!^ -5MXYqj4YUxoQJ62Cyz4sM6BobZDrjG6HHGTzuwiKvHHkbsEE9jQ4E5m7yfbVvM0O!LF!^ -zvwrSOM1tkZihKSTpR0j2+taji914tjBssbn12TMZQL5ItGnhR3luY8mEwT9MNkZ!LF!^ -xg0VcREoAH+pu9FE0vPUgLVzhJ3be7qZTTSRqv08bmW+y1plu80GbppePcgYhEow!LF!^ -dlW4l6XPJaHVSn1lSFHE6QAx6sqiAnBz0NoTPIaLyQKBgQDZqDOlhCRciMRicSXn!LF!^ -7yid9rhEmdMkySJHTVFOidFWwlBcp0fGxxn8UNSBcXdSy7GLlUtH41W9PWl8tp9U!LF!^ -hQiiXORxOJ7ZcB80uNKXF01hpPj2DpFPWyHFxpDkWiTAYpZl68rOlYujxZUjJIej!LF!^ -VvcykBC2BlEOG9uZv2kxcqLyJwKBgQDEYULTxaTuLIa17wU3nAhaainKB3vHxw9B!LF!^ -Ksy5p3ND43UNEKkQm7K/WENx0q47TA1mKD9i+BhaLod98mu0YZ+BCUNgWKcBHK8c!LF!^ -uXpauvM/pLhFLXZ2jvEJVpFY3J79FSRK8bwE9RgKfVKMMgEk4zOyZowS8WScOqiy!LF!^ -hnQn1vKTJQKBgElhYuAnl9a2qXcC7KOwRsJS3rcKIVxijzL4xzOyVShp5IwIPbOv!LF!^ -hnxBiBOH/JGmaNpFYBcBdvORE9JfA4KMQ2fx53agfzWRjoPI1/7mdUk5RFI4gRb/!LF!^ -A3jZRBoopgFSe6ArCbnyQxzYzToG48/Wzwp19ZxYrtUR4UyJct6f5n27AoGBAJDh!LF!^ -KIpQQDOvCdtjcbfrF4aM2DPCfaGPzENJriwxy6oEPzDaX8Bu/dqI5Ykt43i/zQrX!LF!^ -GpyLaHvv4+oZVTiI5UIvcVO9U8hQPyiz9f7F+fu0LHZs6f7hyhYXlbe3XFxeop3f!LF!^ -5dTKdWgXuTTRF2L9dABkA2deS9mutRKwezWBMQk5AoGBALPtX0FrT1zIosibmlud!LF!^ -tu49A/0KZu4PBjrFMYTSEWGNJez3Fb2VsJwylVl6HivwbP61FhlYfyksCzQQFU71!LF!^ -+x7Nmybp7PmpEBECr3deoZKQ/acNHn0iwb0It+YqV5+TquQebqgwK6WCLsMuiYKT!LF!^ -bg/ch9Rhxbq22yrVgWHh6epp!LF!^ ------END PRIVATE KEY-----!LF! - - -set ROOT_CA=-----BEGIN CERTIFICATE-----!LF!^ -MIIExjCCA66gAwIBAgIUd+SvPvzan5P2TQbEZ4zj4Gt6FYowDQYJKoZIhvcNAQEL!LF!^ -BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt!LF!^ -cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl!LF!^ -IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v!LF!^ -dCBDQTAeFw0yMzA4MjkwNDIwMDNaFw0yMzA5MjgwNDIwMDNaMIGPMRMwEQYKCZIm!LF!^ -iZPyLGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQ!LF!^ -RXhhbXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290!LF!^ -IENBMSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0EwggEiMA0GCSqG!LF!^ -SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEPyN7J9VGPyJcQmCBl5TGwfSzvVdWwoQU!LF!^ -j9aEsdfFJ6pBCDQSsj8Lv4RqL0dZra7h7SpZLLX/YZcnjikrYC+rP5OwsI9xEE/4!LF!^ -U98CsTBPhIMgqFK6SzNE5494BsAk4cL72dOOc8tX19oDS/PvBULbNkthQ0aAF1dg!LF!^ -vbrHvu7hq7LisB5ZRGHVE1k/AbCs2PaaKkn2jCw/b+U0Ml9qPuuEgz2mAqJDGYoA!LF!^ -WSR4YXrOcrmPuRqbws464YZbJW898/0Pn/U300ed+4YHiNYLLJp51AMkR4YEw969!LF!^ -VRPbWIvLrd0PQBooC/eLrL6rvud/GpYhdQEUx8qcNCKd4bz3OaQ5AgMBAAGjggEW!LF!^ -MIIBEjAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQU!LF!^ -F4ffoFrrZhKn1dD4uhJFPLcrAJwwgc8GA1UdIwSBxzCBxIAUF4ffoFrrZhKn1dD4!LF!^ -uhJFPLcrAJyhgZWkgZIwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJ!LF!^ -k/IsZAEZFgdleGFtcGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYD!LF!^ -VQQLDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUg!LF!^ -Q29tIEluYy4gUm9vdCBDQYIUd+SvPvzan5P2TQbEZ4zj4Gt6FYowDQYJKoZIhvcN!LF!^ -AQELBQADggEBAIopqco/k9RSjouTeKP4z0EVUxdD4qnNh1GLSRqyAVe0aChyKF5f!LF!^ -qt1Bd1XCY8D16RgekkKGHDpJhGCpel+vtIoXPBxUaGQNYxmJCf5OzLMODlcrZk5i!LF!^ -jHIcv/FMeK02NBcz/WQ3mbWHVwXLhmwqa2zBsF4FmPCJAbFLchLhkAv1HJifHbnD!LF!^ -jQzlKyl5jxam/wtjWxSm0iyso0z2TgyzY+MESqjEqB1hZkCFzD1xtUOCxbXgtKae!LF!^ -dgfHVFuovr3fNLV3GvQk0s9okDwDUcqV7DSH61e5bUMfE84o3of8YA7+HUoPV5Du!LF!^ -8sTOKRf7ncGXdDRA8aofW268pTCuIu3+g/Y=!LF!^ ------END CERTIFICATE-----!LF! - - -echo !ADMIN_CERT! > "%OPENSEARCH_CONF_DIR%kirk.pem" -echo !NODE_CERT! > "%OPENSEARCH_CONF_DIR%esnode.pem" -echo !ROOT_CA! > "%OPENSEARCH_CONF_DIR%root-ca.pem" -echo !NODE_KEY! > "%OPENSEARCH_CONF_DIR%esnode-key.pem" -echo !ADMIN_CERT_KEY! > "%OPENSEARCH_CONF_DIR%kirk-key.pem" - -echo. >> "%OPENSEARCH_CONF_FILE%" -echo ######## Start OpenSearch Security Demo Configuration ######## >> "%OPENSEARCH_CONF_FILE%" -echo # WARNING: revise all the lines below before you go into production >> "%OPENSEARCH_CONF_FILE%" -echo plugins.security.ssl.transport.pemcert_filepath: esnode.pem >> "%OPENSEARCH_CONF_FILE%" -echo plugins.security.ssl.transport.pemkey_filepath: esnode-key.pem >> "%OPENSEARCH_CONF_FILE%" -echo plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem >> "%OPENSEARCH_CONF_FILE%" -echo plugins.security.ssl.transport.enforce_hostname_verification: false >> "%OPENSEARCH_CONF_FILE%" -echo plugins.security.ssl.http.enabled: true >> "%OPENSEARCH_CONF_FILE%" -echo plugins.security.ssl.http.pemcert_filepath: esnode.pem >> "%OPENSEARCH_CONF_FILE%" -echo plugins.security.ssl.http.pemkey_filepath: esnode-key.pem >> "%OPENSEARCH_CONF_FILE%" -echo plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem >> "%OPENSEARCH_CONF_FILE%" -echo plugins.security.allow_unsafe_democertificates: true >> "%OPENSEARCH_CONF_FILE%" -if %initsecurity% == 1 ( - echo plugins.security.allow_default_init_securityindex: true >> "%OPENSEARCH_CONF_FILE%" -) -echo plugins.security.authcz.admin_dn: >> "%OPENSEARCH_CONF_FILE%" -echo - CN=kirk,OU=client,O=client,L=test, C=de >> "%OPENSEARCH_CONF_FILE%" -echo. >> "%OPENSEARCH_CONF_FILE%" -echo plugins.security.audit.type: internal_opensearch >> "%OPENSEARCH_CONF_FILE%" -echo plugins.security.enable_snapshot_restore_privilege: true >> "%OPENSEARCH_CONF_FILE%" -echo plugins.security.check_snapshot_restore_write_privileges: true >> "%OPENSEARCH_CONF_FILE%" -echo plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"] >> "%OPENSEARCH_CONF_FILE%" -echo plugins.security.system_indices.enabled: true >> "%OPENSEARCH_CONF_FILE%" -echo plugins.security.system_indices.indices: [".plugins-ml-config", ".plugins-ml-connector", ".plugins-ml-model-group", ".plugins-ml-model", ".plugins-ml-task", ".plugins-ml-conversation-meta", ".plugins-ml-conversation-interactions", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opensearch-notifications-*", ".opensearch-notebooks", ".opensearch-observability", ".ql-datasources", ".opendistro-asynchronous-search-response*", ".replication-metadata-store", ".opensearch-knn-models", ".geospatial-ip2geo-data*"] >> "%OPENSEARCH_CONF_FILE%" - -setlocal enabledelayedexpansion - -set "ADMIN_PASSWORD_FILE=%OPENSEARCH_CONF_DIR%initialAdminPassword.txt" -set "INTERNAL_USERS_FILE=%OPENSEARCH_CONF_DIR%opensearch-security\internal_users.yml" - -echo "what is in the config directory" -dir %OPENSEARCH_CONF_DIR% - -echo "what is in the password file" -type "%ADMIN_PASSWORD_FILE%" - - -if "%initialAdminPassword%" NEQ "" ( - set "ADMIN_PASSWORD=!initialAdminPassword!" +if defined OPENSEARCH_JAVA_HOME ( + set BIN_PATH="%OPENSEARCH_JAVA_HOME%\bin\java.exe" +) else if defined JAVA_HOME ( + set BIN_PATH="%JAVA_HOME%\bin\java.exe" ) else ( - for /f %%a in ('type "%ADMIN_PASSWORD_FILE%"') do set "ADMIN_PASSWORD=%%a" -) - -if not defined ADMIN_PASSWORD ( - echo Unable to find the admin password for the cluster. Please set initialAdminPassword or create a file %ADMIN_PASSWORD_FILE% with a single line that contains the password. + echo Unable to find java runtime + echo OPENSEARCH_JAVA_HOME or JAVA_HOME must be defined exit /b 1 ) -echo " ***************************************************" -echo " *** ADMIN PASSWORD SET TO: %ADMIN_PASSWORD% ***" -echo " ***************************************************" - -set "HASH_SCRIPT=%OPENSEARCH_PLUGINS_DIR%\opensearch-security\tools\hash.bat" - -REM Run the command and capture its output -for /f %%a in ('%HASH_SCRIPT% -p !ADMIN_PASSWORD!') do ( - set "HASHED_ADMIN_PASSWORD=%%a" -) - -if errorlevel 1 ( - echo Failed to hash the admin password - exit /b 1 -) - -set "default_line= hash: "$2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG"" -set "search=%default_line%" -set "replace= hash: "%HASHED_ADMIN_PASSWORD%"" - -setlocal enableextensions -for /f "delims=" %%i in ('type "%INTERNAL_USERS_FILE%" ^& break ^> "%INTERNAL_USERS_FILE%" ') do ( - set "line=%%i" - setlocal enabledelayedexpansion - >>"%INTERNAL_USERS_FILE%" echo(!line:%search%=%replace%! - endlocal -) - -:: network.host ->nul findstr /b /c:"network.host" "%OPENSEARCH_CONF_FILE%" && ( - echo network.host already present -) || ( - if %cluster_mode% == 1 ( - echo network.host: 0.0.0.0 >> "%OPENSEARCH_CONF_FILE%" - echo node.name: smoketestnode >> "%OPENSEARCH_CONF_FILE%" - echo cluster.initial_cluster_manager_nodes: smoketestnode >> "%OPENSEARCH_CONF_FILE%" - ) -) - ->nul findstr /b /c:"node.max_local_storage_nodes" "%OPENSEARCH_CONF_FILE%" && ( - echo node.max_local_storage_nodes already present -) || ( - echo node.max_local_storage_nodes: 3 >> "%OPENSEARCH_CONF_FILE%" -) - -echo ######## End OpenSearch Security Demo Configuration ######## >> "%OPENSEARCH_CONF_FILE%" - -echo ### Success -echo ### Execute this script now on all your nodes and then start all nodes -:: Generate securityadmin_demo.bat -echo. > securityadmin_demo.bat -echo %OPENSEARCH_PLUGINS_DIR%opensearch-security\tools\securityadmin.bat -cd %OPENSEARCH_CONF_DIR%opensearch-security -icl -key %OPENSEARCH_CONF_DIR%kirk-key.pem -cert %OPENSEARCH_CONF_DIR%kirk.pem -cacert %OPENSEARCH_CONF_DIR%root-ca.pem -nhnv >> securityadmin_demo.bat - -if %initsecurity% == 0 ( - echo ### After the whole cluster is up execute: - type securityadmin_demo.bat - echo ### or run ./securityadmin_demo.bat - echo ### After that you can also use the Security Plugin ConfigurationGUI -) else ( - echo ### OpenSearch Security will be automatically initialized. - echo ### If you like to change the runtime configuration - echo ### change the files in ../../../config/opensearch-security and execute: - type securityadmin_demo.bat - echo ### or run ./securityadmin_demo.bat - echo ### To use the Security Plugin ConfigurationGUI -) - -echo ### To access your secured cluster open https://: and log in with admin/admin. -echo ### [Ignore the SSL certificate warning because we installed self-signed demo certificates] +%BIN_PATH% -Dorg.apache.logging.log4j.simplelog.StatusLogger.level=OFF -cp "%DIR%\..\*;%DIR%\..\..\..\lib\*;%DIR%\..\deps\*" org.opensearch.security.tools.democonfig.Installer %DIR% %* 2> nul \ No newline at end of file diff --git a/tools/install_demo_configuration.sh b/tools/install_demo_configuration.sh index 01bc1bfed1..7835f7c675 100755 --- a/tools/install_demo_configuration.sh +++ b/tools/install_demo_configuration.sh @@ -1,11 +1,6 @@ #!/bin/bash #install_demo_configuration.sh [-y] -echo "**************************************************************************" -echo "** This tool will be deprecated in the next major release of OpenSearch **" -echo "** https://github.com/opensearch-project/security/issues/1755 **" -echo "**************************************************************************" - SCRIPT_PATH="${BASH_SOURCE[0]}" if ! [ -x "$(command -v realpath)" ]; then if [ -L "$SCRIPT_PATH" ]; then @@ -21,455 +16,17 @@ else DIR="$( cd "$( dirname "$(realpath "$SCRIPT_PATH")" )" && pwd -P)" fi -echo "OpenSearch Security Demo Installer" -echo " ** Warning: Do not use on production or public reachable systems **" - -OPTIND=1 -assumeyes=0 -initsecurity=0 -cluster_mode=0 -skip_updates=-1 - -function show_help() { - echo "install_demo_configuration.sh [-y] [-i] [-c]" - echo " -h show help" - echo " -y confirm all installation dialogues automatically" - echo " -i initialize Security plugin with default configuration (default is to ask if -y is not given)" - echo " -c enable cluster mode by binding to all network interfaces (default is to ask if -y is not given)" - echo " -s skip updates if config is already applied to opensearch.yml" -} - -while getopts "h?yics" opt; do - case "$opt" in - h|\?) - show_help - exit 0 - ;; - y) assumeyes=1 - ;; - i) initsecurity=1 - ;; - c) cluster_mode=1 - ;; - s) skip_updates=0 - esac -done - -shift $((OPTIND-1)) - -[ "$1" = "--" ] && shift - -if [ "$assumeyes" == 0 ]; then - read -r -p "Install demo certificates? [y/N] " response - case "$response" in - [yY][eE][sS]|[yY]) - ;; - *) - exit 0 - ;; - esac -fi - -if [ "$initsecurity" == 0 ] && [ "$assumeyes" == 0 ]; then - read -r -p "Initialize Security Modules? [y/N] " response - case "$response" in - [yY][eE][sS]|[yY]) - initsecurity=1 - ;; - *) - initsecurity=0 - ;; - esac -fi - -if [ "$cluster_mode" == 0 ] && [ "$assumeyes" == 0 ]; then - echo "Cluster mode requires maybe additional setup of:" - echo " - Virtual memory (vm.max_map_count)" - echo "" - read -r -p "Enable cluster mode? [y/N] " response - case "$response" in - [yY][eE][sS]|[yY]) - cluster_mode=1 - ;; - *) - cluster_mode=0 - ;; - esac -fi - -set -e -BASE_DIR="$DIR/../../.." -if [ -d "$BASE_DIR" ]; then - CUR="$(pwd)" - cd "$BASE_DIR" - BASE_DIR="$(pwd)" - cd "$CUR" - echo "Basedir: $BASE_DIR" -else - echo "DEBUG: basedir does not exist" -fi - -OPENSEARCH_CONF_FILE="$BASE_DIR/config/opensearch.yml" -OPENSEARCH_BIN_DIR="$BASE_DIR/bin" -OPENSEARCH_PLUGINS_DIR="$BASE_DIR/plugins" -OPENSEARCH_MODULES_DIR="$BASE_DIR/modules" -OPENSEARCH_LIB_PATH="$BASE_DIR/lib" -SUDO_CMD="" -OPENSEARCH_INSTALL_TYPE=".tar.gz" - -#Check if its a rpm/deb install -if [ "/usr/share/opensearch" -ef "$BASE_DIR" ]; then - OPENSEARCH_CONF_FILE="/usr/share/opensearch/config/opensearch.yml" - - if [ ! -f "$OPENSEARCH_CONF_FILE" ]; then - OPENSEARCH_CONF_FILE="/etc/opensearch/opensearch.yml" - fi - - if [ -x "$(command -v sudo)" ]; then - SUDO_CMD="sudo" - echo "This script maybe require your root password for 'sudo' privileges" - fi - - OPENSEARCH_INSTALL_TYPE="rpm/deb" -fi - -if [ $SUDO_CMD ]; then - if ! [ -x "$(command -v $SUDO_CMD)" ]; then - echo "Unable to locate 'sudo' command. Quit." - exit 1 - fi -fi - -if $SUDO_CMD test -f "$OPENSEARCH_CONF_FILE"; then - : -else - echo "Unable to determine OpenSearch config directory. Quit." - exit -1 -fi - -if [ ! -d "$OPENSEARCH_BIN_DIR" ]; then - echo "Unable to determine OpenSearch bin directory. Quit." - exit -1 -fi - -if [ ! -d "$OPENSEARCH_PLUGINS_DIR" ]; then - echo "Unable to determine OpenSearch plugins directory. Quit." - exit -1 -fi - -if [ ! -d "$OPENSEARCH_MODULES_DIR" ]; then - echo "Unable to determine OpenSearch modules directory. Quit." - #exit -1 -fi - -if [ ! -d "$OPENSEARCH_LIB_PATH" ]; then - echo "Unable to determine OpenSearch lib directory. Quit." - exit -1 -fi - -OPENSEARCH_CONF_DIR=$(dirname "${OPENSEARCH_CONF_FILE}") -OPENSEARCH_CONF_DIR=`cd "$OPENSEARCH_CONF_DIR" ; pwd` - -if [ ! -d "$OPENSEARCH_PLUGINS_DIR/opensearch-security" ]; then - echo "OpenSearch Security plugin not installed. Quit." - exit -1 -fi - -OPENSEARCH_VERSION=("$OPENSEARCH_LIB_PATH/opensearch-*.jar") -OPENSEARCH_VERSION=$(echo $OPENSEARCH_VERSION | sed 's/.*opensearch-\(.*\)\.jar/\1/') - -SECURITY_VERSION=("$OPENSEARCH_PLUGINS_DIR/opensearch-security/opensearch-security-*.jar") -SECURITY_VERSION=$(echo $SECURITY_VERSION | sed 's/.*opensearch-security-\(.*\)\.jar/\1/') - -OS=$(sb_release -ds 2>/dev/null || cat /etc/*release 2>/dev/null | head -n1 || uname -om) -echo "OpenSearch install type: $OPENSEARCH_INSTALL_TYPE on $OS" -echo "OpenSearch config dir: $OPENSEARCH_CONF_DIR" -echo "OpenSearch config file: $OPENSEARCH_CONF_FILE" -echo "OpenSearch bin dir: $OPENSEARCH_BIN_DIR" -echo "OpenSearch plugins dir: $OPENSEARCH_PLUGINS_DIR" -echo "OpenSearch lib dir: $OPENSEARCH_LIB_PATH" -echo "Detected OpenSearch Version: $OPENSEARCH_VERSION" -echo "Detected OpenSearch Security Version: $SECURITY_VERSION" - -if $SUDO_CMD grep --quiet -i plugins.security "$OPENSEARCH_CONF_FILE"; then - echo "$OPENSEARCH_CONF_FILE seems to be already configured for Security. Quit." - exit $skip_updates -fi - -set +e - -read -r -d '' ADMIN_CERT << EOM ------BEGIN CERTIFICATE----- -MIIEmDCCA4CgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iYwDQYJKoZIhvcNAQEL -BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt -cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl -IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v -dCBDQTAeFw0yMzA4MjkyMDA2MzdaFw0zMzA4MjYyMDA2MzdaME0xCzAJBgNVBAYT -AmRlMQ0wCwYDVQQHDAR0ZXN0MQ8wDQYDVQQKDAZjbGllbnQxDzANBgNVBAsMBmNs -aWVudDENMAsGA1UEAwwEa2lyazCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC -ggEBAJVcOAQlCiuB9emCljROAXnlsPbG7PE3kNz2sN+BbGuw686Wgyl3uToVHvVs -paMmLUqm1KYz9wMSWTIBZgpJ9hYaIbGxD4RBb7qTAJ8Q4ddCV2f7T4lxao/6ixI+ -O0l/BG9E3mRGo/r0w+jtTQ3aR2p6eoxaOYbVyEMYtFI4QZTkcgGIPGxm05y8xonx -vV5pbSW9L7qAVDzQC8EYGQMMI4ccu0NcHKWtmTYJA/wDPE2JwhngHwbcIbc4cDz6 -cG0S3FmgiKGuuSqUy35v/k3y7zMHQSdx7DSR2tzhH/bBL/9qGvpT71KKrxPtaxS0 -bAqPcEkKWDo7IMlGGW7LaAWfGg8CAwEAAaOCASswggEnMAwGA1UdEwEB/wQCMAAw -DgYDVR0PAQH/BAQDAgXgMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMCMIHPBgNVHSME -gccwgcSAFBeH36Ba62YSp9XQ+LoSRTy3KwCcoYGVpIGSMIGPMRMwEQYKCZImiZPy -LGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQRXhh -bXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENB -MSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0GCFHfkrz782p+T9k0G -xGeM4+BrehWKMB0GA1UdDgQWBBSjMS8tgguX/V7KSGLoGg7K6XMzIDANBgkqhkiG -9w0BAQsFAAOCAQEANMwD1JYlwAh82yG1gU3WSdh/tb6gqaSzZK7R6I0L7slaXN9m -y2ErUljpTyaHrdiBFmPhU/2Kj2r+fIUXtXdDXzizx/JdmueT0nG9hOixLqzfoC9p -fAhZxM62RgtyZoaczQN82k1/geMSwRpEndFe3OH7arkS/HSbIFxQhAIy229eWe5d -1bUzP59iu7f3r567I4ob8Vy7PP+Ov35p7Vv4oDHHwgsdRzX6pvL6mmwVrQ3BfVec -h9Dqprr+ukYmjho76g6k5cQuRaB6MxqldzUg+2E7IHQP8MCF+co51uZq2nl33mtp -RGr6JbdHXc96zsLTL3saJQ8AWEfu1gbTVrwyRA== ------END CERTIFICATE----- -EOM - -read -r -d '' ADMIN_CERT_KEY << EOM ------BEGIN PRIVATE KEY----- -MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCVXDgEJQorgfXp -gpY0TgF55bD2xuzxN5Dc9rDfgWxrsOvOloMpd7k6FR71bKWjJi1KptSmM/cDElky -AWYKSfYWGiGxsQ+EQW+6kwCfEOHXQldn+0+JcWqP+osSPjtJfwRvRN5kRqP69MPo -7U0N2kdqenqMWjmG1chDGLRSOEGU5HIBiDxsZtOcvMaJ8b1eaW0lvS+6gFQ80AvB -GBkDDCOHHLtDXBylrZk2CQP8AzxNicIZ4B8G3CG3OHA8+nBtEtxZoIihrrkqlMt+ -b/5N8u8zB0Encew0kdrc4R/2wS//ahr6U+9Siq8T7WsUtGwKj3BJClg6OyDJRhlu -y2gFnxoPAgMBAAECggEAP5TOycDkx+megAWVoHV2fmgvgZXkBrlzQwUG/VZQi7V4 -ZGzBMBVltdqI38wc5MtbK3TCgHANnnKgor9iq02Z4wXDwytPIiti/ycV9CDRKvv0 -TnD2hllQFjN/IUh5n4thHWbRTxmdM7cfcNgX3aZGkYbLBVVhOMtn4VwyYu/Mxy8j -xClZT2xKOHkxqwmWPmdDTbAeZIbSv7RkIGfrKuQyUGUaWhrPslvYzFkYZ0umaDgQ -OAthZew5Bz3OfUGOMPLH61SVPuJZh9zN1hTWOvT65WFWfsPd2yStI+WD/5PU1Doo -1RyeHJO7s3ug8JPbtNJmaJwHe9nXBb/HXFdqb976yQKBgQDNYhpu+MYSYupaYqjs -9YFmHQNKpNZqgZ4ceRFZ6cMJoqpI5dpEMqToFH7tpor72Lturct2U9nc2WR0HeEs -/6tiptyMPTFEiMFb1opQlXF2ae7LeJllntDGN0Q6vxKnQV+7VMcXA0Y8F7tvGDy3 -qJu5lfvB1mNM2I6y/eMxjBuQhwKBgQC6K41DXMFro0UnoO879pOQYMydCErJRmjG -/tZSy3Wj4KA/QJsDSViwGfvdPuHZRaG9WtxdL6kn0w1exM9Rb0bBKl36lvi7o7xv -M+Lw9eyXMkww8/F5d7YYH77gIhGo+RITkKI3+5BxeBaUnrGvmHrpmpgRXWmINqr0 -0jsnN3u0OQKBgCf45vIgItSjQb8zonLz2SpZjTFy4XQ7I92gxnq8X0Q5z3B+o7tQ -K/4rNwTju/sGFHyXAJlX+nfcK4vZ4OBUJjP+C8CTjEotX4yTNbo3S6zjMyGQqDI5 -9aIOUY4pb+TzeUFJX7If5gR+DfGyQubvvtcg1K3GHu9u2l8FwLj87sRzAoGAflQF -RHuRiG+/AngTPnZAhc0Zq0kwLkpH2Rid6IrFZhGLy8AUL/O6aa0IGoaMDLpSWUJp -nBY2S57MSM11/MVslrEgGmYNnI4r1K25xlaqV6K6ztEJv6n69327MS4NG8L/gCU5 -3pEm38hkUi8pVYU7in7rx4TCkrq94OkzWJYurAkCgYATQCL/rJLQAlJIGulp8s6h -mQGwy8vIqMjAdHGLrCS35sVYBXG13knS52LJHvbVee39AbD5/LlWvjJGlQMzCLrw -F7oILW5kXxhb8S73GWcuMbuQMFVHFONbZAZgn+C9FW4l7XyRdkrbR1MRZ2km8YMs -/AHmo368d4PSNRMMzLHw8Q== ------END PRIVATE KEY----- -EOM - -read -r -d '' NODE_CERT << EOM ------BEGIN CERTIFICATE----- -MIIEPDCCAySgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iIwDQYJKoZIhvcNAQEL -BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt -cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl -IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v -dCBDQTAeFw0yMzA4MjkwNDIzMTJaFw0zMzA4MjYwNDIzMTJaMFcxCzAJBgNVBAYT -AmRlMQ0wCwYDVQQHDAR0ZXN0MQ0wCwYDVQQKDARub2RlMQ0wCwYDVQQLDARub2Rl -MRswGQYDVQQDDBJub2RlLTAuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA -A4IBDwAwggEKAoIBAQCm93kXteDQHMAvbUPNPW5pyRHKDD42XGWSgq0k1D29C/Ud -yL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0 -HGkn47XVu3EwbfrTENg3jFu+Oem6a/501SzITzJWtS0cn2dIFOBimTVpT/4Zv5qr -XA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8n -dibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b6l+KLo3IKpfTbAIJXIO+M67FLtWKtttD -ao94B069skzKk6FPgW/OZh6PRCD0oxOavV+ld2SjAgMBAAGjgcYwgcMwRwYDVR0R -BEAwPogFKgMEBQWCEm5vZGUtMC5leGFtcGxlLmNvbYIJbG9jYWxob3N0hxAAAAAA -AAAAAAAAAAAAAAABhwR/AAABMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEF -BQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU0/qDQaY10jIo -wCjLUpz/HfQXyt8wHwYDVR0jBBgwFoAUF4ffoFrrZhKn1dD4uhJFPLcrAJwwDQYJ -KoZIhvcNAQELBQADggEBAD2hkndVih6TWxoe/oOW0i2Bq7ScNO/n7/yHWL04HJmR -MaHv/Xjc8zLFLgHuHaRvC02ikWIJyQf5xJt0Oqu2GVbqXH9PBGKuEP2kCsRRyU27 -zTclAzfQhqmKBTYQ/3lJ3GhRQvXIdYTe+t4aq78TCawp1nSN+vdH/1geG6QjMn5N -1FU8tovDd4x8Ib/0dv8RJx+n9gytI8n/giIaDCEbfLLpe4EkV5e5UNpOnRgJjjuy -vtZutc81TQnzBtkS9XuulovDE0qI+jQrKkKu8xgGLhgH0zxnPkKtUg2I3Aq6zl1L -zYkEOUF8Y25J6WeY88Yfnc0iigI+Pnz5NK8R9GL7TYo= ------END CERTIFICATE----- -EOM - -read -r -d '' NODE_KEY << EOM ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCm93kXteDQHMAv -bUPNPW5pyRHKDD42XGWSgq0k1D29C/UdyL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0 -o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0HGkn47XVu3EwbfrTENg3jFu+Oem6a/50 -1SzITzJWtS0cn2dIFOBimTVpT/4Zv5qrXA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1 -MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8ndibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b -6l+KLo3IKpfTbAIJXIO+M67FLtWKtttDao94B069skzKk6FPgW/OZh6PRCD0oxOa -vV+ld2SjAgMBAAECggEAQK1+uAOZeaSZggW2jQut+MaN4JHLi61RH2cFgU3COLgo -FIiNjFn8f2KKU3gpkt1It8PjlmprpYut4wHI7r6UQfuv7ZrmncRiPWHm9PB82+ZQ -5MXYqj4YUxoQJ62Cyz4sM6BobZDrjG6HHGTzuwiKvHHkbsEE9jQ4E5m7yfbVvM0O -zvwrSOM1tkZihKSTpR0j2+taji914tjBssbn12TMZQL5ItGnhR3luY8mEwT9MNkZ -xg0VcREoAH+pu9FE0vPUgLVzhJ3be7qZTTSRqv08bmW+y1plu80GbppePcgYhEow -dlW4l6XPJaHVSn1lSFHE6QAx6sqiAnBz0NoTPIaLyQKBgQDZqDOlhCRciMRicSXn -7yid9rhEmdMkySJHTVFOidFWwlBcp0fGxxn8UNSBcXdSy7GLlUtH41W9PWl8tp9U -hQiiXORxOJ7ZcB80uNKXF01hpPj2DpFPWyHFxpDkWiTAYpZl68rOlYujxZUjJIej -VvcykBC2BlEOG9uZv2kxcqLyJwKBgQDEYULTxaTuLIa17wU3nAhaainKB3vHxw9B -Ksy5p3ND43UNEKkQm7K/WENx0q47TA1mKD9i+BhaLod98mu0YZ+BCUNgWKcBHK8c -uXpauvM/pLhFLXZ2jvEJVpFY3J79FSRK8bwE9RgKfVKMMgEk4zOyZowS8WScOqiy -hnQn1vKTJQKBgElhYuAnl9a2qXcC7KOwRsJS3rcKIVxijzL4xzOyVShp5IwIPbOv -hnxBiBOH/JGmaNpFYBcBdvORE9JfA4KMQ2fx53agfzWRjoPI1/7mdUk5RFI4gRb/ -A3jZRBoopgFSe6ArCbnyQxzYzToG48/Wzwp19ZxYrtUR4UyJct6f5n27AoGBAJDh -KIpQQDOvCdtjcbfrF4aM2DPCfaGPzENJriwxy6oEPzDaX8Bu/dqI5Ykt43i/zQrX -GpyLaHvv4+oZVTiI5UIvcVO9U8hQPyiz9f7F+fu0LHZs6f7hyhYXlbe3XFxeop3f -5dTKdWgXuTTRF2L9dABkA2deS9mutRKwezWBMQk5AoGBALPtX0FrT1zIosibmlud -tu49A/0KZu4PBjrFMYTSEWGNJez3Fb2VsJwylVl6HivwbP61FhlYfyksCzQQFU71 -+x7Nmybp7PmpEBECr3deoZKQ/acNHn0iwb0It+YqV5+TquQebqgwK6WCLsMuiYKT -bg/ch9Rhxbq22yrVgWHh6epp ------END PRIVATE KEY----- -EOM - -read -r -d '' ROOT_CA << EOM ------BEGIN CERTIFICATE----- -MIIExjCCA66gAwIBAgIUd+SvPvzan5P2TQbEZ4zj4Gt6FYowDQYJKoZIhvcNAQEL -BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt -cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl -IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v -dCBDQTAeFw0yMzA4MjkwNDIwMDNaFw0yMzA5MjgwNDIwMDNaMIGPMRMwEQYKCZIm -iZPyLGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQ -RXhhbXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290 -IENBMSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0EwggEiMA0GCSqG -SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEPyN7J9VGPyJcQmCBl5TGwfSzvVdWwoQU -j9aEsdfFJ6pBCDQSsj8Lv4RqL0dZra7h7SpZLLX/YZcnjikrYC+rP5OwsI9xEE/4 -U98CsTBPhIMgqFK6SzNE5494BsAk4cL72dOOc8tX19oDS/PvBULbNkthQ0aAF1dg -vbrHvu7hq7LisB5ZRGHVE1k/AbCs2PaaKkn2jCw/b+U0Ml9qPuuEgz2mAqJDGYoA -WSR4YXrOcrmPuRqbws464YZbJW898/0Pn/U300ed+4YHiNYLLJp51AMkR4YEw969 -VRPbWIvLrd0PQBooC/eLrL6rvud/GpYhdQEUx8qcNCKd4bz3OaQ5AgMBAAGjggEW -MIIBEjAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQU -F4ffoFrrZhKn1dD4uhJFPLcrAJwwgc8GA1UdIwSBxzCBxIAUF4ffoFrrZhKn1dD4 -uhJFPLcrAJyhgZWkgZIwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJ -k/IsZAEZFgdleGFtcGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYD -VQQLDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUg -Q29tIEluYy4gUm9vdCBDQYIUd+SvPvzan5P2TQbEZ4zj4Gt6FYowDQYJKoZIhvcN -AQELBQADggEBAIopqco/k9RSjouTeKP4z0EVUxdD4qnNh1GLSRqyAVe0aChyKF5f -qt1Bd1XCY8D16RgekkKGHDpJhGCpel+vtIoXPBxUaGQNYxmJCf5OzLMODlcrZk5i -jHIcv/FMeK02NBcz/WQ3mbWHVwXLhmwqa2zBsF4FmPCJAbFLchLhkAv1HJifHbnD -jQzlKyl5jxam/wtjWxSm0iyso0z2TgyzY+MESqjEqB1hZkCFzD1xtUOCxbXgtKae -dgfHVFuovr3fNLV3GvQk0s9okDwDUcqV7DSH61e5bUMfE84o3of8YA7+HUoPV5Du -8sTOKRf7ncGXdDRA8aofW268pTCuIu3+g/Y= ------END CERTIFICATE----- -EOM - -set -e - -echo "$ADMIN_CERT" | $SUDO_CMD tee "$OPENSEARCH_CONF_DIR/kirk.pem" > /dev/null -echo "$NODE_CERT" | $SUDO_CMD tee "$OPENSEARCH_CONF_DIR/esnode.pem" > /dev/null -echo "$ROOT_CA" | $SUDO_CMD tee "$OPENSEARCH_CONF_DIR/root-ca.pem" > /dev/null -echo "$NODE_KEY" | $SUDO_CMD tee "$OPENSEARCH_CONF_DIR/esnode-key.pem" > /dev/null -echo "$ADMIN_CERT_KEY" | $SUDO_CMD tee "$OPENSEARCH_CONF_DIR/kirk-key.pem" > /dev/null - -chmod 0600 "$OPENSEARCH_CONF_DIR/kirk.pem" -chmod 0600 "$OPENSEARCH_CONF_DIR/esnode.pem" -chmod 0600 "$OPENSEARCH_CONF_DIR/root-ca.pem" -chmod 0600 "$OPENSEARCH_CONF_DIR/esnode-key.pem" -chmod 0600 "$OPENSEARCH_CONF_DIR/kirk-key.pem" - -echo "" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" -echo "######## Start OpenSearch Security Demo Configuration ########" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -echo "# WARNING: revise all the lines below before you go into production" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -echo "plugins.security.ssl.transport.pemcert_filepath: esnode.pem" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -echo "plugins.security.ssl.transport.pemkey_filepath: esnode-key.pem" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -echo "plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -echo "plugins.security.ssl.transport.enforce_hostname_verification: false" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -echo "plugins.security.ssl.http.enabled: true" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -echo "plugins.security.ssl.http.pemcert_filepath: esnode.pem" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -echo "plugins.security.ssl.http.pemkey_filepath: esnode-key.pem" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -echo "plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -echo "plugins.security.allow_unsafe_democertificates: true" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -if [ "$initsecurity" == 1 ]; then - echo "plugins.security.allow_default_init_securityindex: true" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -fi -echo "plugins.security.authcz.admin_dn:" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -echo " - CN=kirk,OU=client,O=client,L=test, C=de" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -echo "" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -echo "plugins.security.audit.type: internal_opensearch" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -echo "plugins.security.enable_snapshot_restore_privilege: true" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -echo "plugins.security.check_snapshot_restore_write_privileges: true" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -echo 'plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"]' | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -echo 'plugins.security.system_indices.enabled: true' | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -echo 'plugins.security.system_indices.indices: [".plugins-ml-config", ".plugins-ml-connector", ".plugins-ml-model-group", ".plugins-ml-model", ".plugins-ml-task", ".plugins-ml-conversation-meta", ".plugins-ml-conversation-interactions", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opensearch-notifications-*", ".opensearch-notebooks", ".opensearch-observability", ".ql-datasources", ".opendistro-asynchronous-search-response*", ".replication-metadata-store", ".opensearch-knn-models", ".geospatial-ip2geo-data*"]' | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null - -## Read the admin password from the file or use the initialAdminPassword if set -ADMIN_PASSWORD_FILE="$OPENSEARCH_CONF_DIR/initialAdminPassword.txt" -INTERNAL_USERS_FILE="$OPENSEARCH_CONF_DIR/opensearch-security/internal_users.yml" - -if [[ -n "$initialAdminPassword" ]]; then - ADMIN_PASSWORD="$initialAdminPassword" -elif [[ -f "$ADMIN_PASSWORD_FILE" && -s "$ADMIN_PASSWORD_FILE" ]]; then - ADMIN_PASSWORD=$(head -n 1 "$ADMIN_PASSWORD_FILE") -else - echo "Unable to find the admin password for the cluster. Please run 'export initialAdminPassword=' or create a file $ADMIN_PASSWORD_FILE with a single line that contains the password." - exit 1 -fi - -echo " ***************************************************" -echo " *** ADMIN PASSWORD SET TO: $ADMIN_PASSWORD ***" -echo " ***************************************************" - -$SUDO_CMD chmod +x "$OPENSEARCH_PLUGINS_DIR/opensearch-security/tools/hash.sh" - -# Use the Hasher script to hash the admin password -HASHED_ADMIN_PASSWORD=$($OPENSEARCH_PLUGINS_DIR/opensearch-security/tools/hash.sh -p "$ADMIN_PASSWORD" | tail -n 1) - -if [ $? -ne 0 ]; then - echo "Hash the admin password failure, see console for details" - exit 1 -fi - -# Find the line number containing 'admin:' in the internal_users.yml file -ADMIN_HASH_LINE=$(grep -n 'admin:' "$INTERNAL_USERS_FILE" | cut -f1 -d:) - -awk -v hashed_admin_password="$HASHED_ADMIN_PASSWORD" ' - /^ *hash: *"\$2a\$12\$VcCDgh2NDk07JGN0rjGbM.Ad41qVR\/YFJcgHp0UGns5JDymv..TOG"/ { - sub(/"\$2a\$12\$VcCDgh2NDk07JGN0rjGbM.Ad41qVR\/YFJcgHp0UGns5JDymv..TOG"/, "\"" hashed_admin_password "\""); - } - { print } -' "$INTERNAL_USERS_FILE" > temp_file && mv temp_file "$INTERNAL_USERS_FILE" - -#network.host -if $SUDO_CMD grep --quiet -i "^network.host" "$OPENSEARCH_CONF_FILE"; then - : #already present -else - if [ "$cluster_mode" == 1 ]; then - echo "network.host: 0.0.0.0" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null - echo "node.name: smoketestnode" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null - echo "cluster.initial_cluster_manager_nodes: smoketestnode" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null - fi -fi - -if $SUDO_CMD grep --quiet -i "^node.max_local_storage_nodes" "$OPENSEARCH_CONF_FILE"; then - : #already present -else - echo 'node.max_local_storage_nodes: 3' | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null -fi - - - -echo "######## End OpenSearch Security Demo Configuration ########" | $SUDO_CMD tee -a "$OPENSEARCH_CONF_FILE" > /dev/null - -$SUDO_CMD chmod +x "$OPENSEARCH_PLUGINS_DIR/opensearch-security/tools/securityadmin.sh" - -OPENSEARCH_PLUGINS_DIR=`cd "$OPENSEARCH_PLUGINS_DIR" ; pwd` - -echo "### Success" -echo "### Execute this script now on all your nodes and then start all nodes" -#Generate securityadmin_demo.sh -echo "#!/bin/bash" | $SUDO_CMD tee securityadmin_demo.sh > /dev/null -echo $SUDO_CMD \""$OPENSEARCH_PLUGINS_DIR/opensearch-security/tools/securityadmin.sh"\" -cd \""$OPENSEARCH_CONF_DIR/opensearch-security"\" -icl -key \""$OPENSEARCH_CONF_DIR/kirk-key.pem"\" -cert \""$OPENSEARCH_CONF_DIR/kirk.pem"\" -cacert \""$OPENSEARCH_CONF_DIR/root-ca.pem"\" -nhnv | $SUDO_CMD tee -a securityadmin_demo.sh > /dev/null -$SUDO_CMD chmod +x securityadmin_demo.sh +BIN_PATH="java" -if [ "$initsecurity" == 0 ]; then - echo "### After the whole cluster is up execute: " - $SUDO_CMD cat securityadmin_demo.sh | tail -1 - echo "### or run ./securityadmin_demo.sh" - echo "### After that you can also use the Security Plugin ConfigurationGUI" +# now set the path to java: first OPENSEARCH_JAVA_HOME, then JAVA_HOME +if [ -n "$OPENSEARCH_JAVA_HOME" ]; then + BIN_PATH="$OPENSEARCH_JAVA_HOME/bin/java" +elif [ -n "$JAVA_HOME" ]; then + BIN_PATH="$JAVA_HOME/bin/java" else - echo "### OpenSearch Security will be automatically initialized." - echo "### If you like to change the runtime configuration " - echo "### change the files in ../../../config/opensearch-security and execute: " - $SUDO_CMD cat securityadmin_demo.sh | tail -1 - echo "### or run ./securityadmin_demo.sh" - echo "### To use the Security Plugin ConfigurationGUI" + echo "Unable to find java runtime" + echo "OPENSEARCH_JAVA_HOME or JAVA_HOME must be defined" + exit 1 fi -echo "### To access your secured cluster open https://: and log in with admin/admin." -echo "### (Ignore the SSL certificate warning because we installed self-signed demo certificates)" +"$BIN_PATH" $JAVA_OPTS -Dorg.apache.logging.log4j.simplelog.StatusLogger.level=OFF -cp "$DIR/../*:$DIR/../../../lib/*:$DIR/../deps/*" org.opensearch.security.tools.democonfig.Installer "$DIR" "$@" 2>/dev/null From 9be1b272f99421cdf7470cf30c44e11f1061f371 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Fri, 17 Nov 2023 04:18:28 -0500 Subject: [PATCH 011/204] Run precommit checks before checkstyle task (#3727) ### Description This PR runs the License Header check with the checkstyle CI check to ensure that PRs contain licenses on top of new files. This PR also adds the license on 2 files where its missing today. * Category (Enhancement, New feature, Bug fix, Test fix, Refactoring, Maintenance, Documentation) Maintenance ### Issues Resolved - Resolves https://github.com/opensearch-project/security/issues/3471 - Resolves https://github.com/opensearch-project/security/issues/3691 ### Check List - [ ] New functionality includes testing - [ ] New functionality has been documented - [ ] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: Craig Perkins --- build.gradle | 1 + .../org/opensearch/security/filter/NettyAttribute.java | 10 ++++++++++ .../opensearch/security/filter/SecurityResponse.java | 1 - .../security/filter/SecurityRestUtilsTests.java | 10 ++++++++++ 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 7717034434..2f933c8886 100644 --- a/build.gradle +++ b/build.gradle @@ -386,6 +386,7 @@ checkstyle { } tasks.withType(Checkstyle) { + dependsOn(':precommit') reports { ignoreFailures = false } diff --git a/src/main/java/org/opensearch/security/filter/NettyAttribute.java b/src/main/java/org/opensearch/security/filter/NettyAttribute.java index 685e94e199..46bf0296cb 100644 --- a/src/main/java/org/opensearch/security/filter/NettyAttribute.java +++ b/src/main/java/org/opensearch/security/filter/NettyAttribute.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.security.filter; import java.util.Optional; diff --git a/src/main/java/org/opensearch/security/filter/SecurityResponse.java b/src/main/java/org/opensearch/security/filter/SecurityResponse.java index 5041936d2e..1c6146d0e1 100644 --- a/src/main/java/org/opensearch/security/filter/SecurityResponse.java +++ b/src/main/java/org/opensearch/security/filter/SecurityResponse.java @@ -36,7 +36,6 @@ public class SecurityResponse { public SecurityResponse(final int status, final Exception e) { this.status = status; - populateHeaders(CONTENT_TYPE_APP_JSON); this.body = generateFailureMessage(e); this.contentType = XContentType.JSON.mediaType(); } diff --git a/src/test/java/org/opensearch/security/filter/SecurityRestUtilsTests.java b/src/test/java/org/opensearch/security/filter/SecurityRestUtilsTests.java index 46b0e82f2a..50ae2157c9 100644 --- a/src/test/java/org/opensearch/security/filter/SecurityRestUtilsTests.java +++ b/src/test/java/org/opensearch/security/filter/SecurityRestUtilsTests.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.security.filter; import org.junit.Test; From 4496440de11e1d1435ff5803942bbdad0c6be513 Mon Sep 17 00:00:00 2001 From: Darshit Chanpura <35282393+DarshitChanpura@users.noreply.github.com> Date: Fri, 17 Nov 2023 13:15:07 -0500 Subject: [PATCH 012/204] Adds missing default security config lines when updating opensearch.yml (#3734) Adds 4 config lines that were missed in the original PR: https://github.com/opensearch-project/security/blob/deff84265cd22badf9cca02a3240aeb000acb439/tools/install_demo_configuration.sh#L384C1-L388C1 --------- Signed-off-by: Darshit Chanpura --- .../security/tools/democonfig/Certificates.java | 11 +++++++++++ .../tools/democonfig/ExecutionEnvironment.java | 11 +++++++++++ .../tools/democonfig/SecuritySettingsConfigurer.java | 10 +++++++--- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/opensearch/security/tools/democonfig/Certificates.java b/src/main/java/org/opensearch/security/tools/democonfig/Certificates.java index 6821147e8c..c776a5e29b 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/Certificates.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/Certificates.java @@ -1,3 +1,14 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + package org.opensearch.security.tools.democonfig; /** diff --git a/src/main/java/org/opensearch/security/tools/democonfig/ExecutionEnvironment.java b/src/main/java/org/opensearch/security/tools/democonfig/ExecutionEnvironment.java index 9f901c4487..e9a8273c5f 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/ExecutionEnvironment.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/ExecutionEnvironment.java @@ -1,3 +1,14 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + package org.opensearch.security.tools.democonfig; /** diff --git a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java index 1e318c38b9..a5daa579dd 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java @@ -212,9 +212,13 @@ static String buildSecurityConfigString() { securityConfigLines.append("plugins.security.authcz.admin_dn:\n - CN=kirk,OU=client,O=client,L=test, C=de\n\n"); - securityConfigLines.append("plugins.security.system_indices.enabled: true\n" + "plugins.security.system_indices.indices: [") - .append(SYSTEM_INDICES) - .append("]\n"); + securityConfigLines.append("plugins.security.audit.type: internal_opensearch\n"); + securityConfigLines.append("plugins.security.enable_snapshot_restore_privilege: true\n"); + securityConfigLines.append("plugins.security.check_snapshot_restore_write_privileges: true\n"); + securityConfigLines.append("plugins.security.restapi.roles_enabled: [\"all_access\", \"security_rest_api_access\"]\n"); + + securityConfigLines.append("plugins.security.system_indices.enabled: true\n"); + securityConfigLines.append("plugins.security.system_indices.indices: [").append(SYSTEM_INDICES).append("]\n"); if (!isNetworkHostAlreadyPresent(OPENSEARCH_CONF_FILE)) { if (cluster_mode) { From ca8aafe3ff75b1dbc39f4ed857d5c385e0025bcf Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Fri, 17 Nov 2023 12:44:33 -0600 Subject: [PATCH 013/204] Removing flaky test case testAutoInit (#3732) Signed-off-by: Peter Nied --- .../RestApiComplianceAuditlogTest.java | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/src/test/java/org/opensearch/security/auditlog/compliance/RestApiComplianceAuditlogTest.java b/src/test/java/org/opensearch/security/auditlog/compliance/RestApiComplianceAuditlogTest.java index dfabe50431..552e661ff3 100644 --- a/src/test/java/org/opensearch/security/auditlog/compliance/RestApiComplianceAuditlogTest.java +++ b/src/test/java/org/opensearch/security/auditlog/compliance/RestApiComplianceAuditlogTest.java @@ -125,31 +125,6 @@ public void testRestApiRolesDisabledGet() throws Exception { validateMsgs(TestAuditlogImpl.messages); } - @Test - public void testAutoInit() throws Exception { - - Settings additionalSettings = Settings.builder() - .put("plugins.security.audit.type", TestAuditlogImpl.class.getName()) - .put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_ENABLE_TRANSPORT, true) - .put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_ENABLE_REST, true) - .put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_RESOLVE_BULK_REQUESTS, false) - .put(ConfigConstants.OPENDISTRO_SECURITY_COMPLIANCE_HISTORY_EXTERNAL_CONFIG_ENABLED, true) - .put(ConfigConstants.SECURITY_COMPLIANCE_HISTORY_INTERNAL_CONFIG_ENABLED, true) - .put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_TRANSPORT_CATEGORIES, "authenticated,GRANTED_PRIVILEGES") - .put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_REST_CATEGORIES, "authenticated,GRANTED_PRIVILEGES") - .build(); - - setup(additionalSettings); - - Thread.sleep(1500); - - Assert.assertTrue(TestAuditlogImpl.messages.size() > 2); - Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("audit_request_effective_user")); - Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("COMPLIANCE_INTERNAL_CONFIG_WRITE")); - Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("COMPLIANCE_EXTERNAL_CONFIG")); - validateMsgs(TestAuditlogImpl.messages); - } - @Test public void testRestApiNewUser() throws Exception { From b72a9cf96a77caefb4a0cd9ba816168408efeb1b Mon Sep 17 00:00:00 2001 From: Daniel Widdis Date: Fri, 17 Nov 2023 13:16:41 -0800 Subject: [PATCH 014/204] Force newer version of Eclipse core transitive dependency (resolves CVE-2023-4218) (#3737) ### Description The Spotless Gradle Plugin brings in a transitive dependency on Eclipse Core Runtime 3.26.100. That version is impacted by a CVE. This forces the newest version, currently 3.29.0. Note that newer versions than 3.26 require JDK17+ to run spotless. Signed-off-by: Daniel Widdis --- .github/workflows/code-hygiene.yml | 2 +- build.gradle | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/code-hygiene.yml b/.github/workflows/code-hygiene.yml index 1b46c65a63..6ed51248e9 100644 --- a/.github/workflows/code-hygiene.yml +++ b/.github/workflows/code-hygiene.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/setup-java@v3 with: distribution: temurin # Temurin is a distribution of adoptium - java-version: 11 + java-version: 17 - uses: gradle/gradle-build-action@v2 with: diff --git a/build.gradle b/build.gradle index 2f933c8886..ea5df5419e 100644 --- a/build.gradle +++ b/build.gradle @@ -488,6 +488,9 @@ configurations { // for spotbugs dependency conflict force "org.apache.commons:commons-lang3:${versions.commonslang}" + // for spotless transitive dependency CVE + force "org.eclipse.platform:org.eclipse.core.runtime:3.29.0" + // For integrationTest force "org.apache.httpcomponents:httpclient:4.5.14" force "org.apache.httpcomponents:httpcore:4.4.16" From 18d4851e341a5f2112dea35830ebe48734121616 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Nov 2023 13:49:08 +0100 Subject: [PATCH 015/204] Bump org.springframework:spring-core from 5.3.30 to 5.3.31 (#3744) Bumps [org.springframework:spring-core](https://github.com/spring-projects/spring-framework) from 5.3.30 to 5.3.31.
Release notes

Sourced from org.springframework:spring-core's releases.

v5.3.31

:star: New Features

  • Log4jLog needs to re-resolve ExtendedLogger on deserialization (for compatibility with Log4J 2.21) #31583

:lady_beetle: Bug Fixes

  • MessageBuilder#createMessage should not define the payload as @Nullable #31611
  • Avoid duplicate JAR resources in PathMatchingResourcePatternResolver on MS Windows #31603
  • Spring web integration commons fileupload receives files and other parameter uploads, with a null pointer #31564
  • Function column out doesn't resolve to SqlOutParameter #31560
  • Resolve to empty MultiValueMap when no matrix variables are provided #31484
  • BeanUtils.copyProperties() consumes large amount of memory #31481
  • CGLIB BeanCopier falls back to ClassLoader.defineClass for public target #31436
  • R2DBC Connection is closed during transaction when using TransactionAwareConnectionFactoryProxy #31411
  • HibernateJpaDialect and HibernateExceptionTranslator throw SQLExceptionTranslator-provided exception instead of returning it #31410
  • NamedParameterJdbcTemplate throws unexpected exception for null query #31394
  • LazyResolutionMessage does not implement proper toString #31385
  • Illegal reflective access in ContextOverridingClassLoader.isEligibleForOverriding #31233

:notebook_with_decorative_cover: Documentation

  • Clarify documentation for @Transactional on interfaces #31401
  • Default behavior of BeanPropertyRowMapper.getColumnValue(ResultSet, int, Class) inconsistent with code #31349
  • Referencing a @Bean method in a @Configuration class' @PostConstruct method leads to circular reference #31339
  • Incorrect reference information about CGLIB supported method visibility #31311

:hammer: Dependency Upgrades

  • Upgrade to Reactor 2020.0.38 #31584
Commits
  • 8c85c31 Release v5.3.31
  • b28a5f8 Upgrade to Reactor 2020.0.38 and Netty 4.1.101
  • e660859 Consistent ordering of overloaded operations
  • a6ab308 Fix wrong nullability requirement
  • c489234 Polish contribution
  • af59358 Avoid duplicate resources in PathMatchingResourcePatternResolver on Windows
  • 75193b2 Upgrade to Log4J 2.21.1, Tomcat 9.0.82, Jetty 9.4.53, Undertow 2.2.28, Netty ...
  • f97e819 Log4jLog re-resolves ExtendedLogger on deserialization
  • f8e1ce3 Check for procedure vs function constants in CallMetaDataContext
  • d3ec939 Guard for empty FileItems in CommonsFileUploadSupport
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.springframework:spring-core&package-manager=gradle&previous-version=5.3.30&new-version=5.3.31)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index ea5df5419e..aaf623721b 100644 --- a/build.gradle +++ b/build.gradle @@ -697,7 +697,7 @@ dependencies { testCompileOnly 'org.apiguardian:apiguardian-api:1.1.2' // Kafka test execution testRuntimeOnly 'org.springframework.retry:spring-retry:1.3.4' - testRuntimeOnly ('org.springframework:spring-core:5.3.30') { + testRuntimeOnly ('org.springframework:spring-core:5.3.31') { exclude(group:'org.springframework', module: 'spring-jcl' ) } testRuntimeOnly 'org.scala-lang:scala-library:2.13.12' From 4913601bda880556a1893a9cc8e437323c0db1c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Nov 2023 13:49:21 +0100 Subject: [PATCH 016/204] Bump org.springframework:spring-beans from 5.3.30 to 5.3.31 (#3743) Bumps [org.springframework:spring-beans](https://github.com/spring-projects/spring-framework) from 5.3.30 to 5.3.31.
Release notes

Sourced from org.springframework:spring-beans's releases.

v5.3.31

:star: New Features

  • Log4jLog needs to re-resolve ExtendedLogger on deserialization (for compatibility with Log4J 2.21) #31583

:lady_beetle: Bug Fixes

  • MessageBuilder#createMessage should not define the payload as @Nullable #31611
  • Avoid duplicate JAR resources in PathMatchingResourcePatternResolver on MS Windows #31603
  • Spring web integration commons fileupload receives files and other parameter uploads, with a null pointer #31564
  • Function column out doesn't resolve to SqlOutParameter #31560
  • Resolve to empty MultiValueMap when no matrix variables are provided #31484
  • BeanUtils.copyProperties() consumes large amount of memory #31481
  • CGLIB BeanCopier falls back to ClassLoader.defineClass for public target #31436
  • R2DBC Connection is closed during transaction when using TransactionAwareConnectionFactoryProxy #31411
  • HibernateJpaDialect and HibernateExceptionTranslator throw SQLExceptionTranslator-provided exception instead of returning it #31410
  • NamedParameterJdbcTemplate throws unexpected exception for null query #31394
  • LazyResolutionMessage does not implement proper toString #31385
  • Illegal reflective access in ContextOverridingClassLoader.isEligibleForOverriding #31233

:notebook_with_decorative_cover: Documentation

  • Clarify documentation for @Transactional on interfaces #31401
  • Default behavior of BeanPropertyRowMapper.getColumnValue(ResultSet, int, Class) inconsistent with code #31349
  • Referencing a @Bean method in a @Configuration class' @PostConstruct method leads to circular reference #31339
  • Incorrect reference information about CGLIB supported method visibility #31311

:hammer: Dependency Upgrades

  • Upgrade to Reactor 2020.0.38 #31584
Commits
  • 8c85c31 Release v5.3.31
  • b28a5f8 Upgrade to Reactor 2020.0.38 and Netty 4.1.101
  • e660859 Consistent ordering of overloaded operations
  • a6ab308 Fix wrong nullability requirement
  • c489234 Polish contribution
  • af59358 Avoid duplicate resources in PathMatchingResourcePatternResolver on Windows
  • 75193b2 Upgrade to Log4J 2.21.1, Tomcat 9.0.82, Jetty 9.4.53, Undertow 2.2.28, Netty ...
  • f97e819 Log4jLog re-resolves ExtendedLogger on deserialization
  • f8e1ce3 Check for procedure vs function constants in CallMetaDataContext
  • d3ec939 Guard for empty FileItems in CommonsFileUploadSupport
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.springframework:spring-beans&package-manager=gradle&previous-version=5.3.30&new-version=5.3.31)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index aaf623721b..355d743a58 100644 --- a/build.gradle +++ b/build.gradle @@ -685,7 +685,7 @@ dependencies { testImplementation "org.apache.kafka:kafka-clients:${kafka_version}:test" testImplementation 'commons-validator:commons-validator:1.7' testImplementation 'org.springframework.kafka:spring-kafka-test:2.9.13' - testImplementation 'org.springframework:spring-beans:5.3.30' + testImplementation 'org.springframework:spring-beans:5.3.31' testImplementation 'org.junit.jupiter:junit-jupiter:5.10.1' testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.1' // Only osx-x86_64, osx-aarch_64, linux-x86_64, linux-aarch_64, windows-x86_64 are available From a7138a44366bd4d8a5041a0c2fa146db2343bafa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Nov 2023 13:50:01 +0100 Subject: [PATCH 017/204] Bump actions/github-script from 6 to 7 (#3742) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/github-script](https://github.com/actions/github-script) from 6 to 7.
Release notes

Sourced from actions/github-script's releases.

v7.0.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/github-script/compare/v6.4.1...v7.0.0

v6.4.1

What's Changed

New Contributors

Full Changelog: https://github.com/actions/github-script/compare/v6.4.0...v6.4.1

v6.4.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/github-script/compare/v6.3.3...v6.4.0

v6.3.3

What's Changed

New Contributors

Full Changelog: https://github.com/actions/github-script/compare/v6.3.2...v6.3.3

v6.3.2

What's Changed

... (truncated)

Commits
  • 60a0d83 Merge pull request #440 from actions/joshmgross/v7.0.1
  • b7fb200 Update version to 7.0.1
  • 12e22ed Merge pull request #439 from actions/joshmgross/avoid-setting-base-url
  • d319f8f Avoid setting baseUrl to undefined when input is not provided
  • e69ef54 Merge pull request #425 from actions/joshmgross/node-20
  • ee0914b Update licenses
  • d6fc56f Use @types/node for Node 20
  • 384d6cf Fix quotations in tests
  • 8472492 Only validate GraphQL previews
  • 84903f5 Remove node-fetch from type
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/github-script&package-manager=github_actions&previous-version=6&new-version=7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/add-untriaged.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/add-untriaged.yml b/.github/workflows/add-untriaged.yml index 15b9a55651..864fd26dd5 100644 --- a/.github/workflows/add-untriaged.yml +++ b/.github/workflows/add-untriaged.yml @@ -8,7 +8,7 @@ jobs: apply-label: runs-on: ubuntu-latest steps: - - uses: actions/github-script@v6 + - uses: actions/github-script@v7 with: script: | github.rest.issues.addLabels({ From daa584d80705fd6ee254716dc8342df4cc6dbbd1 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Mon, 20 Nov 2023 10:41:19 -0500 Subject: [PATCH 018/204] Add spring_version variable to have single instance of spring version (#3747) ### Description Extracts `spring_version` to a variable so it can be re-used. This will cut down slightly on maintenance where dependabot can make multiple PRs to update this version. Recent examples: - https://github.com/opensearch-project/security/pull/3744 - https://github.com/opensearch-project/security/pull/3743 * Category (Enhancement, New feature, Bug fix, Test fix, Refactoring, Maintenance, Documentation) Maintenance ### Check List - [ ] New functionality includes testing - [ ] New functionality has been documented - [ ] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: Craig Perkins --- build.gradle | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 355d743a58..3c118a9f19 100644 --- a/build.gradle +++ b/build.gradle @@ -32,6 +32,7 @@ buildscript { jjwt_version = '0.12.3' guava_version = '32.1.3-jre' jaxb_version = '2.3.9' + spring_version = '5.3.31' if (buildVersionQualifier) { opensearch_build += "-${buildVersionQualifier}" @@ -685,7 +686,7 @@ dependencies { testImplementation "org.apache.kafka:kafka-clients:${kafka_version}:test" testImplementation 'commons-validator:commons-validator:1.7' testImplementation 'org.springframework.kafka:spring-kafka-test:2.9.13' - testImplementation 'org.springframework:spring-beans:5.3.31' + testImplementation "org.springframework:spring-beans:${spring_version}" testImplementation 'org.junit.jupiter:junit-jupiter:5.10.1' testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.1' // Only osx-x86_64, osx-aarch_64, linux-x86_64, linux-aarch_64, windows-x86_64 are available @@ -697,7 +698,7 @@ dependencies { testCompileOnly 'org.apiguardian:apiguardian-api:1.1.2' // Kafka test execution testRuntimeOnly 'org.springframework.retry:spring-retry:1.3.4' - testRuntimeOnly ('org.springframework:spring-core:5.3.31') { + testRuntimeOnly ("org.springframework:spring-core:${spring_version}") { exclude(group:'org.springframework', module: 'spring-jcl' ) } testRuntimeOnly 'org.scala-lang:scala-library:2.13.12' From 5114bb71b91cfd412b335c14c0a73ea51a3eb9a2 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Mon, 20 Nov 2023 14:59:43 -0500 Subject: [PATCH 019/204] Add release notes for 2.11.1 (#3749) ### Description Add release notes for 2.11.1 Signed-off-by: Craig Perkins --- .../opensearch-security.release-notes-2.11.1.0.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 release-notes/opensearch-security.release-notes-2.11.1.0.md diff --git a/release-notes/opensearch-security.release-notes-2.11.1.0.md b/release-notes/opensearch-security.release-notes-2.11.1.0.md new file mode 100644 index 0000000000..0164f2819e --- /dev/null +++ b/release-notes/opensearch-security.release-notes-2.11.1.0.md @@ -0,0 +1,7 @@ +## 2023-11-21 Version 2.11.1.0 + +Compatible with OpenSearch 2.11.1 + +### Bug Fixes +* Fix regression on concurrent gzipped requests ([#3599](https://github.com/opensearch-project/security/pull/3599)) +* Fix issue with response content-types changed in 2.11 ([#3721](https://github.com/opensearch-project/security/pull/3721)) From 26617e2b04c3d3ab8faf04564193f13a19c8db74 Mon Sep 17 00:00:00 2001 From: Prabhas Kurapati <66924475+prabhask5@users.noreply.github.com> Date: Tue, 21 Nov 2023 08:21:02 -0800 Subject: [PATCH 020/204] HttpStatus is only imported from `org.apache.http` (#3741) Signed-off-by: Prabhas Kurapati --- checkstyle/checkstyle.xml | 1 + gradle/formatting.gradle | 4 ++++ .../java/org/opensearch/security/http/AsyncTests.java | 2 +- .../security/http/OnBehalfOfJwtAuthenticationTest.java | 2 +- .../security/http/ServiceAccountAuthenticationTest.java | 2 +- .../org/opensearch/security/rest/AuthZinRestLayerTests.java | 2 +- .../java/org/opensearch/security/rest/CompressionTests.java | 2 +- .../java/org/opensearch/security/rest/WhoAmITests.java | 2 +- .../java/com/amazon/dlic/auth/ldap/LdapBackendIntegTest.java | 2 +- .../com/amazon/dlic/auth/ldap2/LdapBackendIntegTest2.java | 2 +- .../opensearch/security/AdvancedSecurityMigrationTests.java | 2 +- src/test/java/org/opensearch/security/AggregationTests.java | 2 +- .../org/opensearch/security/DataStreamIntegrationTests.java | 2 +- .../security/EncryptionInTransitMigrationTests.java | 2 +- src/test/java/org/opensearch/security/HealthTests.java | 2 +- .../java/org/opensearch/security/HttpIntegrationTests.java | 2 +- .../java/org/opensearch/security/IndexIntegrationTests.java | 2 +- .../security/IndexTemplateClusterPermissionsCheckTest.java | 2 +- .../opensearch/security/InitializationIntegrationTests.java | 2 +- src/test/java/org/opensearch/security/IntegrationTests.java | 2 +- .../java/org/opensearch/security/PitIntegrationTests.java | 2 +- src/test/java/org/opensearch/security/ResolveAPITests.java | 2 +- .../org/opensearch/security/SecurityAdminIEndpointsTests.java | 2 +- .../opensearch/security/SecurityAdminInvalidConfigsTests.java | 2 +- src/test/java/org/opensearch/security/SecurityAdminTests.java | 2 +- src/test/java/org/opensearch/security/SecurityRolesTests.java | 2 +- .../java/org/opensearch/security/SlowIntegrationTests.java | 2 +- .../java/org/opensearch/security/SnapshotRestoreTests.java | 2 +- .../java/org/opensearch/security/SystemIntegratorsTests.java | 2 +- src/test/java/org/opensearch/security/TaskTests.java | 2 +- src/test/java/org/opensearch/security/TracingTests.java | 2 +- .../java/org/opensearch/security/auditlog/AuditTestUtils.java | 2 +- .../security/auditlog/compliance/ComplianceAuditlogTest.java | 2 +- .../auditlog/compliance/RestApiComplianceAuditlogTest.java | 2 +- .../org/opensearch/security/auditlog/impl/TracingTests.java | 2 +- .../security/auditlog/integration/BasicAuditlogTest.java | 2 +- .../security/auditlog/integration/SSLAuditlogTest.java | 2 +- src/test/java/org/opensearch/security/cache/CachingTest.java | 2 +- .../opensearch/security/ccstest/CrossClusterSearchTests.java | 2 +- .../org/opensearch/security/ccstest/RemoteReindexTests.java | 2 +- .../dlic/dlsfls/CustomFieldMaskedComplexMappingTest.java | 2 +- .../security/dlic/dlsfls/CustomFieldMaskedTest.java | 2 +- .../org/opensearch/security/dlic/dlsfls/DateMathTest.java | 2 +- .../org/opensearch/security/dlic/dlsfls/DlsDateMathTest.java | 2 +- .../security/dlic/dlsfls/DlsFlsCrossClusterSearchTest.java | 2 +- .../org/opensearch/security/dlic/dlsfls/DlsNestedTest.java | 2 +- .../opensearch/security/dlic/dlsfls/DlsPropsReplaceTest.java | 2 +- .../org/opensearch/security/dlic/dlsfls/DlsScrollTest.java | 2 +- .../java/org/opensearch/security/dlic/dlsfls/DlsTest.java | 2 +- .../org/opensearch/security/dlic/dlsfls/FieldMaskedTest.java | 2 +- .../java/org/opensearch/security/dlic/dlsfls/Fls983Test.java | 2 +- .../org/opensearch/security/dlic/dlsfls/FlsDlsTestAB.java | 2 +- .../security/dlic/dlsfls/FlsDlsTestForbiddenField.java | 2 +- .../org/opensearch/security/dlic/dlsfls/FlsDlsTestMulti.java | 2 +- .../opensearch/security/dlic/dlsfls/FlsExistsFieldsTest.java | 2 +- .../org/opensearch/security/dlic/dlsfls/FlsFieldsTest.java | 2 +- .../org/opensearch/security/dlic/dlsfls/FlsFieldsWcTest.java | 2 +- .../org/opensearch/security/dlic/dlsfls/FlsFlatTests.java | 2 +- .../org/opensearch/security/dlic/dlsfls/FlsIndexingTests.java | 2 +- .../org/opensearch/security/dlic/dlsfls/FlsKeywordTests.java | 2 +- .../java/org/opensearch/security/dlic/dlsfls/FlsPerfTest.java | 2 +- .../java/org/opensearch/security/dlic/dlsfls/FlsTest.java | 2 +- .../org/opensearch/security/dlic/dlsfls/IndexPatternTest.java | 2 +- .../java/org/opensearch/security/dlic/dlsfls/MFlsTest.java | 2 +- .../dlic/dlsfls/RenameFieldResponseProcessorTest.java | 2 +- .../security/dlic/rest/api/AbstractRestApiUnitTest.java | 2 +- .../org/opensearch/security/dlic/rest/api/AccountApiTest.java | 2 +- .../security/dlic/rest/api/ActionGroupsApiTest.java | 2 +- .../opensearch/security/dlic/rest/api/AllowlistApiTest.java | 2 +- .../opensearch/security/dlic/rest/api/AuditApiActionTest.java | 2 +- .../security/dlic/rest/api/DashboardsInfoActionTest.java | 2 +- .../opensearch/security/dlic/rest/api/FlushCacheApiTest.java | 2 +- .../security/dlic/rest/api/GetConfigurationApiTest.java | 2 +- .../opensearch/security/dlic/rest/api/IndexMissingTest.java | 2 +- .../security/dlic/rest/api/MultiTenancyConfigApiTest.java | 2 +- .../org/opensearch/security/dlic/rest/api/NodesDnApiTest.java | 2 +- .../security/dlic/rest/api/RoleBasedAccessTest.java | 2 +- .../org/opensearch/security/dlic/rest/api/RolesApiTest.java | 2 +- .../security/dlic/rest/api/RolesMappingApiTest.java | 2 +- .../security/dlic/rest/api/SecurityApiAccessTest.java | 2 +- .../security/dlic/rest/api/SecurityConfigApiActionTest.java | 2 +- .../security/dlic/rest/api/SecurityHealthActionTest.java | 2 +- .../security/dlic/rest/api/SecurityInfoActionTest.java | 2 +- .../opensearch/security/dlic/rest/api/SslCertsApiTest.java | 2 +- .../security/dlic/rest/api/TenantInfoActionTest.java | 2 +- .../org/opensearch/security/dlic/rest/api/UserApiTest.java | 2 +- .../opensearch/security/dlic/rest/api/WhitelistApiTest.java | 2 +- .../opensearch/security/filter/SecurityRestFilterTests.java | 2 +- .../security/multitenancy/test/MultitenancyTests.java | 2 +- .../multitenancy/test/TenancyMultitenancyEnabledTests.java | 2 +- .../multitenancy/test/TenancyPrivateTenantEnabledTests.java | 2 +- .../security/privileges/PrivilegesEvaluatorTest.java | 2 +- .../security/protected_indices/ProtectedIndicesTests.java | 2 +- .../security/system_indices/SystemIndexDisabledTests.java | 2 +- .../system_indices/SystemIndexPermissionDisabledTests.java | 2 +- .../system_indices/SystemIndexPermissionEnabledTests.java | 2 +- 96 files changed, 99 insertions(+), 94 deletions(-) diff --git a/checkstyle/checkstyle.xml b/checkstyle/checkstyle.xml index b679ce24ce..04a36c49c1 100644 --- a/checkstyle/checkstyle.xml +++ b/checkstyle/checkstyle.xml @@ -121,6 +121,7 @@ + diff --git a/gradle/formatting.gradle b/gradle/formatting.gradle index 2248c1d9a0..4fdea90277 100644 --- a/gradle/formatting.gradle +++ b/gradle/formatting.gradle @@ -17,6 +17,10 @@ allprojects { eclipse().configFile rootProject.file('formatter/formatterConfig.xml') trimTrailingWhitespace() endWithNewline(); + custom 'Replace illegal HttpStatus import w/ correct one', { + // e.g., replace org.apache.hc.core5.http.HttpStatus with org.apache.http.HttpStatus + it.replaceAll('org.apache.hc.core5.http.HttpStatus', 'org.apache.http.HttpStatus') + } // See DEVELOPER_GUIDE.md for details of when to enable this. if (System.getProperty('spotless.paddedcell') != null) { diff --git a/src/integrationTest/java/org/opensearch/security/http/AsyncTests.java b/src/integrationTest/java/org/opensearch/security/http/AsyncTests.java index 16ebd29885..8103d50e74 100644 --- a/src/integrationTest/java/org/opensearch/security/http/AsyncTests.java +++ b/src/integrationTest/java/org/opensearch/security/http/AsyncTests.java @@ -18,7 +18,7 @@ import java.util.concurrent.TimeUnit; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.ClassRule; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/integrationTest/java/org/opensearch/security/http/OnBehalfOfJwtAuthenticationTest.java b/src/integrationTest/java/org/opensearch/security/http/OnBehalfOfJwtAuthenticationTest.java index 2b56573dfe..78af7ffc05 100644 --- a/src/integrationTest/java/org/opensearch/security/http/OnBehalfOfJwtAuthenticationTest.java +++ b/src/integrationTest/java/org/opensearch/security/http/OnBehalfOfJwtAuthenticationTest.java @@ -22,8 +22,8 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.http.HttpStatus; import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; diff --git a/src/integrationTest/java/org/opensearch/security/http/ServiceAccountAuthenticationTest.java b/src/integrationTest/java/org/opensearch/security/http/ServiceAccountAuthenticationTest.java index 762feed686..932ca71e44 100644 --- a/src/integrationTest/java/org/opensearch/security/http/ServiceAccountAuthenticationTest.java +++ b/src/integrationTest/java/org/opensearch/security/http/ServiceAccountAuthenticationTest.java @@ -15,7 +15,7 @@ import java.util.Map; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.ClassRule; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/integrationTest/java/org/opensearch/security/rest/AuthZinRestLayerTests.java b/src/integrationTest/java/org/opensearch/security/rest/AuthZinRestLayerTests.java index ad13d69db7..c38ecbb611 100644 --- a/src/integrationTest/java/org/opensearch/security/rest/AuthZinRestLayerTests.java +++ b/src/integrationTest/java/org/opensearch/security/rest/AuthZinRestLayerTests.java @@ -12,7 +12,7 @@ package org.opensearch.security.rest; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; diff --git a/src/integrationTest/java/org/opensearch/security/rest/CompressionTests.java b/src/integrationTest/java/org/opensearch/security/rest/CompressionTests.java index 40c90764d9..bcbbc37400 100644 --- a/src/integrationTest/java/org/opensearch/security/rest/CompressionTests.java +++ b/src/integrationTest/java/org/opensearch/security/rest/CompressionTests.java @@ -20,9 +20,9 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import org.apache.hc.client5.http.classic.methods.HttpPost; import org.apache.hc.core5.http.ContentType; -import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.io.entity.ByteArrayEntity; import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.http.HttpStatus; import org.junit.ClassRule; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/integrationTest/java/org/opensearch/security/rest/WhoAmITests.java b/src/integrationTest/java/org/opensearch/security/rest/WhoAmITests.java index 0324cd449d..2652a5e2a6 100644 --- a/src/integrationTest/java/org/opensearch/security/rest/WhoAmITests.java +++ b/src/integrationTest/java/org/opensearch/security/rest/WhoAmITests.java @@ -22,7 +22,7 @@ import java.util.stream.Collectors; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; diff --git a/src/test/java/com/amazon/dlic/auth/ldap/LdapBackendIntegTest.java b/src/test/java/com/amazon/dlic/auth/ldap/LdapBackendIntegTest.java index 6fda346a93..24389a1086 100644 --- a/src/test/java/com/amazon/dlic/auth/ldap/LdapBackendIntegTest.java +++ b/src/test/java/com/amazon/dlic/auth/ldap/LdapBackendIntegTest.java @@ -11,8 +11,8 @@ package com.amazon.dlic.auth.ldap; -import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.http.HttpStatus; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; diff --git a/src/test/java/com/amazon/dlic/auth/ldap2/LdapBackendIntegTest2.java b/src/test/java/com/amazon/dlic/auth/ldap2/LdapBackendIntegTest2.java index 6f0958790a..e4f71ff264 100644 --- a/src/test/java/com/amazon/dlic/auth/ldap2/LdapBackendIntegTest2.java +++ b/src/test/java/com/amazon/dlic/auth/ldap2/LdapBackendIntegTest2.java @@ -11,8 +11,8 @@ package com.amazon.dlic.auth.ldap2; -import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.http.HttpStatus; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; diff --git a/src/test/java/org/opensearch/security/AdvancedSecurityMigrationTests.java b/src/test/java/org/opensearch/security/AdvancedSecurityMigrationTests.java index e8ac049385..5cf9485892 100644 --- a/src/test/java/org/opensearch/security/AdvancedSecurityMigrationTests.java +++ b/src/test/java/org/opensearch/security/AdvancedSecurityMigrationTests.java @@ -15,7 +15,7 @@ import java.util.Arrays; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.After; import org.junit.Assert; import org.junit.Before; diff --git a/src/test/java/org/opensearch/security/AggregationTests.java b/src/test/java/org/opensearch/security/AggregationTests.java index a61d5d169d..c6591125d5 100644 --- a/src/test/java/org/opensearch/security/AggregationTests.java +++ b/src/test/java/org/opensearch/security/AggregationTests.java @@ -26,7 +26,7 @@ package org.opensearch.security; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/DataStreamIntegrationTests.java b/src/test/java/org/opensearch/security/DataStreamIntegrationTests.java index 773244c7ea..2f4e665001 100644 --- a/src/test/java/org/opensearch/security/DataStreamIntegrationTests.java +++ b/src/test/java/org/opensearch/security/DataStreamIntegrationTests.java @@ -11,7 +11,7 @@ package org.opensearch.security; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/EncryptionInTransitMigrationTests.java b/src/test/java/org/opensearch/security/EncryptionInTransitMigrationTests.java index 462cd591e6..a028f2d43d 100644 --- a/src/test/java/org/opensearch/security/EncryptionInTransitMigrationTests.java +++ b/src/test/java/org/opensearch/security/EncryptionInTransitMigrationTests.java @@ -10,7 +10,7 @@ */ package org.opensearch.security; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/HealthTests.java b/src/test/java/org/opensearch/security/HealthTests.java index c36440f1a2..385757ea53 100644 --- a/src/test/java/org/opensearch/security/HealthTests.java +++ b/src/test/java/org/opensearch/security/HealthTests.java @@ -26,7 +26,7 @@ package org.opensearch.security; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/HttpIntegrationTests.java b/src/test/java/org/opensearch/security/HttpIntegrationTests.java index 60abaf8efe..3a437ea80a 100644 --- a/src/test/java/org/opensearch/security/HttpIntegrationTests.java +++ b/src/test/java/org/opensearch/security/HttpIntegrationTests.java @@ -31,9 +31,9 @@ import java.nio.file.Files; import com.fasterxml.jackson.databind.JsonNode; -import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.NoHttpResponseException; import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/IndexIntegrationTests.java b/src/test/java/org/opensearch/security/IndexIntegrationTests.java index 3d024f28f7..a5c137d61e 100644 --- a/src/test/java/org/opensearch/security/IndexIntegrationTests.java +++ b/src/test/java/org/opensearch/security/IndexIntegrationTests.java @@ -31,7 +31,7 @@ import java.util.Date; import java.util.TimeZone; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/IndexTemplateClusterPermissionsCheckTest.java b/src/test/java/org/opensearch/security/IndexTemplateClusterPermissionsCheckTest.java index 03d26e2062..e08367d2b2 100644 --- a/src/test/java/org/opensearch/security/IndexTemplateClusterPermissionsCheckTest.java +++ b/src/test/java/org/opensearch/security/IndexTemplateClusterPermissionsCheckTest.java @@ -11,7 +11,7 @@ package org.opensearch.security; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/InitializationIntegrationTests.java b/src/test/java/org/opensearch/security/InitializationIntegrationTests.java index 78b03a5fab..79ab0c020b 100644 --- a/src/test/java/org/opensearch/security/InitializationIntegrationTests.java +++ b/src/test/java/org/opensearch/security/InitializationIntegrationTests.java @@ -32,9 +32,9 @@ import com.fasterxml.jackson.databind.JsonNode; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.HttpVersion; import org.apache.hc.core5.http2.HttpVersionPolicy; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/IntegrationTests.java b/src/test/java/org/opensearch/security/IntegrationTests.java index 31a46be331..0777594834 100644 --- a/src/test/java/org/opensearch/security/IntegrationTests.java +++ b/src/test/java/org/opensearch/security/IntegrationTests.java @@ -29,8 +29,8 @@ import java.util.TreeSet; import com.fasterxml.jackson.databind.JsonNode; -import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Assume; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/PitIntegrationTests.java b/src/test/java/org/opensearch/security/PitIntegrationTests.java index 035cc2ce3e..c1c25fcf9c 100644 --- a/src/test/java/org/opensearch/security/PitIntegrationTests.java +++ b/src/test/java/org/opensearch/security/PitIntegrationTests.java @@ -13,7 +13,7 @@ import java.util.ArrayList; import java.util.List; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/ResolveAPITests.java b/src/test/java/org/opensearch/security/ResolveAPITests.java index 088702acd9..765d933432 100644 --- a/src/test/java/org/opensearch/security/ResolveAPITests.java +++ b/src/test/java/org/opensearch/security/ResolveAPITests.java @@ -15,7 +15,7 @@ package org.opensearch.security; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.junit.Assert; diff --git a/src/test/java/org/opensearch/security/SecurityAdminIEndpointsTests.java b/src/test/java/org/opensearch/security/SecurityAdminIEndpointsTests.java index b8da89e2dc..99cf3b82fe 100644 --- a/src/test/java/org/opensearch/security/SecurityAdminIEndpointsTests.java +++ b/src/test/java/org/opensearch/security/SecurityAdminIEndpointsTests.java @@ -11,7 +11,7 @@ package org.opensearch.security; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/SecurityAdminInvalidConfigsTests.java b/src/test/java/org/opensearch/security/SecurityAdminInvalidConfigsTests.java index 6cb89dc18f..1586878b9f 100644 --- a/src/test/java/org/opensearch/security/SecurityAdminInvalidConfigsTests.java +++ b/src/test/java/org/opensearch/security/SecurityAdminInvalidConfigsTests.java @@ -30,7 +30,7 @@ import java.util.ArrayList; import java.util.List; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/SecurityAdminTests.java b/src/test/java/org/opensearch/security/SecurityAdminTests.java index 760f2a33d7..d2b7dab37d 100644 --- a/src/test/java/org/opensearch/security/SecurityAdminTests.java +++ b/src/test/java/org/opensearch/security/SecurityAdminTests.java @@ -25,7 +25,7 @@ import java.util.List; import java.util.Objects; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/SecurityRolesTests.java b/src/test/java/org/opensearch/security/SecurityRolesTests.java index 24a6bafbb8..0b4dd0b95b 100644 --- a/src/test/java/org/opensearch/security/SecurityRolesTests.java +++ b/src/test/java/org/opensearch/security/SecurityRolesTests.java @@ -26,8 +26,8 @@ package org.opensearch.security; -import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/SlowIntegrationTests.java b/src/test/java/org/opensearch/security/SlowIntegrationTests.java index 28ec9f8d88..76a162ab9f 100644 --- a/src/test/java/org/opensearch/security/SlowIntegrationTests.java +++ b/src/test/java/org/opensearch/security/SlowIntegrationTests.java @@ -29,7 +29,7 @@ import java.io.IOException; import com.google.common.collect.Lists; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/SnapshotRestoreTests.java b/src/test/java/org/opensearch/security/SnapshotRestoreTests.java index 1c884a8e5d..1e9c26d898 100644 --- a/src/test/java/org/opensearch/security/SnapshotRestoreTests.java +++ b/src/test/java/org/opensearch/security/SnapshotRestoreTests.java @@ -29,7 +29,7 @@ import java.util.Arrays; import java.util.List; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/SystemIntegratorsTests.java b/src/test/java/org/opensearch/security/SystemIntegratorsTests.java index 27e44b1ce5..b927ceaba2 100644 --- a/src/test/java/org/opensearch/security/SystemIntegratorsTests.java +++ b/src/test/java/org/opensearch/security/SystemIntegratorsTests.java @@ -27,8 +27,8 @@ package org.opensearch.security; import com.google.common.collect.Lists; -import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/TaskTests.java b/src/test/java/org/opensearch/security/TaskTests.java index 39bb21f164..e58fa5c6a9 100644 --- a/src/test/java/org/opensearch/security/TaskTests.java +++ b/src/test/java/org/opensearch/security/TaskTests.java @@ -17,8 +17,8 @@ package org.opensearch.security; -import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/TracingTests.java b/src/test/java/org/opensearch/security/TracingTests.java index 55dccdee35..7ae663a41f 100644 --- a/src/test/java/org/opensearch/security/TracingTests.java +++ b/src/test/java/org/opensearch/security/TracingTests.java @@ -26,7 +26,7 @@ package org.opensearch.security; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/auditlog/AuditTestUtils.java b/src/test/java/org/opensearch/security/auditlog/AuditTestUtils.java index 98f5fab88e..ad3f6afbce 100644 --- a/src/test/java/org/opensearch/security/auditlog/AuditTestUtils.java +++ b/src/test/java/org/opensearch/security/auditlog/AuditTestUtils.java @@ -15,7 +15,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; diff --git a/src/test/java/org/opensearch/security/auditlog/compliance/ComplianceAuditlogTest.java b/src/test/java/org/opensearch/security/auditlog/compliance/ComplianceAuditlogTest.java index f9d4474c3e..5ba95dc756 100644 --- a/src/test/java/org/opensearch/security/auditlog/compliance/ComplianceAuditlogTest.java +++ b/src/test/java/org/opensearch/security/auditlog/compliance/ComplianceAuditlogTest.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/auditlog/compliance/RestApiComplianceAuditlogTest.java b/src/test/java/org/opensearch/security/auditlog/compliance/RestApiComplianceAuditlogTest.java index 552e661ff3..e93229ee8b 100644 --- a/src/test/java/org/opensearch/security/auditlog/compliance/RestApiComplianceAuditlogTest.java +++ b/src/test/java/org/opensearch/security/auditlog/compliance/RestApiComplianceAuditlogTest.java @@ -11,7 +11,7 @@ package org.opensearch.security.auditlog.compliance; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/auditlog/impl/TracingTests.java b/src/test/java/org/opensearch/security/auditlog/impl/TracingTests.java index 15728537e2..796c73b811 100644 --- a/src/test/java/org/opensearch/security/auditlog/impl/TracingTests.java +++ b/src/test/java/org/opensearch/security/auditlog/impl/TracingTests.java @@ -11,7 +11,7 @@ package org.opensearch.security.auditlog.impl; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/auditlog/integration/BasicAuditlogTest.java b/src/test/java/org/opensearch/security/auditlog/integration/BasicAuditlogTest.java index 89b8426120..c4784d14b8 100644 --- a/src/test/java/org/opensearch/security/auditlog/integration/BasicAuditlogTest.java +++ b/src/test/java/org/opensearch/security/auditlog/integration/BasicAuditlogTest.java @@ -16,8 +16,8 @@ import com.google.common.collect.ImmutableMap; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/auditlog/integration/SSLAuditlogTest.java b/src/test/java/org/opensearch/security/auditlog/integration/SSLAuditlogTest.java index 82c56d4b23..0b92c952f6 100644 --- a/src/test/java/org/opensearch/security/auditlog/integration/SSLAuditlogTest.java +++ b/src/test/java/org/opensearch/security/auditlog/integration/SSLAuditlogTest.java @@ -11,7 +11,7 @@ package org.opensearch.security.auditlog.integration; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.After; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/cache/CachingTest.java b/src/test/java/org/opensearch/security/cache/CachingTest.java index 39f0e1315d..cb71be78e1 100644 --- a/src/test/java/org/opensearch/security/cache/CachingTest.java +++ b/src/test/java/org/opensearch/security/cache/CachingTest.java @@ -11,8 +11,8 @@ package org.opensearch.security.cache; -import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/ccstest/CrossClusterSearchTests.java b/src/test/java/org/opensearch/security/ccstest/CrossClusterSearchTests.java index e2dd28b563..0bf9e0e9df 100644 --- a/src/test/java/org/opensearch/security/ccstest/CrossClusterSearchTests.java +++ b/src/test/java/org/opensearch/security/ccstest/CrossClusterSearchTests.java @@ -27,7 +27,7 @@ package org.opensearch.security.ccstest; import com.google.common.collect.Lists; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.After; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/ccstest/RemoteReindexTests.java b/src/test/java/org/opensearch/security/ccstest/RemoteReindexTests.java index ea329b7b2c..15fe91d822 100644 --- a/src/test/java/org/opensearch/security/ccstest/RemoteReindexTests.java +++ b/src/test/java/org/opensearch/security/ccstest/RemoteReindexTests.java @@ -26,7 +26,7 @@ package org.opensearch.security.ccstest; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.After; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/CustomFieldMaskedComplexMappingTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/CustomFieldMaskedComplexMappingTest.java index 3a03e8add4..8490b42f12 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/CustomFieldMaskedComplexMappingTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/CustomFieldMaskedComplexMappingTest.java @@ -13,7 +13,7 @@ import java.nio.charset.StandardCharsets; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/CustomFieldMaskedTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/CustomFieldMaskedTest.java index 672eb2abb0..226574c588 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/CustomFieldMaskedTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/CustomFieldMaskedTest.java @@ -11,7 +11,7 @@ package org.opensearch.security.dlic.dlsfls; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/DateMathTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/DateMathTest.java index 43b78e9803..db15602867 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/DateMathTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/DateMathTest.java @@ -15,7 +15,7 @@ import java.util.Date; import java.util.TimeZone; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/DlsDateMathTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/DlsDateMathTest.java index bc349ace37..87d2ea6b52 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/DlsDateMathTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/DlsDateMathTest.java @@ -15,7 +15,7 @@ import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/DlsFlsCrossClusterSearchTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/DlsFlsCrossClusterSearchTest.java index 6ac4690a70..ad2b2433cf 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/DlsFlsCrossClusterSearchTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/DlsFlsCrossClusterSearchTest.java @@ -11,7 +11,7 @@ package org.opensearch.security.dlic.dlsfls; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.After; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/DlsNestedTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/DlsNestedTest.java index 67abf5f61b..36e7ec0905 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/DlsNestedTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/DlsNestedTest.java @@ -11,7 +11,7 @@ package org.opensearch.security.dlic.dlsfls; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/DlsPropsReplaceTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/DlsPropsReplaceTest.java index 5fe6419a02..88ebdbe36c 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/DlsPropsReplaceTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/DlsPropsReplaceTest.java @@ -11,7 +11,7 @@ package org.opensearch.security.dlic.dlsfls; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/DlsScrollTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/DlsScrollTest.java index cc7b9e305d..0662c65109 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/DlsScrollTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/DlsScrollTest.java @@ -11,7 +11,7 @@ package org.opensearch.security.dlic.dlsfls; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/DlsTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/DlsTest.java index 587f759315..e4dffcc31f 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/DlsTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/DlsTest.java @@ -11,7 +11,7 @@ package org.opensearch.security.dlic.dlsfls; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/FieldMaskedTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/FieldMaskedTest.java index 2628bebbc0..e18eae5780 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/FieldMaskedTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/FieldMaskedTest.java @@ -11,7 +11,7 @@ package org.opensearch.security.dlic.dlsfls; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/Fls983Test.java b/src/test/java/org/opensearch/security/dlic/dlsfls/Fls983Test.java index c486599ea4..c17b5c9f0c 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/Fls983Test.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/Fls983Test.java @@ -11,7 +11,7 @@ package org.opensearch.security.dlic.dlsfls; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsDlsTestAB.java b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsDlsTestAB.java index 5e7584a1bc..33b8296814 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsDlsTestAB.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsDlsTestAB.java @@ -11,7 +11,7 @@ package org.opensearch.security.dlic.dlsfls; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsDlsTestForbiddenField.java b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsDlsTestForbiddenField.java index a3776e567c..fd164802d3 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsDlsTestForbiddenField.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsDlsTestForbiddenField.java @@ -11,7 +11,7 @@ package org.opensearch.security.dlic.dlsfls; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsDlsTestMulti.java b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsDlsTestMulti.java index e9d32f18ea..5cc9f7423a 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsDlsTestMulti.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsDlsTestMulti.java @@ -11,7 +11,7 @@ package org.opensearch.security.dlic.dlsfls; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsExistsFieldsTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsExistsFieldsTest.java index bc3d306627..b58b80368a 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsExistsFieldsTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsExistsFieldsTest.java @@ -11,7 +11,7 @@ package org.opensearch.security.dlic.dlsfls; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsFieldsTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsFieldsTest.java index a910cf5663..5681479085 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsFieldsTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsFieldsTest.java @@ -13,7 +13,7 @@ import java.io.IOException; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsFieldsWcTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsFieldsWcTest.java index f6cfd036fd..2c3235cf27 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsFieldsWcTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsFieldsWcTest.java @@ -13,7 +13,7 @@ import java.io.IOException; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsFlatTests.java b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsFlatTests.java index 7899d3c2e5..3441108395 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsFlatTests.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsFlatTests.java @@ -15,7 +15,7 @@ import java.util.function.Consumer; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Test; import org.opensearch.action.admin.indices.create.CreateIndexRequest; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsIndexingTests.java b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsIndexingTests.java index 2d7ed0efcf..2552fcdbc3 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsIndexingTests.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsIndexingTests.java @@ -12,7 +12,7 @@ package org.opensearch.security.dlic.dlsfls; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Test; import org.opensearch.action.index.IndexRequest; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsKeywordTests.java b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsKeywordTests.java index 1c51ec99b7..8117b7e0ba 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsKeywordTests.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsKeywordTests.java @@ -14,7 +14,7 @@ import java.util.Arrays; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Test; import org.opensearch.action.index.IndexRequest; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsPerfTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsPerfTest.java index 32092cc8ed..81553662fd 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsPerfTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsPerfTest.java @@ -15,7 +15,7 @@ import java.util.HashMap; import java.util.Map; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsTest.java index 66c962051f..a2787af61c 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsTest.java @@ -11,7 +11,7 @@ package org.opensearch.security.dlic.dlsfls; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/IndexPatternTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/IndexPatternTest.java index 75eb428ee8..29b1a44bcb 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/IndexPatternTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/IndexPatternTest.java @@ -11,7 +11,7 @@ package org.opensearch.security.dlic.dlsfls; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/MFlsTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/MFlsTest.java index 6267aeb9c0..f5408113b6 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/MFlsTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/MFlsTest.java @@ -11,7 +11,7 @@ package org.opensearch.security.dlic.dlsfls; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/RenameFieldResponseProcessorTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/RenameFieldResponseProcessorTest.java index c22d167b6d..55aff8f470 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/RenameFieldResponseProcessorTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/RenameFieldResponseProcessorTest.java @@ -12,7 +12,7 @@ package org.opensearch.security.dlic.dlsfls; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Test; import org.opensearch.action.index.IndexRequest; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/AbstractRestApiUnitTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/AbstractRestApiUnitTest.java index f9c4428bc1..c3c2106b05 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/AbstractRestApiUnitTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/AbstractRestApiUnitTest.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.opensearch.common.settings.Settings; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/AccountApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/AccountApiTest.java index f84e28e755..21f68d11df 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/AccountApiTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/AccountApiTest.java @@ -12,7 +12,7 @@ package org.opensearch.security.dlic.rest.api; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/ActionGroupsApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/ActionGroupsApiTest.java index 46b730abac..fb166779ac 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/ActionGroupsApiTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/ActionGroupsApiTest.java @@ -17,7 +17,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/AllowlistApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/AllowlistApiTest.java index ccce614c07..567421e426 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/AllowlistApiTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/AllowlistApiTest.java @@ -17,7 +17,7 @@ import com.google.common.collect.ImmutableMap; import com.fasterxml.jackson.databind.JsonNode; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/AuditApiActionTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/AuditApiActionTest.java index ce72fe2cef..b512ae2228 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/AuditApiActionTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/AuditApiActionTest.java @@ -25,7 +25,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.After; import org.junit.Rule; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/DashboardsInfoActionTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/DashboardsInfoActionTest.java index 46128f5a71..647e7a2a33 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/DashboardsInfoActionTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/DashboardsInfoActionTest.java @@ -11,7 +11,7 @@ package org.opensearch.security.dlic.rest.api; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/FlushCacheApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/FlushCacheApiTest.java index 120596f046..ee75ccc984 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/FlushCacheApiTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/FlushCacheApiTest.java @@ -12,7 +12,7 @@ package org.opensearch.security.dlic.rest.api; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/GetConfigurationApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/GetConfigurationApiTest.java index 09c4a762b5..8defebc6d1 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/GetConfigurationApiTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/GetConfigurationApiTest.java @@ -12,7 +12,7 @@ package org.opensearch.security.dlic.rest.api; import com.fasterxml.jackson.databind.JsonNode; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/IndexMissingTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/IndexMissingTest.java index aefb0f2550..4632a3920f 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/IndexMissingTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/IndexMissingTest.java @@ -12,7 +12,7 @@ package org.opensearch.security.dlic.rest.api; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/MultiTenancyConfigApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/MultiTenancyConfigApiTest.java index 7132dcc491..8438338869 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/MultiTenancyConfigApiTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/MultiTenancyConfigApiTest.java @@ -12,7 +12,7 @@ package org.opensearch.security.dlic.rest.api; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Test; import org.opensearch.security.support.ConfigConstants; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/NodesDnApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/NodesDnApiTest.java index 44c43863f9..8379a80989 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/NodesDnApiTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/NodesDnApiTest.java @@ -20,7 +20,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/RoleBasedAccessTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/RoleBasedAccessTest.java index cbd751e00c..1f4d0ff247 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/RoleBasedAccessTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/RoleBasedAccessTest.java @@ -11,7 +11,7 @@ package org.opensearch.security.dlic.rest.api; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/RolesApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/RolesApiTest.java index 8b475ec776..bb11bb2226 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/RolesApiTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/RolesApiTest.java @@ -18,7 +18,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/RolesMappingApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/RolesMappingApiTest.java index 8d9b76274c..077c852466 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/RolesMappingApiTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/RolesMappingApiTest.java @@ -17,7 +17,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/SecurityApiAccessTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/SecurityApiAccessTest.java index 81fad7d4ff..1580d07524 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/SecurityApiAccessTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/SecurityApiAccessTest.java @@ -11,7 +11,7 @@ package org.opensearch.security.dlic.rest.api; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/SecurityConfigApiActionTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/SecurityConfigApiActionTest.java index 7b98494e1b..c4066d11a2 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/SecurityConfigApiActionTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/SecurityConfigApiActionTest.java @@ -12,7 +12,7 @@ package org.opensearch.security.dlic.rest.api; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/SecurityHealthActionTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/SecurityHealthActionTest.java index d7a6edfea9..e239050612 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/SecurityHealthActionTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/SecurityHealthActionTest.java @@ -11,7 +11,7 @@ package org.opensearch.security.dlic.rest.api; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/SecurityInfoActionTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/SecurityInfoActionTest.java index db27be85ee..0799525eb8 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/SecurityInfoActionTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/SecurityInfoActionTest.java @@ -11,7 +11,7 @@ package org.opensearch.security.dlic.rest.api; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/SslCertsApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/SslCertsApiTest.java index 75e1e59b0a..8617555925 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/SslCertsApiTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/SslCertsApiTest.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableMap; import com.fasterxml.jackson.core.JsonProcessingException; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/TenantInfoActionTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/TenantInfoActionTest.java index 2c6a45faf7..2e47aae556 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/TenantInfoActionTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/TenantInfoActionTest.java @@ -12,7 +12,7 @@ package org.opensearch.security.dlic.rest.api; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/UserApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/UserApiTest.java index ca467801f0..2a9292b3a9 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/UserApiTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/UserApiTest.java @@ -21,8 +21,8 @@ import com.fasterxml.jackson.databind.JsonNode; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.http.HttpStatus; import org.hamcrest.Matchers; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/WhitelistApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/WhitelistApiTest.java index b9b3cf50b8..0e3d330b52 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/WhitelistApiTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/WhitelistApiTest.java @@ -17,7 +17,7 @@ import com.google.common.collect.ImmutableMap; import com.fasterxml.jackson.databind.JsonNode; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/filter/SecurityRestFilterTests.java b/src/test/java/org/opensearch/security/filter/SecurityRestFilterTests.java index 5adcadb1f2..b46c5a6e32 100644 --- a/src/test/java/org/opensearch/security/filter/SecurityRestFilterTests.java +++ b/src/test/java/org/opensearch/security/filter/SecurityRestFilterTests.java @@ -12,7 +12,7 @@ package org.opensearch.security.filter; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Test; import org.opensearch.security.dlic.rest.api.AbstractRestApiUnitTest; diff --git a/src/test/java/org/opensearch/security/multitenancy/test/MultitenancyTests.java b/src/test/java/org/opensearch/security/multitenancy/test/MultitenancyTests.java index b66902f4b9..0a785d7b80 100644 --- a/src/test/java/org/opensearch/security/multitenancy/test/MultitenancyTests.java +++ b/src/test/java/org/opensearch/security/multitenancy/test/MultitenancyTests.java @@ -14,8 +14,8 @@ import java.util.Map; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/multitenancy/test/TenancyMultitenancyEnabledTests.java b/src/test/java/org/opensearch/security/multitenancy/test/TenancyMultitenancyEnabledTests.java index b25a50d934..32b9bb2156 100644 --- a/src/test/java/org/opensearch/security/multitenancy/test/TenancyMultitenancyEnabledTests.java +++ b/src/test/java/org/opensearch/security/multitenancy/test/TenancyMultitenancyEnabledTests.java @@ -12,8 +12,8 @@ package org.opensearch.security.multitenancy.test; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.http.HttpStatus; import org.junit.Test; import org.opensearch.common.settings.Settings; diff --git a/src/test/java/org/opensearch/security/multitenancy/test/TenancyPrivateTenantEnabledTests.java b/src/test/java/org/opensearch/security/multitenancy/test/TenancyPrivateTenantEnabledTests.java index 1af102802f..4f2d2c3505 100644 --- a/src/test/java/org/opensearch/security/multitenancy/test/TenancyPrivateTenantEnabledTests.java +++ b/src/test/java/org/opensearch/security/multitenancy/test/TenancyPrivateTenantEnabledTests.java @@ -12,8 +12,8 @@ package org.opensearch.security.multitenancy.test; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.http.HttpStatus; import org.junit.Test; import org.opensearch.common.settings.Settings; diff --git a/src/test/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java b/src/test/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java index 4f25c71d66..d5a26024a9 100644 --- a/src/test/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java +++ b/src/test/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java @@ -12,7 +12,7 @@ package org.opensearch.security.privileges; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/protected_indices/ProtectedIndicesTests.java b/src/test/java/org/opensearch/security/protected_indices/ProtectedIndicesTests.java index bc4cc18f61..c1198269b1 100644 --- a/src/test/java/org/opensearch/security/protected_indices/ProtectedIndicesTests.java +++ b/src/test/java/org/opensearch/security/protected_indices/ProtectedIndicesTests.java @@ -31,7 +31,7 @@ import java.util.List; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Test; import org.opensearch.action.admin.cluster.repositories.put.PutRepositoryRequest; diff --git a/src/test/java/org/opensearch/security/system_indices/SystemIndexDisabledTests.java b/src/test/java/org/opensearch/security/system_indices/SystemIndexDisabledTests.java index 9415634596..e14574873e 100644 --- a/src/test/java/org/opensearch/security/system_indices/SystemIndexDisabledTests.java +++ b/src/test/java/org/opensearch/security/system_indices/SystemIndexDisabledTests.java @@ -14,7 +14,7 @@ import java.io.IOException; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/system_indices/SystemIndexPermissionDisabledTests.java b/src/test/java/org/opensearch/security/system_indices/SystemIndexPermissionDisabledTests.java index 25514c4118..37b4f1bc0f 100644 --- a/src/test/java/org/opensearch/security/system_indices/SystemIndexPermissionDisabledTests.java +++ b/src/test/java/org/opensearch/security/system_indices/SystemIndexPermissionDisabledTests.java @@ -14,7 +14,7 @@ import java.io.IOException; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/org/opensearch/security/system_indices/SystemIndexPermissionEnabledTests.java b/src/test/java/org/opensearch/security/system_indices/SystemIndexPermissionEnabledTests.java index 766db1eca8..b44b014d17 100644 --- a/src/test/java/org/opensearch/security/system_indices/SystemIndexPermissionEnabledTests.java +++ b/src/test/java/org/opensearch/security/system_indices/SystemIndexPermissionEnabledTests.java @@ -12,7 +12,7 @@ package org.opensearch.security.system_indices; import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.http.HttpStatus; import org.junit.Before; import org.junit.Test; From c30c084b911d00f94683bcccc0d1acd32461a001 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 21 Nov 2023 14:03:22 -0600 Subject: [PATCH 021/204] Validate content type of responses (#3719) Makes sure that for response with a body they are returning the expected content type Signed-off-by: Peter Nied --- .../framework/cluster/TestRestClient.java | 21 +++++++++++++++ .../security/test/helper/rest/RestHelper.java | 27 +++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java index e38ef949cb..34eb8983cd 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -65,6 +65,7 @@ import org.apache.hc.core5.http.io.entity.StringEntity; import org.apache.hc.core5.http.message.BasicHeader; import org.apache.hc.core5.net.URIBuilder; +import org.apache.http.HttpHeaders; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -77,6 +78,7 @@ import static java.util.Objects.requireNonNull; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; /** @@ -284,7 +286,26 @@ public HttpResponse(CloseableHttpResponse inner) throws IllegalStateException, I this.header = inner.getHeaders(); this.statusCode = inner.getCode(); this.statusReason = inner.getReasonPhrase(); + inner.close(); + + if (this.body.length() != 0) { + verifyContentType(); + } + } + + private void verifyContentType() { + final String contentType = this.getHeader(HttpHeaders.CONTENT_TYPE).getValue(); + if (contentType.contains("application/json")) { + assertThat("Response body format was not json, body: " + body, body.charAt(0), equalTo('{')); + } else { + assertThat( + "Response body format was json, whereas content-type was " + contentType + ", body: " + body, + body.charAt(0), + not(equalTo('{')) + ); + } + } public String getContentType() { diff --git a/src/test/java/org/opensearch/security/test/helper/rest/RestHelper.java b/src/test/java/org/opensearch/security/test/helper/rest/RestHelper.java index 43e7afc559..c137591825 100644 --- a/src/test/java/org/opensearch/security/test/helper/rest/RestHelper.java +++ b/src/test/java/org/opensearch/security/test/helper/rest/RestHelper.java @@ -90,6 +90,10 @@ import org.opensearch.security.test.helper.cluster.ClusterInfo; import org.opensearch.security.test.helper.file.FileHelper; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; + public class RestHelper { protected final Logger log = LogManager.getLogger(RestHelper.class); @@ -402,6 +406,29 @@ public HttpResponse(SimpleHttpResponse inner) throws IllegalStateException, IOEx this.statusCode = inner.getCode(); this.statusReason = inner.getReasonPhrase(); this.protocolVersion = inner.getVersion(); + + if (this.body.length() != 0) { + verifyBodyContentType(); + } + } + + private void verifyBodyContentType() { + final String contentType = this.getHeaders() + .stream() + .filter(h -> HttpHeaders.CONTENT_TYPE.equalsIgnoreCase(h.getName())) + .map(Header::getValue) + .findFirst() + .orElseThrow(() -> new RuntimeException("No content type found. Headers:\n" + getHeaders() + "\n\nBody:\n" + body)); + + if (contentType.contains("application/json")) { + assertThat("Response body format was not json, body: " + body, body.charAt(0), equalTo('{')); + } else { + assertThat( + "Response body format was json, whereas content-type was " + contentType + ", body: " + body, + body.charAt(0), + not(equalTo('{')) + ); + } } public String getContentType() { From e9d3f263a8cd8102f23d869faaabc11533a69a1e Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 22 Nov 2023 11:07:02 -0500 Subject: [PATCH 022/204] Switch UserInjectorPlugin to use reliable per-instance settings (#3760) ### Description Fix flaky tests in TransportUserInjectorIntegTest by removing reliance on a static variable that makes the test unpredictable. This instead adds a setting in the UserInjectorPlugin and relies on each node that has the plugin installed to supply this setting when instantiating the node. * Category (Enhancement, New feature, Bug fix, Test fix, Refactoring, Maintenance, Documentation) Test Fix ### Issues Resolved - Fixes https://github.com/opensearch-project/security/issues/3729 ### Check List - [ ] New functionality includes testing - [ ] New functionality has been documented - [ ] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). Signed-off-by: Craig Perkins --- .../TransportUserInjectorIntegTest.java | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/src/test/java/org/opensearch/security/TransportUserInjectorIntegTest.java b/src/test/java/org/opensearch/security/TransportUserInjectorIntegTest.java index b13e5fbb20..decd886e15 100644 --- a/src/test/java/org/opensearch/security/TransportUserInjectorIntegTest.java +++ b/src/test/java/org/opensearch/security/TransportUserInjectorIntegTest.java @@ -14,6 +14,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.function.Supplier; import com.google.common.collect.Lists; @@ -26,7 +27,9 @@ import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; @@ -47,9 +50,10 @@ public class TransportUserInjectorIntegTest extends SingleClusterTest { + public static final String TEST_INJECTED_USER = "test_injected_user"; + public static class UserInjectorPlugin extends Plugin implements ActionPlugin { Settings settings; - public static String injectedUser = null; public UserInjectorPlugin(final Settings settings, final Path configPath) { this.settings = settings; @@ -69,17 +73,24 @@ public Collection createComponents( IndexNameExpressionResolver indexNameExpressionResolver, Supplier repositoriesServiceSupplier ) { - if (injectedUser != null) threadPool.getThreadContext() - .putTransient(ConfigConstants.OPENDISTRO_SECURITY_INJECTED_USER, injectedUser); + if (!Strings.isNullOrEmpty(settings.get(TEST_INJECTED_USER))) threadPool.getThreadContext() + .putTransient(ConfigConstants.OPENDISTRO_SECURITY_INJECTED_USER, settings.get(TEST_INJECTED_USER)); return new ArrayList<>(); } + + @Override + public List> getSettings() { + List> settings = new ArrayList>(); + settings.add(Setting.simpleString(TEST_INJECTED_USER, Setting.Property.NodeScope, Setting.Property.Filtered)); + return settings; + } } @Test public void testSecurityUserInjection() throws Exception { final Settings clusterNodeSettings = Settings.builder().put(ConfigConstants.SECURITY_UNSUPPORTED_INJECT_USER_ENABLED, true).build(); setup(clusterNodeSettings, new DynamicSecurityConfig().setSecurityRolesMapping("roles_transport_inject_user.yml"), Settings.EMPTY); - final Settings tcSettings = AbstractSecurityUnitTest.nodeRolesSettings(Settings.builder(), false, false) + final Settings.Builder tcSettings = AbstractSecurityUnitTest.nodeRolesSettings(Settings.builder(), false, false) .put(minimumSecuritySettings(Settings.EMPTY).get(0)) .put("cluster.name", clusterInfo.clustername) .put("path.data", "./target/data/" + clusterInfo.clustername + "/cert/data") @@ -89,14 +100,13 @@ public void testSecurityUserInjection() throws Exception { .put("discovery.initial_state_timeout", "8s") .put("plugins.security.allow_default_init_securityindex", "true") .put(ConfigConstants.SECURITY_UNSUPPORTED_INJECT_USER_ENABLED, true) - .putList("discovery.zen.ping.unicast.hosts", clusterInfo.nodeHost + ":" + clusterInfo.nodePort) - .build(); + .putList("discovery.zen.ping.unicast.hosts", clusterInfo.nodeHost + ":" + clusterInfo.nodePort); // 1. without user injection try ( Node node = new PluginAwareNode( false, - tcSettings, + tcSettings.build(), Lists.newArrayList(Netty4ModulePlugin.class, OpenSearchSecurityPlugin.class, UserInjectorPlugin.class) ).start() ) { @@ -106,12 +116,11 @@ public void testSecurityUserInjection() throws Exception { } // 2. with invalid backend roles - UserInjectorPlugin.injectedUser = "ttt|kkk"; Exception exception = null; try ( Node node = new PluginAwareNode( false, - tcSettings, + tcSettings.put(TEST_INJECTED_USER, "ttt|kkk").build(), Lists.newArrayList(Netty4ModulePlugin.class, OpenSearchSecurityPlugin.class, UserInjectorPlugin.class) ).start() ) { @@ -126,11 +135,10 @@ public void testSecurityUserInjection() throws Exception { } // 3. with valid backend roles for injected user - UserInjectorPlugin.injectedUser = "injectedadmin|injecttest"; try ( Node node = new PluginAwareNode( false, - tcSettings, + tcSettings.put(TEST_INJECTED_USER, "injectedadmin|injecttest").build(), Lists.newArrayList(Netty4ModulePlugin.class, OpenSearchSecurityPlugin.class, UserInjectorPlugin.class) ).start() ) { @@ -146,7 +154,7 @@ public void testSecurityUserInjectionWithConfigDisabled() throws Exception { .put(ConfigConstants.SECURITY_UNSUPPORTED_INJECT_USER_ENABLED, false) .build(); setup(clusterNodeSettings, new DynamicSecurityConfig().setSecurityRolesMapping("roles_transport_inject_user.yml"), Settings.EMPTY); - final Settings tcSettings = AbstractSecurityUnitTest.nodeRolesSettings(Settings.builder(), false, false) + final Settings.Builder tcSettings = AbstractSecurityUnitTest.nodeRolesSettings(Settings.builder(), false, false) .put(minimumSecuritySettings(Settings.EMPTY).get(0)) .put("cluster.name", clusterInfo.clustername) .put("path.data", "./target/data/" + clusterInfo.clustername + "/cert/data") @@ -156,14 +164,13 @@ public void testSecurityUserInjectionWithConfigDisabled() throws Exception { .put("discovery.initial_state_timeout", "8s") .put("plugins.security.allow_default_init_securityindex", "true") .put(ConfigConstants.SECURITY_UNSUPPORTED_INJECT_USER_ENABLED, false) - .putList("discovery.zen.ping.unicast.hosts", clusterInfo.nodeHost + ":" + clusterInfo.nodePort) - .build(); + .putList("discovery.zen.ping.unicast.hosts", clusterInfo.nodeHost + ":" + clusterInfo.nodePort); // 1. without user injection try ( Node node = new PluginAwareNode( false, - tcSettings, + tcSettings.build(), Lists.newArrayList(Netty4ModulePlugin.class, OpenSearchSecurityPlugin.class, UserInjectorPlugin.class) ).start() ) { @@ -173,11 +180,10 @@ public void testSecurityUserInjectionWithConfigDisabled() throws Exception { } // with invalid backend roles - UserInjectorPlugin.injectedUser = "ttt|kkk"; try ( Node node = new PluginAwareNode( false, - tcSettings, + tcSettings.put(TEST_INJECTED_USER, "ttt|kkk").build(), Lists.newArrayList(Netty4ModulePlugin.class, OpenSearchSecurityPlugin.class, UserInjectorPlugin.class) ).start() ) { From 3c01fdea5f91286e908cd9c150a1a0edfa09a0ce Mon Sep 17 00:00:00 2001 From: MaciejMierzwa Date: Wed, 22 Nov 2023 20:26:27 +0100 Subject: [PATCH 023/204] Shrink operation privileges evaluation (#3716) ### Description Bug fix. Shrink, or resize operations weren't properly evaluated. More in the task: https://github.com/opensearch-project/security/issues/2141 ### Issues Resolved https://github.com/opensearch-project/security/issues/2141 Is this a backport? If so, please add backport PR # and/or commits # ### Testing [Please provide details of testing done: unit testing, integration testing and manual testing] ### Check List - [x] New functionality includes testing - [x] New functionality has been documented - [x] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: Maciej Mierzwa --- .../security/SearchOperationTest.java | 68 ++++++++++++------- .../resolver/IndexResolverReplacer.java | 5 ++ 2 files changed, 47 insertions(+), 26 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/SearchOperationTest.java b/src/integrationTest/java/org/opensearch/security/SearchOperationTest.java index e39eeeca61..652b0dcf10 100644 --- a/src/integrationTest/java/org/opensearch/security/SearchOperationTest.java +++ b/src/integrationTest/java/org/opensearch/security/SearchOperationTest.java @@ -28,6 +28,8 @@ import org.junit.Test; import org.junit.runner.RunWith; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest; import org.opensearch.action.admin.cluster.repositories.put.PutRepositoryRequest; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; @@ -83,6 +85,7 @@ import org.opensearch.client.indices.PutMappingRequest; import org.opensearch.client.indices.ResizeRequest; import org.opensearch.client.indices.ResizeResponse; +import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexTemplateMetadata; import org.opensearch.common.settings.Settings; @@ -119,6 +122,7 @@ import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.opensearch.client.RequestOptions.DEFAULT; import static org.opensearch.core.rest.RestStatus.ACCEPTED; +import static org.opensearch.core.rest.RestStatus.BAD_REQUEST; import static org.opensearch.core.rest.RestStatus.FORBIDDEN; import static org.opensearch.core.rest.RestStatus.INTERNAL_SERVER_ERROR; import static org.opensearch.rest.RestRequest.Method.DELETE; @@ -335,22 +339,24 @@ public class SearchOperationTest { * indices with names prefixed by the {@link #INDICES_ON_WHICH_USER_CAN_PERFORM_INDEX_OPERATIONS_PREFIX} */ static final User USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES = new User("index-operation-tester").roles( - new Role("index-manager").indexPermissions( - "indices:admin/create", - "indices:admin/get", - "indices:admin/delete", - "indices:admin/close", - "indices:admin/close*", - "indices:admin/open", - "indices:admin/resize", - "indices:monitor/stats", - "indices:monitor/settings/get", - "indices:admin/settings/update", - "indices:admin/mapping/put", - "indices:admin/mappings/get", - "indices:admin/cache/clear", - "indices:admin/aliases" - ).on(INDICES_ON_WHICH_USER_CAN_PERFORM_INDEX_OPERATIONS_PREFIX.concat("*")) + new Role("index-manager").clusterPermissions("cluster:monitor/health") + .indexPermissions( + "indices:admin/create", + "indices:admin/get", + "indices:admin/delete", + "indices:admin/close", + "indices:admin/close*", + "indices:admin/open", + "indices:admin/resize", + "indices:monitor/stats", + "indices:monitor/settings/get", + "indices:admin/settings/update", + "indices:admin/mapping/put", + "indices:admin/mappings/get", + "indices:admin/cache/clear", + "indices:admin/aliases" + ) + .on(INDICES_ON_WHICH_USER_CAN_PERFORM_INDEX_OPERATIONS_PREFIX.concat("*")) ); private static User USER_ALLOWED_TO_CREATE_INDEX = new User("user-allowed-to-create-index").roles( @@ -2272,14 +2278,15 @@ public void openIndex_negative() throws IOException { } @Test - @Ignore // required permissions: "indices:admin/resize", "indices:monitor/stats - // todo even when I assign the `indices:admin/resize` and `indices:monitor/stats` permissions to test user, this test fails. - // Issue: https://github.com/opensearch-project/security/issues/2141 public void shrinkIndex_positive() throws IOException { String sourceIndexName = INDICES_ON_WHICH_USER_CAN_PERFORM_INDEX_OPERATIONS_PREFIX.concat("shrink_index_positive_source"); - Settings sourceIndexSettings = Settings.builder().put("index.blocks.write", true).put("index.number_of_shards", 2).build(); String targetIndexName = INDICES_ON_WHICH_USER_CAN_PERFORM_INDEX_OPERATIONS_PREFIX.concat("shrink_index_positive_target"); + Settings sourceIndexSettings = Settings.builder() + .put("index.number_of_replicas", 1) + .put("index.blocks.write", true) + .put("index.number_of_shards", 4) + .build(); IndexOperationsHelper.createIndex(cluster, sourceIndexName, sourceIndexSettings); try ( @@ -2287,6 +2294,17 @@ public void shrinkIndex_positive() throws IOException { USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES ) ) { + ClusterHealthResponse healthResponse = restHighLevelClient.cluster() + .health( + new ClusterHealthRequest(sourceIndexName).waitForNoRelocatingShards(true) + .waitForActiveShards(4) + .waitForNoInitializingShards(true) + .waitForGreenStatus(), + DEFAULT + ); + + assertThat(healthResponse.getStatus(), is(ClusterHealthStatus.GREEN)); + ResizeRequest resizeRequest = new ResizeRequest(targetIndexName, sourceIndexName); ResizeResponse response = restHighLevelClient.indices().shrink(resizeRequest, DEFAULT); @@ -2329,10 +2347,7 @@ public void shrinkIndex_negative() throws IOException { } @Test - @Ignore // required permissions: "indices:admin/resize", "indices:monitor/stats - // todo even when I assign the `indices:admin/resize` and `indices:monitor/stats` permissions to test user, this test fails. - // Issue: https://github.com/opensearch-project/security/issues/2141 public void cloneIndex_positive() throws IOException { String sourceIndexName = INDICES_ON_WHICH_USER_CAN_PERFORM_INDEX_OPERATIONS_PREFIX.concat("clone_index_positive_source"); Settings sourceIndexSettings = Settings.builder().put("index.blocks.write", true).build(); @@ -2349,6 +2364,10 @@ public void cloneIndex_positive() throws IOException { assertThat(response, isSuccessfulResizeResponse(targetIndexName)); assertThat(cluster, indexExists(targetIndexName)); + + // can't clone the same index twice, target already exists + ResizeRequest repeatResizeRequest = new ResizeRequest(targetIndexName, sourceIndexName); + assertThatThrownBy(() -> restHighLevelClient.indices().clone(repeatResizeRequest, DEFAULT), statusException(BAD_REQUEST)); } } @@ -2386,10 +2405,7 @@ public void cloneIndex_negative() throws IOException { } @Test - @Ignore // required permissions: "indices:admin/resize", "indices:monitor/stats - // todo even when I assign the `indices:admin/resize` and `indices:monitor/stats` permissions to test user, this test fails. - // Issue: https://github.com/opensearch-project/security/issues/2141 public void splitIndex_positive() throws IOException { String sourceIndexName = INDICES_ON_WHICH_USER_CAN_PERFORM_INDEX_OPERATIONS_PREFIX.concat("split_index_positive_source"); Settings sourceIndexSettings = Settings.builder().put("index.blocks.write", true).build(); diff --git a/src/main/java/org/opensearch/security/resolver/IndexResolverReplacer.java b/src/main/java/org/opensearch/security/resolver/IndexResolverReplacer.java index 3ebfbce29b..d7aeb776c7 100644 --- a/src/main/java/org/opensearch/security/resolver/IndexResolverReplacer.java +++ b/src/main/java/org/opensearch/security/resolver/IndexResolverReplacer.java @@ -56,6 +56,7 @@ import org.opensearch.action.admin.indices.datastream.CreateDataStreamAction; import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest; import org.opensearch.action.admin.indices.resolve.ResolveIndexAction; +import org.opensearch.action.admin.indices.shrink.ResizeRequest; import org.opensearch.action.admin.indices.template.put.PutComponentTemplateAction; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkShardRequest; @@ -777,6 +778,10 @@ private boolean getOrReplaceAllIndices(final Object request, final IndicesProvid return false; } ((CreateIndexRequest) request).index(newIndices.length != 1 ? null : newIndices[0]); + } else if (request instanceof ResizeRequest) { + // clone or shrink operations + provider.provide(((ResizeRequest) request).indices(), request, true); + provider.provide(((ResizeRequest) request).getTargetIndexRequest().indices(), request, true); } else if (request instanceof CreateDataStreamAction.Request) { provider.provide(((CreateDataStreamAction.Request) request).indices(), request, false); } else if (request instanceof ReindexRequest) { From 9bc1cf67729ed8af15928e93fc5ddd58c5cc13f6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 14:23:09 +0000 Subject: [PATCH 024/204] Bump com.diffplug.spotless from 6.22.0 to 6.23.0 (#3768) Bumps com.diffplug.spotless from 6.22.0 to 6.23.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.diffplug.spotless&package-manager=gradle&previous-version=6.22.0&new-version=6.23.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 3c118a9f19..8edd62a83b 100644 --- a/build.gradle +++ b/build.gradle @@ -62,7 +62,7 @@ plugins { id 'idea' id 'jacoco' id 'maven-publish' - id 'com.diffplug.spotless' version '6.22.0' + id 'com.diffplug.spotless' version '6.23.0' id 'checkstyle' id 'com.netflix.nebula.ospackage' version "11.5.0" id "org.gradle.test-retry" version "1.5.6" From 3b070c88792e6df1b72798fcb70029fe86df815a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 09:37:46 -0500 Subject: [PATCH 025/204] Bump com.github.spotbugs from 5.2.3 to 5.2.4 (#3767) Bumps com.github.spotbugs from 5.2.3 to 5.2.4. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.github.spotbugs&package-manager=gradle&previous-version=5.2.3&new-version=5.2.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8edd62a83b..125fb5e39e 100644 --- a/build.gradle +++ b/build.gradle @@ -67,7 +67,7 @@ plugins { id 'com.netflix.nebula.ospackage' version "11.5.0" id "org.gradle.test-retry" version "1.5.6" id 'eclipse' - id "com.github.spotbugs" version "5.2.3" + id "com.github.spotbugs" version "5.2.4" id "com.google.osdetector" version "1.7.3" } From 6b8a3e494c7933b93f6ee4f1a7956b728ed48a43 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 28 Nov 2023 14:22:17 -0500 Subject: [PATCH 026/204] Refactor RestApiComplianceAuditlogTest to use TestAuditLogImpl.doThenWaitForMessages (#3733) ### Description Refactors RestApiComplianceAuditlogTest to make this test suite more stable and re-enable an ignored test. The primary problem in this test suite is that messages that are auto populated on startup get polluted with other audit log messages generated when a user calls REST APIs. To make this more stable, a sleep was added after `setup` to ignore the auto init messages and only isolate messages generated on the API calls in each test. `testAutoInit` is also updated to make it stable. There are 4 automatic messages generated on cluster startup that have to do with the seeding of the security index and audit log index. * Category (Enhancement, New feature, Bug fix, Test fix, Refactoring, Maintenance, Documentation) Test Fix ### Issues Resolved https://github.com/opensearch-project/security/issues/3730 ### Check List - [ ] New functionality includes testing - [ ] New functionality has been documented - [ ] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: Craig Perkins --- .../RestApiComplianceAuditlogTest.java | 211 +++++++++++------- 1 file changed, 134 insertions(+), 77 deletions(-) diff --git a/src/test/java/org/opensearch/security/auditlog/compliance/RestApiComplianceAuditlogTest.java b/src/test/java/org/opensearch/security/auditlog/compliance/RestApiComplianceAuditlogTest.java index e93229ee8b..784176e1dd 100644 --- a/src/test/java/org/opensearch/security/auditlog/compliance/RestApiComplianceAuditlogTest.java +++ b/src/test/java/org/opensearch/security/auditlog/compliance/RestApiComplianceAuditlogTest.java @@ -11,18 +11,26 @@ package org.opensearch.security.auditlog.compliance; +import java.util.List; +import java.util.stream.Collectors; + import org.apache.http.HttpStatus; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import org.opensearch.common.settings.Settings; import org.opensearch.security.auditlog.AbstractAuditlogiUnitTest; +import org.opensearch.security.auditlog.impl.AuditCategory; import org.opensearch.security.auditlog.impl.AuditMessage; import org.opensearch.security.auditlog.integration.TestAuditlogImpl; import org.opensearch.security.support.ConfigConstants; +import org.opensearch.security.test.helper.cluster.ClusterConfiguration; import org.opensearch.security.test.helper.rest.RestHelper.HttpResponse; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + public class RestApiComplianceAuditlogTest extends AbstractAuditlogiUnitTest { @Test @@ -40,22 +48,19 @@ public void testRestApiRolesEnabled() throws Exception { .put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_REST_CATEGORIES, "authenticated,GRANTED_PRIVILEGES") .build(); - setup(additionalSettings); - TestAuditlogImpl.clear(); - String body = "{ \"password\":\"some new password\",\"backend_roles\":[\"role1\",\"role2\"] }"; - HttpResponse response = rh.executePutRequest( - "_opendistro/_security/api/internalusers/compuser?pretty", - body, - encodeBasicHeader("admin", "admin") - ); - Thread.sleep(1500); - Assert.assertEquals(HttpStatus.SC_CREATED, response.getStatusCode()); - Assert.assertTrue(TestAuditlogImpl.messages.size() + "", TestAuditlogImpl.messages.size() == 1); - Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("audit_request_effective_user")); - Assert.assertFalse(TestAuditlogImpl.sb.toString().contains("COMPLIANCE_INTERNAL_CONFIG_READ")); - Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("COMPLIANCE_INTERNAL_CONFIG_WRITE")); - Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("UPDATE")); - validateMsgs(TestAuditlogImpl.messages); + setupAndReturnAuditMessages(additionalSettings); + final AuditMessage message = TestAuditlogImpl.doThenWaitForMessage(() -> { + String body = "{ \"password\":\"some new password\",\"backend_roles\":[\"role1\",\"role2\"] }"; + HttpResponse response = rh.executePutRequest( + "_opendistro/_security/api/internalusers/compuser?pretty", + body, + encodeBasicHeader("admin", "admin") + ); + Assert.assertEquals(HttpStatus.SC_CREATED, response.getStatusCode()); + }); + validateMsgs(List.of(message)); + + assertThat(message.toString(), containsString("UPDATE")); } @Test @@ -72,8 +77,7 @@ public void testRestApiRolesDisabled() throws Exception { .put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_REST_CATEGORIES, "authenticated,GRANTED_PRIVILEGES") .build(); - setup(additionalSettings); - TestAuditlogImpl.clear(); + setupAndReturnAuditMessages(additionalSettings); String body = "{ \"password\":\"some new password\",\"backend_roles\":[\"role1\",\"role2\"] }"; rh.enableHTTPClientSSL = true; @@ -81,19 +85,15 @@ public void testRestApiRolesDisabled() throws Exception { rh.sendAdminCertificate = true; rh.keystore = "kirk-keystore.jks"; - HttpResponse response = rh.executePutRequest("_opendistro/_security/api/internalusers/compuser?pretty", body); - Thread.sleep(1500); - Assert.assertEquals(HttpStatus.SC_CREATED, response.getStatusCode()); - Assert.assertTrue(TestAuditlogImpl.messages.size() + "", TestAuditlogImpl.messages.size() == 1); - Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("audit_request_effective_user")); - Assert.assertFalse(TestAuditlogImpl.sb.toString().contains("COMPLIANCE_INTERNAL_CONFIG_READ")); - Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("COMPLIANCE_INTERNAL_CONFIG_WRITE")); - Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("UPDATE")); - validateMsgs(TestAuditlogImpl.messages); + final AuditMessage message = TestAuditlogImpl.doThenWaitForMessage(() -> { + HttpResponse response = rh.executePutRequest("_opendistro/_security/api/internalusers/compuser?pretty", body); + Assert.assertEquals(HttpStatus.SC_CREATED, response.getStatusCode()); + }); + validateMsgs(List.of(message)); + assertThat(message.toString(), containsString("COMPLIANCE_INTERNAL_CONFIG_WRITE")); } @Test - @Ignore public void testRestApiRolesDisabledGet() throws Exception { Settings additionalSettings = Settings.builder() @@ -107,22 +107,42 @@ public void testRestApiRolesDisabledGet() throws Exception { .put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_REST_CATEGORIES, "authenticated,GRANTED_PRIVILEGES") .build(); - setup(additionalSettings); - TestAuditlogImpl.clear(); + setupAndReturnAuditMessages(additionalSettings); rh.enableHTTPClientSSL = true; rh.trustHTTPServerCertificate = true; rh.sendAdminCertificate = true; rh.keystore = "kirk-keystore.jks"; - HttpResponse response = rh.executeGetRequest("_opendistro/_security/api/rolesmapping/opendistro_security_all_access?pretty"); - Thread.sleep(1500); - Assert.assertEquals(HttpStatus.SC_OK, response.getStatusCode()); - Assert.assertTrue(TestAuditlogImpl.messages.size() > 2); - Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("audit_request_effective_user")); - Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("COMPLIANCE_INTERNAL_CONFIG_READ")); - Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("COMPLIANCE_INTERNAL_CONFIG_WRITE")); - Assert.assertTrue(TestAuditlogImpl.sb.toString().contains("UPDATE")); - validateMsgs(TestAuditlogImpl.messages); + final AuditMessage message = TestAuditlogImpl.doThenWaitForMessage(() -> { + HttpResponse response = rh.executeGetRequest("_opendistro/_security/api/rolesmapping/opendistro_security_all_access?pretty"); + Assert.assertEquals(HttpStatus.SC_OK, response.getStatusCode()); + }); + validateMsgs(List.of(message)); + assertThat(message.toString(), containsString("audit_request_effective_user")); + assertThat(message.toString(), containsString("COMPLIANCE_INTERNAL_CONFIG_READ")); + } + + @Test + public void testAutoInit() throws Exception { + + Settings additionalSettings = Settings.builder() + .put("plugins.security.audit.type", TestAuditlogImpl.class.getName()) + .put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_ENABLE_TRANSPORT, true) + .put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_ENABLE_REST, true) + .put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_RESOLVE_BULK_REQUESTS, false) + .put(ConfigConstants.OPENDISTRO_SECURITY_COMPLIANCE_HISTORY_EXTERNAL_CONFIG_ENABLED, true) + .put(ConfigConstants.SECURITY_COMPLIANCE_HISTORY_INTERNAL_CONFIG_ENABLED, true) + .put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_TRANSPORT_CATEGORIES, "authenticated,GRANTED_PRIVILEGES") + .put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_REST_CATEGORIES, "authenticated,GRANTED_PRIVILEGES") + .build(); + + final List messages = setupAndReturnAuditMessages(additionalSettings); + + validateMsgs(messages); + String allMessages = messages.stream().map(AuditMessage::toString).collect(Collectors.joining(",")); + assertThat(allMessages, containsString("audit_request_effective_user")); + assertThat(allMessages, containsString("COMPLIANCE_INTERNAL_CONFIG_WRITE")); + assertThat(allMessages, containsString("COMPLIANCE_EXTERNAL_CONFIG")); } @Test @@ -136,20 +156,22 @@ public void testRestApiNewUser() throws Exception { .put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_RESOLVE_BULK_REQUESTS, false) .put(ConfigConstants.OPENDISTRO_SECURITY_COMPLIANCE_HISTORY_EXTERNAL_CONFIG_ENABLED, false) .put(ConfigConstants.SECURITY_COMPLIANCE_HISTORY_INTERNAL_CONFIG_ENABLED, true) - .put(ConfigConstants.OPENDISTRO_SECURITY_COMPLIANCE_HISTORY_WRITE_IGNORE_USERS, "admin") .build(); - setup(additionalSettings); - TestAuditlogImpl.clear(); - String body = "{ \"password\":\"some new password\",\"backend_roles\":[\"role1\",\"role2\"] }"; - HttpResponse response = rh.executePutRequest( - "_opendistro/_security/api/internalusers/compuser?pretty", - body, - encodeBasicHeader("admin", "admin") - ); - Thread.sleep(1500); - Assert.assertEquals(response.getBody(), HttpStatus.SC_CREATED, response.getStatusCode()); - Assert.assertTrue(TestAuditlogImpl.messages.size() + "", TestAuditlogImpl.messages.isEmpty()); + setupAndReturnAuditMessages(additionalSettings); + + final AuditMessage message = TestAuditlogImpl.doThenWaitForMessage(() -> { + String body = "{ \"password\":\"some new password\",\"backend_roles\":[\"role1\",\"role2\"] }"; + HttpResponse response = rh.executePutRequest( + "_opendistro/_security/api/internalusers/compuser?pretty", + body, + encodeBasicHeader("admin", "admin") + ); + Assert.assertEquals(response.getBody(), HttpStatus.SC_CREATED, response.getStatusCode()); + }); + validateMsgs(List.of(message)); + assertThat(message.toString(), containsString("audit_request_effective_user")); + assertThat(message.toString(), containsString("COMPLIANCE_INTERNAL_CONFIG_WRITE")); } @Test @@ -167,23 +189,21 @@ public void testRestInternalConfigRead() throws Exception { .put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_REST_CATEGORIES, "authenticated,GRANTED_PRIVILEGES") .build(); - setup(additionalSettings); - TestAuditlogImpl.clear(); + setupAndReturnAuditMessages(additionalSettings); rh.enableHTTPClientSSL = true; rh.trustHTTPServerCertificate = true; rh.sendAdminCertificate = true; rh.keystore = "kirk-keystore.jks"; - HttpResponse response = rh.executeGetRequest("_opendistro/_security/api/internalusers/admin?pretty"); - Thread.sleep(1500); - String auditLogImpl = TestAuditlogImpl.sb.toString(); - Assert.assertEquals(HttpStatus.SC_OK, response.getStatusCode()); - Assert.assertTrue(TestAuditlogImpl.messages.size() + "", TestAuditlogImpl.messages.size() == 1); - Assert.assertTrue(auditLogImpl.contains("audit_request_effective_user")); - Assert.assertTrue(auditLogImpl.contains("COMPLIANCE_INTERNAL_CONFIG_READ")); - Assert.assertFalse(auditLogImpl.contains("COMPLIANCE_INTERNAL_CONFIG_WRITE")); - Assert.assertFalse(auditLogImpl.contains("UPDATE")); - validateMsgs(TestAuditlogImpl.messages); + + final AuditMessage message = TestAuditlogImpl.doThenWaitForMessage(() -> { + HttpResponse response = rh.executeGetRequest("_opendistro/_security/api/internalusers/admin?pretty"); + String auditLogImpl = TestAuditlogImpl.sb.toString(); + Assert.assertEquals(HttpStatus.SC_OK, response.getStatusCode()); + Assert.assertTrue(auditLogImpl.contains("COMPLIANCE_INTERNAL_CONFIG_READ")); + }); + validateMsgs(List.of(message)); + assertThat(message.toString(), containsString("COMPLIANCE_INTERNAL_CONFIG_READ")); } @Test @@ -196,26 +216,63 @@ public void testBCryptHashRedaction() throws Exception { .put(ConfigConstants.SECURITY_COMPLIANCE_HISTORY_INTERNAL_CONFIG_ENABLED, true) .put(ConfigConstants.OPENDISTRO_SECURITY_COMPLIANCE_HISTORY_WRITE_LOG_DIFFS, true) .build(); - setup(settings); + setupAndReturnAuditMessages(settings); rh.sendAdminCertificate = true; rh.keystore = "kirk-keystore.jks"; // read internal users and verify no BCrypt hash is present in audit logs - TestAuditlogImpl.clear(); - rh.executeGetRequest("/_opendistro/_security/api/internalusers"); - Assert.assertEquals(1, TestAuditlogImpl.messages.size()); - Assert.assertFalse(AuditMessage.BCRYPT_HASH.matcher(TestAuditlogImpl.sb.toString()).matches()); + final AuditMessage message1 = TestAuditlogImpl.doThenWaitForMessage(() -> { + rh.executeGetRequest("/_opendistro/_security/api/internalusers"); + }); + + Assert.assertFalse(AuditMessage.BCRYPT_HASH.matcher(message1.toString()).matches()); // read internal user worf and verify no BCrypt hash is present in audit logs - TestAuditlogImpl.clear(); - rh.executeGetRequest("/_opendistro/_security/api/internalusers/worf"); - Assert.assertEquals(1, TestAuditlogImpl.messages.size()); - Assert.assertFalse(AuditMessage.BCRYPT_HASH.matcher(TestAuditlogImpl.sb.toString()).matches()); + final AuditMessage message2 = TestAuditlogImpl.doThenWaitForMessage(() -> { + rh.executeGetRequest("/_opendistro/_security/api/internalusers/worf"); + Assert.assertFalse(AuditMessage.BCRYPT_HASH.matcher(TestAuditlogImpl.sb.toString()).matches()); + }); + + Assert.assertFalse(AuditMessage.BCRYPT_HASH.matcher(message2.toString()).matches()); // create internal user and verify no BCrypt hash is present in audit logs - TestAuditlogImpl.clear(); - rh.executePutRequest("/_opendistro/_security/api/internalusers/test", "{ \"password\":\"some new user password\"}"); - Assert.assertEquals(1, TestAuditlogImpl.messages.size()); - Assert.assertFalse(AuditMessage.BCRYPT_HASH.matcher(TestAuditlogImpl.sb.toString()).matches()); + final AuditMessage message3 = TestAuditlogImpl.doThenWaitForMessage(() -> { + rh.executePutRequest("/_opendistro/_security/api/internalusers/test", "{ \"password\":\"some new user password\"}"); + }); + + Assert.assertFalse(AuditMessage.BCRYPT_HASH.matcher(message3.toString()).matches()); + } + + private List setupAndReturnAuditMessages(Settings settings) { + // When OPENDISTRO_SECURITY_COMPLIANCE_HISTORY_EXTERNAL_CONFIG_ENABLED is set to true, there is: + // - 1 message with COMPLIANCE_INTERNAL_CONFIG_WRITE as category. + // - 1 message with COMPLIANCE_EXTERNAL_CONFIG as category for each node. + int numNodes = ClusterConfiguration.DEFAULT.getNodes(); + boolean externalConfigEnabled = settings.getAsBoolean( + ConfigConstants.OPENDISTRO_SECURITY_COMPLIANCE_HISTORY_EXTERNAL_CONFIG_ENABLED, + false + ); + int expectedMessageCount = externalConfigEnabled ? (numNodes + 1) : 1; + final List messages = TestAuditlogImpl.doThenWaitForMessages(() -> { + try { + setup(settings); + } catch (final Exception ex) { + throw new RuntimeException(ex); + } + }, expectedMessageCount); + int complianceInternalConfigWriteCount = 0; + int complianceExternalConfigCount = 0; + for (AuditMessage message : messages) { + if (AuditCategory.COMPLIANCE_INTERNAL_CONFIG_WRITE.equals(message.getCategory())) { + complianceInternalConfigWriteCount++; + } else if (AuditCategory.COMPLIANCE_EXTERNAL_CONFIG.equals(message.getCategory())) { + complianceExternalConfigCount++; + } + } + assertThat(complianceInternalConfigWriteCount, equalTo(1)); + if (externalConfigEnabled) { + assertThat(complianceExternalConfigCount, equalTo(numNodes)); + } + return messages; } } From 481b373121a05cac0ce45b05a913c68912d161cd Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 30 Nov 2023 13:51:24 -0500 Subject: [PATCH 027/204] Move logic that determines whether to use JDK serialization after getInnerChannel to account for PA interceptor (#3769) ### Description Ensure that channel.getVersion() is called after extraction of inner channel. Signed-off-by: Craig Perkins --- .../transport/SecuritySSLRequestHandler.java | 13 +-- .../SecuritySSLRequestHandlerTests.java | 82 +++++++++++++++++++ 2 files changed, 90 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/opensearch/security/ssl/transport/SecuritySSLRequestHandler.java b/src/main/java/org/opensearch/security/ssl/transport/SecuritySSLRequestHandler.java index 78c98dd99f..39312e29ad 100644 --- a/src/main/java/org/opensearch/security/ssl/transport/SecuritySSLRequestHandler.java +++ b/src/main/java/org/opensearch/security/ssl/transport/SecuritySSLRequestHandler.java @@ -21,6 +21,7 @@ import java.security.cert.Certificate; import java.security.cert.X509Certificate; import java.util.Arrays; +import java.util.Set; import javax.net.ssl.SSLPeerUnverifiedException; import org.apache.logging.log4j.LogManager; @@ -55,6 +56,8 @@ public class SecuritySSLRequestHandler implements Tr private final SslExceptionHandler errorHandler; private final SSLConfig SSLConfig; + private static final Set DEFAULT_CHANNEL_TYPES = Set.of("direct", "transport"); + public SecuritySSLRequestHandler( String action, TransportRequestHandler actualHandler, @@ -86,6 +89,11 @@ public final void messageReceived(T request, TransportChannel channel, Task task ThreadContext threadContext = getThreadContext(); + String channelType = channel.getChannelType(); + if (!DEFAULT_CHANNEL_TYPES.contains(channelType)) { + channel = getInnerChannel(channel); + } + threadContext.putTransient( ConfigConstants.USE_JDK_SERIALIZATION, channel.getVersion().before(ConfigConstants.FIRST_CUSTOM_SERIALIZATION_SUPPORTED_OS_VERSION) @@ -97,11 +105,6 @@ public final void messageReceived(T request, TransportChannel channel, Task task throw exception; } - String channelType = channel.getChannelType(); - if (!channelType.equals("direct") && !channelType.equals("transport")) { - channel = getInnerChannel(channel); - } - if (!"transport".equals(channel.getChannelType())) { // netty4 messageReceivedDecorate(request, actualHandler, channel, task); return; diff --git a/src/test/java/org/opensearch/security/transport/SecuritySSLRequestHandlerTests.java b/src/test/java/org/opensearch/security/transport/SecuritySSLRequestHandlerTests.java index b6967b0e68..2d10b6f84f 100644 --- a/src/test/java/org/opensearch/security/transport/SecuritySSLRequestHandlerTests.java +++ b/src/test/java/org/opensearch/security/transport/SecuritySSLRequestHandlerTests.java @@ -9,12 +9,15 @@ */ package org.opensearch.security.transport; +import java.io.IOException; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.opensearch.Version; import org.opensearch.common.settings.Settings; +import org.opensearch.core.transport.TransportResponse; import org.opensearch.security.ssl.SslExceptionHandler; import org.opensearch.security.ssl.transport.PrincipalExtractor; import org.opensearch.security.ssl.transport.SSLConfig; @@ -27,11 +30,13 @@ import org.opensearch.transport.TransportRequestHandler; import org.mockito.ArgumentMatchers; +import org.mockito.InOrder; import org.mockito.Mock; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -93,4 +98,81 @@ public void testUseJDKSerializationHeaderIsSetOnMessageReceived() throws Excepti Assert.assertThrows(Exception.class, () -> securitySSLRequestHandler.messageReceived(transportRequest, transportChannel, task)); Assert.assertFalse(threadPool.getThreadContext().getTransient(ConfigConstants.USE_JDK_SERIALIZATION)); } + + @Test + public void testUseJDKSerializationHeaderIsSetWithWrapperChannel() throws Exception { + TransportRequest transportRequest = mock(TransportRequest.class); + TransportChannel transportChannel = mock(TransportChannel.class); + TransportChannel wrappedChannel = new WrappedTransportChannel(transportChannel); + Task task = mock(Task.class); + doNothing().when(transportChannel).sendResponse(ArgumentMatchers.any(Exception.class)); + when(transportChannel.getVersion()).thenReturn(Version.V_2_10_0); + when(transportChannel.getChannelType()).thenReturn("other"); + + Assert.assertThrows(Exception.class, () -> securitySSLRequestHandler.messageReceived(transportRequest, wrappedChannel, task)); + Assert.assertTrue(threadPool.getThreadContext().getTransient(ConfigConstants.USE_JDK_SERIALIZATION)); + + threadPool.getThreadContext().stashContext(); + when(transportChannel.getVersion()).thenReturn(Version.V_2_11_0); + Assert.assertThrows(Exception.class, () -> securitySSLRequestHandler.messageReceived(transportRequest, wrappedChannel, task)); + Assert.assertFalse(threadPool.getThreadContext().getTransient(ConfigConstants.USE_JDK_SERIALIZATION)); + + threadPool.getThreadContext().stashContext(); + when(transportChannel.getVersion()).thenReturn(Version.V_3_0_0); + Assert.assertThrows(Exception.class, () -> securitySSLRequestHandler.messageReceived(transportRequest, wrappedChannel, task)); + Assert.assertFalse(threadPool.getThreadContext().getTransient(ConfigConstants.USE_JDK_SERIALIZATION)); + } + + @Test + public void testUseJDKSerializationHeaderIsSetAfterGetInnerChannel() throws Exception { + TransportRequest transportRequest = mock(TransportRequest.class); + TransportChannel transportChannel = mock(TransportChannel.class); + WrappedTransportChannel wrappedChannel = mock(WrappedTransportChannel.class); + Task task = mock(Task.class); + when(wrappedChannel.getInnerChannel()).thenReturn(transportChannel); + when(wrappedChannel.getChannelType()).thenReturn("other"); + doNothing().when(transportChannel).sendResponse(ArgumentMatchers.any(Exception.class)); + when(transportChannel.getVersion()).thenReturn(Version.V_2_10_0); + + Assert.assertThrows(Exception.class, () -> securitySSLRequestHandler.messageReceived(transportRequest, wrappedChannel, task)); + Assert.assertTrue(threadPool.getThreadContext().getTransient(ConfigConstants.USE_JDK_SERIALIZATION)); + + InOrder inOrder = inOrder(wrappedChannel, transportChannel); + + inOrder.verify(wrappedChannel).getInnerChannel(); + inOrder.verify(transportChannel).getVersion(); + } + + public class WrappedTransportChannel implements TransportChannel { + + private TransportChannel inner; + + public WrappedTransportChannel(TransportChannel inner) { + this.inner = inner; + } + + @Override + public String getProfileName() { + return "WrappedTransportChannelProfileName"; + } + + public TransportChannel getInnerChannel() { + return this.inner; + } + + @Override + public void sendResponse(TransportResponse response) throws IOException { + inner.sendResponse(response); + } + + @Override + public void sendResponse(Exception e) throws IOException { + + } + + @Override + public String getChannelType() { + return "WrappedTransportChannelType"; + } + } } From 87de7e2a9a540c03de1ea5669b30ec78e05f95c8 Mon Sep 17 00:00:00 2001 From: Prabhas Kurapati <66924475+prabhask5@users.noreply.github.com> Date: Sat, 2 Dec 2023 02:31:36 -0800 Subject: [PATCH 028/204] [Maintenance-2930] Review use of JSON Flattener (#3674) ### Description Implement JsonFlattener helper class as written in #2926 to deprecate the use of the unnecessary JsonFlattener third party module. * Category (Enhancement, New feature, Bug fix, Test fix, Refactoring, Maintenance, Documentation) Maintenance * Why these changes are required? The JsonFlattener module was being utilized in only one place for one specific purpose, so these functions can be implemented as part of the OpenSearch codebase instead of importing an unnecessary third party module. * What is the old behavior before changes and new behavior after changes? Hopefully nothing. ### Issues Resolved - #2930 Is this a backport? If so, please add backport PR # and/or commits # No ### Testing Tests checked to make sure functions are not broken. ### Check List - [x] New functionality includes testing - [x] New functionality has been documented - [x] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: Prabhas Kurapati --- build.gradle | 7 - .../compliance/FieldReadCallback.java | 3 +- .../security/support/JsonFlattener.java | 66 +++ .../security/support/JsonFlattenerTest.java | 404 ++++++++++++++++++ 4 files changed, 471 insertions(+), 9 deletions(-) create mode 100644 src/main/java/org/opensearch/security/support/JsonFlattener.java create mode 100644 src/test/java/org/opensearch/security/support/JsonFlattenerTest.java diff --git a/build.gradle b/build.gradle index 125fb5e39e..92de00aa26 100644 --- a/build.gradle +++ b/build.gradle @@ -585,13 +585,6 @@ dependencies { implementation "io.jsonwebtoken:jjwt-api:${jjwt_version}" implementation "io.jsonwebtoken:jjwt-impl:${jjwt_version}" implementation "io.jsonwebtoken:jjwt-jackson:${jjwt_version}" - // JSON flattener - implementation ("com.github.wnameless.json:json-base:2.4.3") { - exclude group: "org.glassfish", module: "jakarta.json" - exclude group: "com.google.code.gson", module: "gson" - exclude group: "org.json", module: "json" - } - implementation 'com.github.wnameless.json:json-flattener:0.16.6' // JSON patch implementation 'com.flipkart.zjsonpatch:zjsonpatch:0.4.14' implementation 'org.apache.commons:commons-collections4:4.4' diff --git a/src/main/java/org/opensearch/security/compliance/FieldReadCallback.java b/src/main/java/org/opensearch/security/compliance/FieldReadCallback.java index 210a198e2e..4cce5bb61f 100644 --- a/src/main/java/org/opensearch/security/compliance/FieldReadCallback.java +++ b/src/main/java/org/opensearch/security/compliance/FieldReadCallback.java @@ -33,11 +33,10 @@ import org.opensearch.security.auditlog.AuditLog; import org.opensearch.security.dlic.rest.support.Utils; import org.opensearch.security.support.HeaderHelper; +import org.opensearch.security.support.JsonFlattener; import org.opensearch.security.support.SourceFieldsContext; import org.opensearch.security.support.WildcardMatcher; -import com.github.wnameless.json.flattener.JsonFlattener; - //TODO We need to deal with caching!! //Currently we disable caching (and realtime requests) when FLS or DLS is applied //Check if we can hook in into the caches diff --git a/src/main/java/org/opensearch/security/support/JsonFlattener.java b/src/main/java/org/opensearch/security/support/JsonFlattener.java new file mode 100644 index 0000000000..ba2819a886 --- /dev/null +++ b/src/main/java/org/opensearch/security/support/JsonFlattener.java @@ -0,0 +1,66 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.security.support; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; + +import com.fasterxml.jackson.core.type.TypeReference; + +import org.opensearch.core.common.Strings; +import org.opensearch.security.DefaultObjectMapper; + +public class JsonFlattener { + + public static Map flattenAsMap(String jsonString) { + try { + final TypeReference> typeReference = new TypeReference<>() { + }; + final Map jsonMap = DefaultObjectMapper.objectMapper.readValue(jsonString, typeReference); + final Map flattenMap = new LinkedHashMap<>(); + flattenEntries("", jsonMap.entrySet(), flattenMap); + return flattenMap; + } catch (final IOException ioe) { + throw new IllegalArgumentException("Unparseable json", ioe); + } + } + + private static void flattenEntries(String prefix, final Iterable> entries, final Map result) { + if (!Strings.isNullOrEmpty(prefix)) { + prefix += "."; + } + + for (final Map.Entry e : entries) { + flattenElement(prefix.concat(e.getKey()), e.getValue(), result); + } + } + + @SuppressWarnings("unchecked") + private static void flattenElement(String prefix, final Object source, final Map result) { + if (source instanceof Iterable) { + flattenCollection(prefix, (Iterable) source, result); + } + if (source instanceof Map) { + flattenEntries(prefix, ((Map) source).entrySet(), result); + } + result.put(prefix, source); + } + + private static void flattenCollection(String prefix, final Iterable objects, final Map result) { + int counter = 0; + for (final Object o : objects) { + flattenElement(prefix + "[" + counter + "]", o, result); + counter++; + } + } + +} diff --git a/src/test/java/org/opensearch/security/support/JsonFlattenerTest.java b/src/test/java/org/opensearch/security/support/JsonFlattenerTest.java new file mode 100644 index 0000000000..2880de387b --- /dev/null +++ b/src/test/java/org/opensearch/security/support/JsonFlattenerTest.java @@ -0,0 +1,404 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.security.support; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.junit.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.is; + +public class JsonFlattenerTest { + @Test + public void testFlattenAsMapBasic() { + Map flattenedMap1 = JsonFlattener.flattenAsMap("{\"key\": {\"nested\": 1}, \"another.key\": [\"one\", \"two\"] }"); + assertThat(flattenedMap1.keySet(), containsInAnyOrder("key.nested", "key", "another.key[0]", "another.key[1]", "another.key")); + assertThat( + flattenedMap1.values(), + containsInAnyOrder(1, "one", "two", Arrays.asList("one", "two"), Collections.singletonMap("nested", 1)) + ); + Map flattenedMap2 = JsonFlattener.flattenAsMap("{\"a\":1, \"b\":2, \"cn\":{\"c\":[3,4]}}"); + assertThat(flattenedMap2.keySet(), containsInAnyOrder("a", "b", "cn.c[0]", "cn.c[1]", "cn.c", "cn")); + assertThat( + flattenedMap2.values(), + containsInAnyOrder(1, 2, 3, 4, Arrays.asList(3, 4), Collections.singletonMap("c", Arrays.asList(3, 4))) + ); + Map flattenedMap3 = JsonFlattener.flattenAsMap("{}"); + assertThat(flattenedMap3.keySet(), is(empty())); + assertThat(flattenedMap3.values(), is(empty())); + } + + @Test + public void testFlattenAsMapComplex() { + Map flattenedMap1 = JsonFlattener.flattenAsMap("{\n" + // + " \"a\": {\n" + // + " \"b\": 1,\n" + // + " \"c\": null,\n" + // + " \"d\": [\n" + // + " false,\n" + // + " true\n" + // + " ]\n" + // + " },\n" + // + " \"e\": \"f\",\n" + // + " \"g\": 2.30\n" + // + "}"); + assertThat(flattenedMap1.keySet(), containsInAnyOrder("a.b", "a.c", "a.d[0]", "a.d[1]", "a.d", "a", "e", "g")); + HashMap subMap1 = new HashMap<>(); + subMap1.put("b", 1); + subMap1.put("c", null); + subMap1.put("d", Arrays.asList(false, true)); + assertThat(flattenedMap1.values(), containsInAnyOrder(1, null, false, true, Arrays.asList(false, true), subMap1, "f", 2.3)); + Map flattenedMap2 = JsonFlattener.flattenAsMap( + "{\"a\":{\"b\":1,\"c\":null,\"d\":[false,{\"i\":{\"j\":[false,true,\"xy\"]}}]},\"e\":\"f\",\"g\":2.3,\"z\":[]}" + ); + assertThat( + flattenedMap2.keySet(), + containsInAnyOrder( + "a.b", + "a.c", + "a.d[0]", + "a.d[1].i.j[0]", + "a.d[1].i.j[1]", + "a.d[1].i.j[2]", + "a.d[1].i.j", + "a.d[1].i", + "a.d[1]", + "a.d", + "a", + "e", + "g", + "z" + ) + ); + subMap1 = new HashMap<>(); + subMap1.put("b", 1); + subMap1.put("c", null); + subMap1.put( + "d", + Arrays.asList(false, Collections.singletonMap("i", Collections.singletonMap("j", Arrays.asList(false, true, "xy")))) + ); + assertThat( + flattenedMap2.values(), + containsInAnyOrder( + 1, + null, + false, + false, + true, + "xy", + Arrays.asList(false, true, "xy"), + Collections.singletonMap("j", Arrays.asList(false, true, "xy")), + Collections.singletonMap("i", Collections.singletonMap("j", Arrays.asList(false, true, "xy"))), + Arrays.asList(false, Collections.singletonMap("i", Collections.singletonMap("j", Arrays.asList(false, true, "xy")))), + subMap1, + "f", + 2.3, + Collections.emptyList() + ) + ); + Map flattenedMap3 = JsonFlattener.flattenAsMap("{\n" + // + "\t\"glossary\": {\n" + // + "\t\t\"title\": \"example glossary\",\n" + // + "\t\t\"GlossDiv\": {\n" + // + "\t\t\t\"title\": \"S\",\n" + // + "\t\t\t\"GlossList\": {\n" + // + "\t\t\t\t\"GlossEntry\": {\n" + // + "\t\t\t\t\t\"ID\": \"SGML\",\n" + // + "\t\t\t\t\t\"SortAs\": \"SGML\",\n" + // + "\t\t\t\t\t\"GlossTerm\": \"Standard Generalized Markup Language\",\n" + // + "\t\t\t\t\t\"Acronym\": \"SGML\",\n" + // + "\t\t\t\t\t\"Abbrev\": \"ISO 8879:1986\",\n" + // + "\t\t\t\t\t\"GlossDef\": {\n" + // + "\t\t\t\t\t\t\"para\": \"A meta-markup language, used to create markup languages such as DocBook.\",\n" + // + "\t\t\t\t\t\t\"GlossSeeAlso\": [\n" + // + "\t\t\t\t\t\t\t\"GML\",\n" + // + "\t\t\t\t\t\t\t\"XML\"\n" + // + "\t\t\t\t\t\t]\n" + // + "\t\t\t\t\t},\n" + // + "\t\t\t\t\t\"GlossSee\": \"markup\"\n" + // + "\t\t\t\t}\n" + // + "\t\t\t}\n" + // + "\t\t}\n" + // + "\t}\n" + // + "}"); + assertThat( + flattenedMap3.keySet(), + containsInAnyOrder( + "glossary.title", + "glossary.GlossDiv.title", + "glossary.GlossDiv.GlossList.GlossEntry.ID", + "glossary.GlossDiv.GlossList.GlossEntry.SortAs", + "glossary.GlossDiv.GlossList.GlossEntry.GlossTerm", + "glossary.GlossDiv.GlossList.GlossEntry.Acronym", + "glossary.GlossDiv.GlossList.GlossEntry.Abbrev", + "glossary.GlossDiv.GlossList.GlossEntry.GlossDef.para", + "glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso[0]", + "glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso[1]", + "glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso", + "glossary.GlossDiv.GlossList.GlossEntry.GlossDef", + "glossary.GlossDiv.GlossList.GlossEntry.GlossSee", + "glossary.GlossDiv.GlossList.GlossEntry", + "glossary.GlossDiv.GlossList", + "glossary.GlossDiv", + "glossary" + ) + ); + assertThat( + flattenedMap3.values(), + containsInAnyOrder( + "example glossary", + "S", + "SGML", + "SGML", + "Standard Generalized Markup Language", + "SGML", + "ISO 8879:1986", + "A meta-markup language, used to create markup languages such as DocBook.", + "GML", + "XML", + Arrays.asList("GML", "XML"), + Map.of( + "para", + "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso", + Arrays.asList("GML", "XML") + ), + "markup", + Map.of( + "ID", + "SGML", + "SortAs", + "SGML", + "GlossTerm", + "Standard Generalized Markup Language", + "Acronym", + "SGML", + "Abbrev", + "ISO 8879:1986", + "GlossDef", + Map.of( + "para", + "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso", + Arrays.asList("GML", "XML") + ), + "GlossSee", + "markup" + ), + Map.of( + "GlossEntry", + Map.of( + "ID", + "SGML", + "SortAs", + "SGML", + "GlossTerm", + "Standard Generalized Markup Language", + "Acronym", + "SGML", + "Abbrev", + "ISO 8879:1986", + "GlossDef", + Map.of( + "para", + "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso", + Arrays.asList("GML", "XML") + ), + "GlossSee", + "markup" + ) + ), + Map.of( + "title", + "S", + "GlossList", + Map.of( + "GlossEntry", + Map.of( + "ID", + "SGML", + "SortAs", + "SGML", + "GlossTerm", + "Standard Generalized Markup Language", + "Acronym", + "SGML", + "Abbrev", + "ISO 8879:1986", + "GlossDef", + Map.of( + "para", + "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso", + Arrays.asList("GML", "XML") + ), + "GlossSee", + "markup" + ) + ) + ), + Map.of( + "title", + "example glossary", + "GlossDiv", + Map.of( + "title", + "S", + "GlossList", + Map.of( + "GlossEntry", + Map.of( + "ID", + "SGML", + "SortAs", + "SGML", + "GlossTerm", + "Standard Generalized Markup Language", + "Acronym", + "SGML", + "Abbrev", + "ISO 8879:1986", + "GlossDef", + Map.of( + "para", + "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso", + Arrays.asList("GML", "XML") + ), + "GlossSee", + "markup" + ) + ) + ) + ) + ) + ); + Map flattenedMap4 = JsonFlattener.flattenAsMap("{\n" + // + "\t\"arrayOfObjects\": [\n" + // + "\t\ttrue,\n" + // + "\t\t{\n" + // + "\t\t\t\"x\": 1,\n" + // + "\t\t\t\"y\": 2,\n" + // + "\t\t\t\"z\": [\n" + // + "\t\t\t\t3,\n" + // + "\t\t\t\t4,\n" + // + "\t\t\t\t5\n" + // + "\t\t\t]\n" + // + "\t\t},\n" + // + "\t\t[\n" + // + "\t\t\t6,\n" + // + "\t\t\t7,\n" + // + "\t\t\t8\n" + // + "\t\t],\n" + // + "\t\t[\n" + // + "\t\t\t[\n" + // + "\t\t\t\t9,\n" + // + "\t\t\t\t10\n" + // + "\t\t\t],\n" + // + "\t\t\t11,\n" + // + "\t\t\t12\n" + // + "\t\t],\n" + // + "\t\tfalse\n" + // + "\t],\n" + // + "\t\"boolean\": true,\n" + // + "\t\"color\": \"#82b92c\",\n" + // + "\t\"null\": null,\n" + // + "\t\"number\": 123,\n" + // + "\t\"object\": {\n" + // + "\t\t\"a\": \"b\",\n" + // + "\t\t\"c\": \"d\",\n" + // + "\t\t\"e\": \"f\"\n" + // + "\t},\n" + // + "\t\"string\": \"Hello World\"\n" + // + "}"); + assertThat( + flattenedMap4.keySet(), + containsInAnyOrder( + "arrayOfObjects[0]", + "arrayOfObjects[1].x", + "arrayOfObjects[1].y", + "arrayOfObjects[1].z[0]", + "arrayOfObjects[1].z[1]", + "arrayOfObjects[1].z[2]", + "arrayOfObjects[1].z", + "arrayOfObjects[1]", + "arrayOfObjects[2][0]", + "arrayOfObjects[2][1]", + "arrayOfObjects[2][2]", + "arrayOfObjects[2]", + "arrayOfObjects[3][0][0]", + "arrayOfObjects[3][0][1]", + "arrayOfObjects[3][0]", + "arrayOfObjects[3][1]", + "arrayOfObjects[3][2]", + "arrayOfObjects[3]", + "arrayOfObjects[4]", + "arrayOfObjects", + "boolean", + "color", + "null", + "number", + "object.a", + "object.c", + "object.e", + "object", + "string" + ) + ); + assertThat( + flattenedMap4.values(), + containsInAnyOrder( + true, + 1, + 2, + 3, + 4, + 5, + Arrays.asList(3, 4, 5), + Map.of("x", 1, "y", 2, "z", Arrays.asList(3, 4, 5)), + 6, + 7, + 8, + Arrays.asList(6, 7, 8), + 9, + 10, + Arrays.asList(9, 10), + 11, + 12, + Arrays.asList(Arrays.asList(9, 10), 11, 12), + false, + Arrays.asList( + true, + Map.of("x", 1, "y", 2, "z", Arrays.asList(3, 4, 5)), + Arrays.asList(6, 7, 8), + Arrays.asList(Arrays.asList(9, 10), 11, 12), + false + ), + true, + "#82b92c", + null, + 123, + "b", + "d", + "f", + Map.of("a", "b", "c", "d", "e", "f"), + "Hello World" + ) + ); + } +} From 00658914d2b3158d3a46c86cb08622ab8f714380 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 07:44:43 -0500 Subject: [PATCH 029/204] Bump actions/setup-java from 3 to 4 (#3783) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/setup-java](https://github.com/actions/setup-java) from 3 to 4.
Release notes

Sourced from actions/setup-java's releases.

v4.0.0

What's Changed

In the scope of this release, the version of the Node.js runtime was updated to 20. The majority of dependencies were updated to the latest versions. From now on, the code for the setup-java will run on Node.js 20 instead of Node.js 16.

Breaking changes

Non-breaking changes

New Contributors

Full Changelog: https://github.com/actions/setup-java/compare/v3...v4.0.0

v3.13.0

What's changed

In the scope of this release, support for Dragonwell JDK was added by @​Accelerator1996 in actions/setup-java#532

steps:
 - name: Checkout
   uses: actions/checkout@v3
 - name: Setup-java
   uses: actions/setup-java@v3
   with:
     distribution: 'dragonwell'
     java-version: '17'

Several inaccuracies were also fixed:

New Contributors

Full Changelog: https://github.com/actions/setup-java/compare/v3...v3.13.0

v3.12.0

... (truncated)

Commits
  • 387ac29 Upgrade Node to v20 (#558)
  • 9eda6b5 feat: implement cache-dependency-path option to control caching dependency (#...
  • 78078da Update @​actions/cache dependency and documentation (#549)
  • 5caaba6 add support for microsoft openjdk 21.0.0 (#546)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-java&package-manager=github_actions&previous-version=3&new-version=4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 16 ++++++++-------- .github/workflows/code-hygiene.yml | 6 +++--- .github/workflows/integration-tests.yml | 2 +- .github/workflows/maven-publish.yml | 2 +- .github/workflows/plugin_install.yml | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 399dec5e48..64b8787937 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: separateTestsNames: ${{ steps.set-matrix.outputs.separateTestsNames }} steps: - name: Set up JDK for build and test - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: temurin # Temurin is a distribution of adoptium java-version: 17 @@ -45,7 +45,7 @@ jobs: steps: - name: Set up JDK for build and test - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: temurin # Temurin is a distribution of adoptium java-version: ${{ matrix.jdk }} @@ -116,7 +116,7 @@ jobs: steps: - name: Set up JDK for build and test - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: temurin # Temurin is a distribution of adoptium java-version: ${{ matrix.jdk }} @@ -151,7 +151,7 @@ jobs: steps: - name: Set up JDK for build and test - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: temurin # Temurin is a distribution of adoptium java-version: ${{ matrix.jdk }} @@ -169,7 +169,7 @@ jobs: backward-compatibility-build: runs-on: ubuntu-latest steps: - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: temurin # Temurin is a distribution of adoptium java-version: 17 @@ -193,7 +193,7 @@ jobs: runs-on: ${{ matrix.platform }} steps: - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: temurin # Temurin is a distribution of adoptium java-version: ${{ matrix.jdk }} @@ -214,7 +214,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: temurin # Temurin is a distribution of adoptium java-version: 11 @@ -229,7 +229,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: temurin # Temurin is a distribution of adoptium java-version: 11 diff --git a/.github/workflows/code-hygiene.yml b/.github/workflows/code-hygiene.yml index 6ed51248e9..e6afd8fede 100644 --- a/.github/workflows/code-hygiene.yml +++ b/.github/workflows/code-hygiene.yml @@ -19,7 +19,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: temurin # Temurin is a distribution of adoptium java-version: 17 @@ -35,7 +35,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: temurin # Temurin is a distribution of adoptium java-version: 11 @@ -51,7 +51,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: temurin # Temurin is a distribution of adoptium java-version: 11 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 4c2eddcfbc..183276e675 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -15,7 +15,7 @@ jobs: test-run: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] steps: - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: temurin # Temurin is a distribution of adoptium java-version: ${{ matrix.jdk }} diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml index 1d904020ca..a837289795 100644 --- a/.github/workflows/maven-publish.yml +++ b/.github/workflows/maven-publish.yml @@ -17,7 +17,7 @@ jobs: contents: write steps: - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: temurin # Temurin is a distribution of adoptium java-version: 11 diff --git a/.github/workflows/plugin_install.yml b/.github/workflows/plugin_install.yml index 92d923bb0d..c051e2f6a3 100644 --- a/.github/workflows/plugin_install.yml +++ b/.github/workflows/plugin_install.yml @@ -20,7 +20,7 @@ jobs: uses: peternied/random-name@v1 - name: Set up JDK - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: temurin # Temurin is a distribution of adoptium java-version: ${{ matrix.jdk }} From ad992c5d3cce5c04843dd364b5f5a18570f5a853 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 07:46:23 -0500 Subject: [PATCH 030/204] Bump com.nimbusds:nimbus-jose-jwt from 9.37.1 to 9.37.2 (#3785) Bumps [com.nimbusds:nimbus-jose-jwt](https://bitbucket.org/connect2id/nimbus-jose-jwt) from 9.37.1 to 9.37.2.
Changelog

Sourced from com.nimbusds:nimbus-jose-jwt's changelog.

version 1.0 (2012-03-01)

  • First version based on the OpenInfoCard JWT, JWS and JWE code base.

version 1.1 (2012-03-06)

  • Introduces type-safe enumeration of the JSON Web Algorithms (JWA).
  • Refactors the JWT class.

version 1.2 (2012-03-08)

  • Moves JWS and JWE code into separate classes.

version 1.3 (2012-03-09)

  • Switches to Apache Commons Codec for Base64URL encoding and decoding
  • Consolidates the crypto utilities within the package.
  • Introduces a JWT content serialiser class.

version 1.4 (2012-03-09)

  • Refactoring of JWT class and JUnit tests.

version 1.5 (2012-03-18)

  • Switches to JSON Smart for JSON serialisation and parsing.
  • Introduces claims set class with JSON objects, string, Base64URL and byte array views.

version 1.6 (2012-03-20)

  • Creates class for representing, serialising and parsing JSON Web Keys (JWK).
  • Introduces separate class for representing JWT headers.

version 1.7 (2012-04-01)

  • Introduces separate classes for plain, JWS and JWE headers.
  • Introduces separate classes for plain, signed and encrypted JWTs.
  • Removes the JWTContent class.
  • Removes password-based (PE820) encryption support.

version 1.8 (2012-04-03)

  • Adds support for the ZIP JWE header parameter.
  • Removes unsupported algorithms from the JWA enumeration.

version 1.9 (2012-04-03)

  • Renames JWEHeader.{get|set}EncryptionAlgorithm() to JWEHeader.{get|set}EncryptionMethod().

version 1.9.1 (2012-04-03)

  • Upgrades JSON Smart JAR to 1.1.1.

version 1.10 (2012-04-14)

  • Introduces serialize() method to base abstract JWT class.

version 1.11 (2012-05-13)

  • JWT.serialize() throws checked JWTException instead of

... (truncated)

Commits
  • 8c81358 [maven-release-plugin] prepare for next development iteration
  • 3b3b77e The PasswordBasedDecrypter (PBKDF2) must enforce a limit on the maximum allow...
  • e283ea0 [maven-release-plugin] prepare release 9.37.2
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.nimbusds:nimbus-jose-jwt&package-manager=gradle&previous-version=9.37.1&new-version=9.37.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 92de00aa26..e0a7e27341 100644 --- a/build.gradle +++ b/build.gradle @@ -579,7 +579,7 @@ dependencies { implementation 'commons-cli:commons-cli:1.6.0' implementation "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}" implementation 'org.ldaptive:ldaptive:1.2.3' - implementation 'com.nimbusds:nimbus-jose-jwt:9.37.1' + implementation 'com.nimbusds:nimbus-jose-jwt:9.37.2' //JWT implementation "io.jsonwebtoken:jjwt-api:${jjwt_version}" From 791e886cd10c4e90361e385aae2ec3b4d4233bce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 12:58:15 +0000 Subject: [PATCH 031/204] Bump commons-io:commons-io from 2.15.0 to 2.15.1 (#3784) Bumps commons-io:commons-io from 2.15.0 to 2.15.1. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=commons-io:commons-io&package-manager=gradle&previous-version=2.15.0&new-version=2.15.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index e0a7e27341..9d41e5b220 100644 --- a/build.gradle +++ b/build.gradle @@ -715,7 +715,7 @@ dependencies { integrationTestImplementation 'junit:junit:4.13.2' integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}" integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}" - integrationTestImplementation 'commons-io:commons-io:2.15.0' + integrationTestImplementation 'commons-io:commons-io:2.15.1' integrationTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}" integrationTestImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}" integrationTestImplementation 'org.hamcrest:hamcrest:2.2' From 56ca03fe5ff1465d9081779823898d021ee278f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 13:04:43 +0000 Subject: [PATCH 032/204] Bump com.github.spotbugs from 5.2.4 to 5.2.5 (#3788) Bumps com.github.spotbugs from 5.2.4 to 5.2.5. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.github.spotbugs&package-manager=gradle&previous-version=5.2.4&new-version=5.2.5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 9d41e5b220..a9d73f84d8 100644 --- a/build.gradle +++ b/build.gradle @@ -67,7 +67,7 @@ plugins { id 'com.netflix.nebula.ospackage' version "11.5.0" id "org.gradle.test-retry" version "1.5.6" id 'eclipse' - id "com.github.spotbugs" version "5.2.4" + id "com.github.spotbugs" version "5.2.5" id "com.google.osdetector" version "1.7.3" } From fd08bda37fa7e30fde791cfad647f954fbff9f6f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 13:26:45 +0000 Subject: [PATCH 033/204] Bump org.gradle.test-retry from 1.5.6 to 1.5.7 (#3787) Bumps org.gradle.test-retry from 1.5.6 to 1.5.7. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.gradle.test-retry&package-manager=gradle&previous-version=1.5.6&new-version=1.5.7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a9d73f84d8..b5e979cadf 100644 --- a/build.gradle +++ b/build.gradle @@ -65,7 +65,7 @@ plugins { id 'com.diffplug.spotless' version '6.23.0' id 'checkstyle' id 'com.netflix.nebula.ospackage' version "11.5.0" - id "org.gradle.test-retry" version "1.5.6" + id "org.gradle.test-retry" version "1.5.7" id 'eclipse' id "com.github.spotbugs" version "5.2.5" id "com.google.osdetector" version "1.7.3" From 256d4566f2401410c805c8777fb8cdf7fc3e41e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 09:24:14 -0500 Subject: [PATCH 034/204] Bump com.diffplug.spotless from 6.23.0 to 6.23.3 (#3786) Bumps com.diffplug.spotless from 6.23.0 to 6.23.3. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.diffplug.spotless&package-manager=gradle&previous-version=6.23.0&new-version=6.23.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index b5e979cadf..b2d8e7c292 100644 --- a/build.gradle +++ b/build.gradle @@ -62,7 +62,7 @@ plugins { id 'idea' id 'jacoco' id 'maven-publish' - id 'com.diffplug.spotless' version '6.23.0' + id 'com.diffplug.spotless' version '6.23.3' id 'checkstyle' id 'com.netflix.nebula.ospackage' version "11.5.0" id "org.gradle.test-retry" version "1.5.7" From 2abd71be7293964994649a11ee5967df4a0fa179 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Tue, 5 Dec 2023 12:02:12 -0500 Subject: [PATCH 035/204] Upgrade logback-classic to address CVE CVE-2023-6378 (#3801) ### Description Force resolve logback-classic to 1.3.12 to address https://github.com/advisories/GHSA-vmq6-5m68-f53m ### Issues Resolved [List any issues this PR will resolve] Is this a backport? If so, please add backport PR # and/or commits # ### Testing [Please provide details of testing done: unit testing, integration testing and manual testing] ### Check List - [ ] New functionality includes testing - [ ] New functionality has been documented - [ ] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). Signed-off-by: Derek Ho --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index b2d8e7c292..67055a9b4d 100644 --- a/build.gradle +++ b/build.gradle @@ -497,6 +497,7 @@ configurations { force "org.apache.httpcomponents:httpcore:4.4.16" force "com.google.errorprone:error_prone_annotations:2.23.0" force "org.checkerframework:checker-qual:3.40.0" + force "ch.qos.logback:logback-classic:1.3.12" } } From 07833294e750be097f75c8ccd7d40b882147112f Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Tue, 5 Dec 2023 14:37:13 -0500 Subject: [PATCH 036/204] Revert "Upgrade logback-classic to address CVE CVE-2023-6378" (#3804) Reverts opensearch-project/security#3801 --- build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/build.gradle b/build.gradle index 67055a9b4d..b2d8e7c292 100644 --- a/build.gradle +++ b/build.gradle @@ -497,7 +497,6 @@ configurations { force "org.apache.httpcomponents:httpcore:4.4.16" force "com.google.errorprone:error_prone_annotations:2.23.0" force "org.checkerframework:checker-qual:3.40.0" - force "ch.qos.logback:logback-classic:1.3.12" } } From 93cf507cceca6989444c6b9a047538aa42fb5bf3 Mon Sep 17 00:00:00 2001 From: MaciejMierzwa Date: Fri, 8 Dec 2023 23:12:13 +0100 Subject: [PATCH 037/204] Enable windows platform for integration tests cases (#3706) Signed-off-by: Maciej Mierzwa --- .github/workflows/ci.yml | 2 +- .../security/DlsIntegrationTests.java | 36 ++++++++++--------- .../cluster/SearchRequestFactory.java | 8 +++++ .../security/SlowIntegrationTests.java | 36 ------------------- 4 files changed, 29 insertions(+), 53 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 64b8787937..fc5e322a6d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,7 +111,7 @@ jobs: fail-fast: false matrix: jdk: [11, 17, 21] - platform: [ubuntu-latest] # Removed windows https://github.com/opensearch-project/security/issues/3423 + platform: [ubuntu-latest, windows-latest] runs-on: ${{ matrix.platform }} steps: diff --git a/src/integrationTest/java/org/opensearch/security/DlsIntegrationTests.java b/src/integrationTest/java/org/opensearch/security/DlsIntegrationTests.java index d1957e50a6..aa7202cddf 100644 --- a/src/integrationTest/java/org/opensearch/security/DlsIntegrationTests.java +++ b/src/integrationTest/java/org/opensearch/security/DlsIntegrationTests.java @@ -52,6 +52,7 @@ import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS; import static org.opensearch.test.framework.cluster.SearchRequestFactory.averageAggregationRequest; +import static org.opensearch.test.framework.cluster.SearchRequestFactory.searchRequestWithSort; import static org.opensearch.test.framework.matcher.SearchResponseMatchers.containAggregationWithNameAndType; import static org.opensearch.test.framework.matcher.SearchResponseMatchers.isSuccessfulSearchResponse; import static org.opensearch.test.framework.matcher.SearchResponseMatchers.numberOfTotalHitsIsEqualTo; @@ -281,7 +282,7 @@ public static void createTestData() { public void testShouldSearchAll() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(READ_ALL_USER)) { - SearchRequest searchRequest = new SearchRequest(FIRST_INDEX_NAME); + SearchRequest searchRequest = searchRequestWithSort(FIRST_INDEX_NAME); SearchResponse searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); assertThat(searchResponse, isSuccessfulSearchResponse()); @@ -293,7 +294,7 @@ public void testShouldSearchAll() throws IOException { assertThat(searchResponse, searchHitContainsFieldWithValue(4, FIELD_ARTIST, ARTIST_YES)); assertThat(searchResponse, searchHitContainsFieldWithValue(5, FIELD_ARTIST, ARTIST_UNKNOWN)); - searchRequest = new SearchRequest(SECOND_INDEX_NAME); + searchRequest = searchRequestWithSort(SECOND_INDEX_NAME); searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); assertThat(searchResponse, isSuccessfulSearchResponse()); @@ -301,7 +302,7 @@ public void testShouldSearchAll() throws IOException { assertThat(searchResponse, searchHitContainsFieldWithValue(0, FIELD_ARTIST, ARTIST_NO)); } try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(READ_FIRST_AND_SECOND_USER)) { - SearchRequest searchRequest = new SearchRequest(FIRST_INDEX_NAME); + SearchRequest searchRequest = searchRequestWithSort(FIRST_INDEX_NAME); SearchResponse searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); assertThat(searchResponse, isSuccessfulSearchResponse()); @@ -313,7 +314,7 @@ public void testShouldSearchAll() throws IOException { assertThat(searchResponse, searchHitContainsFieldWithValue(4, FIELD_ARTIST, ARTIST_YES)); assertThat(searchResponse, searchHitContainsFieldWithValue(5, FIELD_ARTIST, ARTIST_UNKNOWN)); - searchRequest = new SearchRequest(SECOND_INDEX_NAME); + searchRequest = searchRequestWithSort(SECOND_INDEX_NAME); searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); assertThat(searchResponse, isSuccessfulSearchResponse()); @@ -326,14 +327,14 @@ public void testShouldSearchAll() throws IOException { public void testShouldSearchI1_S2I2_S3() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(READ_WHERE_FIELD_ARTIST_MATCHES_ARTIST_STRING)) { - SearchRequest searchRequest = new SearchRequest(FIRST_INDEX_NAME); + SearchRequest searchRequest = searchRequestWithSort(FIRST_INDEX_NAME); SearchResponse searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); assertThat(searchResponse, isSuccessfulSearchResponse()); assertThat(searchResponse, numberOfTotalHitsIsEqualTo(1)); assertThat(searchResponse, searchHitContainsFieldWithValue(0, FIELD_ARTIST, ARTIST_STRING)); - searchRequest = new SearchRequest(SECOND_INDEX_NAME); + searchRequest = searchRequestWithSort(SECOND_INDEX_NAME); searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); assertThat(searchResponse, isSuccessfulSearchResponse()); @@ -349,7 +350,7 @@ public void testShouldSearchI1_S3I1_S6I2_S2() throws IOException { READ_WHERE_FIELD_ARTIST_MATCHES_ARTIST_TWINS_OR_FIELD_STARS_GREATER_THAN_FIVE ) ) { - SearchRequest searchRequest = new SearchRequest(FIRST_INDEX_NAME); + SearchRequest searchRequest = searchRequestWithSort(FIRST_INDEX_NAME); SearchResponse searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); assertThat(searchResponse, isSuccessfulSearchResponse()); @@ -357,7 +358,7 @@ public void testShouldSearchI1_S3I1_S6I2_S2() throws IOException { assertThat(searchResponse, searchHitContainsFieldWithValue(0, FIELD_ARTIST, ARTIST_TWINS)); assertThat(searchResponse, searchHitContainsFieldWithValue(1, FIELD_ARTIST, ARTIST_UNKNOWN)); - searchRequest = new SearchRequest(SECOND_INDEX_NAME); + searchRequest = searchRequestWithSort(SECOND_INDEX_NAME); searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); assertThat(searchResponse, isSuccessfulSearchResponse()); @@ -373,7 +374,7 @@ public void testShouldSearchI1_S1I1_S3I2_S2I2_S4() throws IOException { READ_WHERE_FIELD_ARTIST_MATCHES_ARTIST_TWINS_OR_MATCHES_ARTIST_FIRST ) ) { - SearchRequest searchRequest = new SearchRequest(FIRST_INDEX_NAME); + SearchRequest searchRequest = searchRequestWithSort(FIRST_INDEX_NAME); SearchResponse searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); assertThat(searchResponse, isSuccessfulSearchResponse()); @@ -381,7 +382,7 @@ public void testShouldSearchI1_S1I1_S3I2_S2I2_S4() throws IOException { assertThat(searchResponse, searchHitContainsFieldWithValue(0, FIELD_ARTIST, ARTIST_TWINS)); assertThat(searchResponse, searchHitContainsFieldWithValue(1, FIELD_ARTIST, ARTIST_FIRST)); - searchRequest = new SearchRequest(SECOND_INDEX_NAME); + searchRequest = searchRequestWithSort(SECOND_INDEX_NAME); searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); assertThat(searchResponse, isSuccessfulSearchResponse()); @@ -394,7 +395,7 @@ public void testShouldSearchI1_S1I1_S3I2_S2I2_S4() throws IOException { public void testShouldSearchStarsLessThanThree() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(READ_WHERE_STARS_LESS_THAN_THREE)) { - SearchRequest searchRequest = new SearchRequest(FIRST_INDEX_NAME); + SearchRequest searchRequest = searchRequestWithSort(FIRST_INDEX_NAME); SearchResponse searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); assertThat(searchResponse, isSuccessfulSearchResponse()); @@ -402,7 +403,7 @@ public void testShouldSearchStarsLessThanThree() throws IOException { assertThat(searchResponse, searchHitContainsFieldWithValue(0, FIELD_ARTIST, ARTIST_FIRST)); assertThat(searchResponse, searchHitContainsFieldWithValue(1, FIELD_ARTIST, ARTIST_STRING)); - searchRequest = new SearchRequest(SECOND_INDEX_NAME); + searchRequest = searchRequestWithSort(SECOND_INDEX_NAME); searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); assertThat(searchResponse, isSuccessfulSearchResponse()); @@ -417,7 +418,7 @@ public void testSearchForAllDocumentsWithIndexPattern() throws IOException { // DLS try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(READ_ALL_USER)) { - SearchRequest searchRequest = new SearchRequest("*".concat(FIRST_INDEX_NAME)); + SearchRequest searchRequest = searchRequestWithSort("*".concat(FIRST_INDEX_NAME)); SearchResponse searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); assertThat(searchResponse, isSuccessfulSearchResponse()); @@ -429,7 +430,7 @@ public void testSearchForAllDocumentsWithIndexPattern() throws IOException { assertThat(searchResponse, searchHitContainsFieldWithValue(4, FIELD_ARTIST, ARTIST_YES)); assertThat(searchResponse, searchHitContainsFieldWithValue(5, FIELD_ARTIST, ARTIST_UNKNOWN)); - searchRequest = new SearchRequest("*".concat(SECOND_INDEX_NAME)); + searchRequest = searchRequestWithSort("*".concat(SECOND_INDEX_NAME)); searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); assertThat(searchResponse, isSuccessfulSearchResponse()); @@ -442,7 +443,7 @@ public void testSearchForAllDocumentsWithIndexPattern() throws IOException { public void testSearchForAllDocumentsWithAlias() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(READ_ALL_USER)) { - SearchRequest searchRequest = new SearchRequest(FIRST_INDEX_ALIAS); + SearchRequest searchRequest = searchRequestWithSort(FIRST_INDEX_ALIAS); SearchResponse searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); assertThat(searchResponse, isSuccessfulSearchResponse()); @@ -454,12 +455,15 @@ public void testSearchForAllDocumentsWithAlias() throws IOException { assertThat(searchResponse, searchHitContainsFieldWithValue(4, FIELD_ARTIST, ARTIST_YES)); assertThat(searchResponse, searchHitContainsFieldWithValue(5, FIELD_ARTIST, ARTIST_UNKNOWN)); - searchRequest = new SearchRequest("*".concat(SECOND_INDEX_NAME)); + searchRequest = searchRequestWithSort("*".concat(SECOND_INDEX_NAME)); searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); assertThat(searchResponse, isSuccessfulSearchResponse()); assertThat(searchResponse, numberOfTotalHitsIsEqualTo(4)); assertThat(searchResponse, searchHitContainsFieldWithValue(0, FIELD_ARTIST, ARTIST_NO)); + assertThat(searchResponse, searchHitContainsFieldWithValue(1, FIELD_ARTIST, ARTIST_TWINS)); + assertThat(searchResponse, searchHitContainsFieldWithValue(2, FIELD_ARTIST, ARTIST_STRING)); + assertThat(searchResponse, searchHitContainsFieldWithValue(3, FIELD_ARTIST, ARTIST_FIRST)); } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/SearchRequestFactory.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/SearchRequestFactory.java index b40aa9cfcb..cfd0e09088 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/SearchRequestFactory.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/SearchRequestFactory.java @@ -70,6 +70,14 @@ public static SearchRequest searchRequestWithScroll(String indexName, int pageSi return searchRequest; } + public static SearchRequest searchRequestWithSort(String indexName) { + SearchRequest searchRequest = new SearchRequest(indexName); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.sort(new FieldSortBuilder("_id").order(SortOrder.ASC)); + searchRequest.source(searchSourceBuilder); + return searchRequest; + } + public static SearchRequest searchAll(String... indexNames) { SearchRequest searchRequest = new SearchRequest(indexNames); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); diff --git a/src/test/java/org/opensearch/security/SlowIntegrationTests.java b/src/test/java/org/opensearch/security/SlowIntegrationTests.java index 76a162ab9f..0c264ad440 100644 --- a/src/test/java/org/opensearch/security/SlowIntegrationTests.java +++ b/src/test/java/org/opensearch/security/SlowIntegrationTests.java @@ -26,19 +26,14 @@ package org.opensearch.security; -import java.io.IOException; - import com.google.common.collect.Lists; -import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; import org.opensearch.action.admin.cluster.node.info.NodesInfoRequest; -import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.TimeValue; import org.opensearch.node.Node; import org.opensearch.node.PluginAwareNode; import org.opensearch.security.ssl.util.SSLConfigConstants; @@ -47,7 +42,6 @@ import org.opensearch.security.test.SingleClusterTest; import org.opensearch.security.test.helper.cluster.ClusterConfiguration; import org.opensearch.security.test.helper.file.FileHelper; -import org.opensearch.security.test.helper.rest.RestHelper; import org.opensearch.transport.Netty4ModulePlugin; public class SlowIntegrationTests extends SingleClusterTest { @@ -216,34 +210,4 @@ public void testNodeClientDisallowedWithNonServerCertificate2() throws Exception Assert.fail(e.toString()); } } - - @Test - public void testDelayInSecurityIndexInitialization() throws Exception { - final Settings settings = Settings.builder() - .put(ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, true) - .put("cluster.routing.allocation.exclude._ip", "127.0.0.1") - .build(); - try { - setup(Settings.EMPTY, null, settings, false); - Assert.fail("Expected IOException here due to red cluster state"); - } catch (IOException e) { - // Index request has a default timeout of 1 minute, adding buffer between nodes initialization and cluster health check - Thread.sleep(1000 * 80); - // Ideally, we would want to remove this cluster setting, but default settings cannot be removed. So overriding with a reserved - // IP address - clusterHelper.nodeClient() - .admin() - .cluster() - .updateSettings( - new ClusterUpdateSettingsRequest().transientSettings( - Settings.builder().put("cluster.routing.allocation.exclude._ip", "192.0.2.0").build() - ) - ); - this.clusterInfo = clusterHelper.waitForCluster(ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(10), 3); - } - RestHelper rh = nonSslRestHelper(); - Thread.sleep(10000); - Assert.assertEquals(HttpStatus.SC_OK, rh.executeGetRequest("", encodeBasicHeader("admin", "admin")).getStatusCode()); - } - } From a5f3e4e5e9bc80583b19df73f2e01b97c6e018e9 Mon Sep 17 00:00:00 2001 From: Pavlos Daoglou Date: Sat, 9 Dec 2023 00:23:40 +0200 Subject: [PATCH 038/204] [BUG-3791] Adds missing permissions in logstash static role (#3792) Signed-off-by: Pavlos Daoglou --- src/main/resources/static_config/static_roles.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/static_config/static_roles.yml b/src/main/resources/static_config/static_roles.yml index d688848a6e..c7820ab627 100644 --- a/src/main/resources/static_config/static_roles.yml +++ b/src/main/resources/static_config/static_roles.yml @@ -132,6 +132,7 @@ logstash: index_permissions: - index_patterns: - "logstash-*" + - "ecs-logstash-*" allowed_actions: - "create_index" - "crud" From 4911798a5a467cc4a34efc8900f2e5210abb45b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 14:41:29 +0100 Subject: [PATCH 039/204] Bump com.nimbusds:nimbus-jose-jwt from 9.37.2 to 9.37.3 (#3817) Bumps [com.nimbusds:nimbus-jose-jwt](https://bitbucket.org/connect2id/nimbus-jose-jwt) from 9.37.2 to 9.37.3.
Changelog

Sourced from com.nimbusds:nimbus-jose-jwt's changelog.

version 1.0 (2012-03-01)

  • First version based on the OpenInfoCard JWT, JWS and JWE code base.

version 1.1 (2012-03-06)

  • Introduces type-safe enumeration of the JSON Web Algorithms (JWA).
  • Refactors the JWT class.

version 1.2 (2012-03-08)

  • Moves JWS and JWE code into separate classes.

version 1.3 (2012-03-09)

  • Switches to Apache Commons Codec for Base64URL encoding and decoding
  • Consolidates the crypto utilities within the package.
  • Introduces a JWT content serialiser class.

version 1.4 (2012-03-09)

  • Refactoring of JWT class and JUnit tests.

version 1.5 (2012-03-18)

  • Switches to JSON Smart for JSON serialisation and parsing.
  • Introduces claims set class with JSON objects, string, Base64URL and byte array views.

version 1.6 (2012-03-20)

  • Creates class for representing, serialising and parsing JSON Web Keys (JWK).
  • Introduces separate class for representing JWT headers.

version 1.7 (2012-04-01)

  • Introduces separate classes for plain, JWS and JWE headers.
  • Introduces separate classes for plain, signed and encrypted JWTs.
  • Removes the JWTContent class.
  • Removes password-based (PE820) encryption support.

version 1.8 (2012-04-03)

  • Adds support for the ZIP JWE header parameter.
  • Removes unsupported algorithms from the JWA enumeration.

version 1.9 (2012-04-03)

  • Renames JWEHeader.{get|set}EncryptionAlgorithm() to JWEHeader.{get|set}EncryptionMethod().

version 1.9.1 (2012-04-03)

  • Upgrades JSON Smart JAR to 1.1.1.

version 1.10 (2012-04-14)

  • Introduces serialize() method to base abstract JWT class.

version 1.11 (2012-05-13)

  • JWT.serialize() throws checked JWTException instead of

... (truncated)

Commits
  • 2432c09 [maven-release-plugin] prepare for next development iteration
  • fa9737c Bumps Tink and BC deps
  • 431f492 [maven-release-plugin] prepare release 9.37.3
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.nimbusds:nimbus-jose-jwt&package-manager=gradle&previous-version=9.37.2&new-version=9.37.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index b2d8e7c292..0f969d375b 100644 --- a/build.gradle +++ b/build.gradle @@ -579,7 +579,7 @@ dependencies { implementation 'commons-cli:commons-cli:1.6.0' implementation "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}" implementation 'org.ldaptive:ldaptive:1.2.3' - implementation 'com.nimbusds:nimbus-jose-jwt:9.37.2' + implementation 'com.nimbusds:nimbus-jose-jwt:9.37.3' //JWT implementation "io.jsonwebtoken:jjwt-api:${jjwt_version}" From 6fc6967bbd0aa4c4c67c448f9cb2e97aca8095c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 14:41:59 +0100 Subject: [PATCH 040/204] Bump org.gradle.test-retry from 1.5.7 to 1.5.8 (#3816) Bumps org.gradle.test-retry from 1.5.7 to 1.5.8. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.gradle.test-retry&package-manager=gradle&previous-version=1.5.7&new-version=1.5.8)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 0f969d375b..42cd0edd22 100644 --- a/build.gradle +++ b/build.gradle @@ -65,7 +65,7 @@ plugins { id 'com.diffplug.spotless' version '6.23.3' id 'checkstyle' id 'com.netflix.nebula.ospackage' version "11.5.0" - id "org.gradle.test-retry" version "1.5.7" + id "org.gradle.test-retry" version "1.5.8" id 'eclipse' id "com.github.spotbugs" version "5.2.5" id "com.google.osdetector" version "1.7.3" From 581b8fcc8f8bd49b7e48144f44aaec3ebaa68c52 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 14:42:37 +0100 Subject: [PATCH 041/204] Bump kafka_version from 3.6.0 to 3.6.1 (#3815) Bumps `kafka_version` from 3.6.0 to 3.6.1. Updates `org.apache.kafka:kafka-clients` from 3.6.0 to 3.6.1 Updates `org.apache.kafka:kafka_2.13` from 3.6.0 to 3.6.1 Updates `org.apache.kafka:kafka-server-common` from 3.6.0 to 3.6.1 Updates `org.apache.kafka:kafka-group-coordinator` from 3.6.0 to 3.6.1 Updates `org.apache.kafka:kafka-metadata` from 3.6.0 to 3.6.1 Updates `org.apache.kafka:kafka-storage` from 3.6.0 to 3.6.1 Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 42cd0edd22..e7d17c27dc 100644 --- a/build.gradle +++ b/build.gradle @@ -25,7 +25,7 @@ buildscript { opensearch_build = version_tokens[0] + '.0' common_utils_version = System.getProperty("common_utils.version", '3.0.0.0-SNAPSHOT') - kafka_version = '3.6.0' + kafka_version = '3.6.1' apache_cxf_version = '4.0.3' open_saml_version = '4.3.0' one_login_java_saml = '2.9.0' From f90d3cfa0f011fe5969d1c3997f9c0f853fb970d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 14:43:04 +0100 Subject: [PATCH 042/204] Bump commons-validator:commons-validator from 1.7 to 1.8.0 (#3814) Bumps commons-validator:commons-validator from 1.7 to 1.8.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=commons-validator:commons-validator&package-manager=gradle&previous-version=1.7&new-version=1.8.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index e7d17c27dc..7f6d452442 100644 --- a/build.gradle +++ b/build.gradle @@ -677,7 +677,7 @@ dependencies { testImplementation "org.apache.kafka:kafka-group-coordinator:${kafka_version}" testImplementation "org.apache.kafka:kafka_2.13:${kafka_version}:test" testImplementation "org.apache.kafka:kafka-clients:${kafka_version}:test" - testImplementation 'commons-validator:commons-validator:1.7' + testImplementation 'commons-validator:commons-validator:1.8.0' testImplementation 'org.springframework.kafka:spring-kafka-test:2.9.13' testImplementation "org.springframework:spring-beans:${spring_version}" testImplementation 'org.junit.jupiter:junit-jupiter:5.10.1' From 72ce183447c92ce2c7377f2cf9446d86126bf4e4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 14:43:31 +0100 Subject: [PATCH 043/204] Bump org.eclipse.platform:org.eclipse.core.runtime from 3.29.0 to 3.30.0 (#3813) Bumps [org.eclipse.platform:org.eclipse.core.runtime](https://github.com/eclipse-platform/eclipse.platform) from 3.29.0 to 3.30.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.eclipse.platform:org.eclipse.core.runtime&package-manager=gradle&previous-version=3.29.0&new-version=3.30.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 7f6d452442..dc82d5d7bb 100644 --- a/build.gradle +++ b/build.gradle @@ -490,7 +490,7 @@ configurations { force "org.apache.commons:commons-lang3:${versions.commonslang}" // for spotless transitive dependency CVE - force "org.eclipse.platform:org.eclipse.core.runtime:3.29.0" + force "org.eclipse.platform:org.eclipse.core.runtime:3.30.0" // For integrationTest force "org.apache.httpcomponents:httpclient:4.5.14" From 4234823444e9a15fca8a6c9fb31718d03f1d26e1 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Mon, 11 Dec 2023 12:31:18 -0500 Subject: [PATCH 044/204] Resolve logback-classic to 1.2.13 (#3823) ### Description Resolve logback-classic to 1.2.13 to resolve https://github.com/advisories/GHSA-gm62-rw4g-vrc4 ### Issues Resolved Fix: #3821 Is this a backport? If so, please add backport PR # and/or commits # ### Testing [Please provide details of testing done: unit testing, integration testing and manual testing] ### Check List - [ ] New functionality includes testing - [ ] New functionality has been documented - [ ] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). Signed-off-by: Derek Ho --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index dc82d5d7bb..2ed4ad6830 100644 --- a/build.gradle +++ b/build.gradle @@ -497,6 +497,7 @@ configurations { force "org.apache.httpcomponents:httpcore:4.4.16" force "com.google.errorprone:error_prone_annotations:2.23.0" force "org.checkerframework:checker-qual:3.40.0" + force "ch.qos.logback:logback-classic:1.2.13" } } From 06d8c29eea9a648c463e6063542283222e5b1805 Mon Sep 17 00:00:00 2001 From: Darshit Chanpura <35282393+DarshitChanpura@users.noreply.github.com> Date: Tue, 12 Dec 2023 17:24:47 -0500 Subject: [PATCH 045/204] Updates demo script to execute with bundled jdk (#3777) Signed-off-by: Darshit Chanpura --- build.gradle | 4 + .../democonfig/CertificateGenerator.java | 10 +- .../tools/democonfig/Certificates.java | 307 ++++++++++-------- .../security/tools/democonfig/Installer.java | 257 +++++++++------ .../SecuritySettingsConfigurer.java | 246 ++++++++------ tools/install_demo_configuration.bat | 29 +- tools/install_demo_configuration.sh | 46 ++- 7 files changed, 555 insertions(+), 344 deletions(-) diff --git a/build.gradle b/build.gradle index 2ed4ad6830..4b22f1665f 100644 --- a/build.gradle +++ b/build.gradle @@ -256,6 +256,8 @@ test { jvmArgs += "-Xmx3072m" if (JavaVersion.current() > JavaVersion.VERSION_1_8) { jvmArgs += "--add-opens=java.base/java.io=ALL-UNNAMED" + // this is needed to reflect access system env map. + jvmArgs += "--add-opens=java.base/java.util=ALL-UNNAMED" } retry { failOnPassedAfterRetry = false @@ -303,6 +305,8 @@ def setCommonTestConfig(Test task) { task.jvmArgs += "-Xmx3072m" if (JavaVersion.current() > JavaVersion.VERSION_1_8) { task.jvmArgs += "--add-opens=java.base/java.io=ALL-UNNAMED" + // this is needed to reflect access system env map. + task.jvmArgs += "--add-opens=java.base/java.util=ALL-UNNAMED" } task.retry { failOnPassedAfterRetry = false diff --git a/src/main/java/org/opensearch/security/tools/democonfig/CertificateGenerator.java b/src/main/java/org/opensearch/security/tools/democonfig/CertificateGenerator.java index a7b39c226e..077bf4610f 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/CertificateGenerator.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/CertificateGenerator.java @@ -19,14 +19,20 @@ /** * This class creates demo certificate files */ -public class CertificateGenerator extends Installer { +public class CertificateGenerator { + + private final Installer installer; + + public CertificateGenerator(Installer installer) { + this.installer = installer; + } /** * Creates demo super-admin, node and root certificates by iterating through Certificates enum */ public void createDemoCertificates() { for (Certificates cert : Certificates.values()) { - String filePath = OPENSEARCH_CONF_DIR + File.separator + cert.getFileName(); + String filePath = this.installer.OPENSEARCH_CONF_DIR + File.separator + cert.getFileName(); writeCertificateToFile(filePath, cert.getContent()); } } diff --git a/src/main/java/org/opensearch/security/tools/democonfig/Certificates.java b/src/main/java/org/opensearch/security/tools/democonfig/Certificates.java index c776a5e29b..8e2af4dac7 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/Certificates.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/Certificates.java @@ -11,168 +11,191 @@ package org.opensearch.security.tools.democonfig; +import java.util.List; +import java.util.function.Supplier; + /** * Enum for demo certificates */ public enum Certificates { ADMIN_CERT( "kirk.pem", - "-----BEGIN CERTIFICATE-----\n" - + "MIIEmDCCA4CgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iYwDQYJKoZIhvcNAQEL\n" - + "BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt\n" - + "cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl\n" - + "IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v\n" - + "dCBDQTAeFw0yMzA4MjkyMDA2MzdaFw0zMzA4MjYyMDA2MzdaME0xCzAJBgNVBAYT\n" - + "AmRlMQ0wCwYDVQQHDAR0ZXN0MQ8wDQYDVQQKDAZjbGllbnQxDzANBgNVBAsMBmNs\n" - + "aWVudDENMAsGA1UEAwwEa2lyazCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC\n" - + "ggEBAJVcOAQlCiuB9emCljROAXnlsPbG7PE3kNz2sN+BbGuw686Wgyl3uToVHvVs\n" - + "paMmLUqm1KYz9wMSWTIBZgpJ9hYaIbGxD4RBb7qTAJ8Q4ddCV2f7T4lxao/6ixI+\n" - + "O0l/BG9E3mRGo/r0w+jtTQ3aR2p6eoxaOYbVyEMYtFI4QZTkcgGIPGxm05y8xonx\n" - + "vV5pbSW9L7qAVDzQC8EYGQMMI4ccu0NcHKWtmTYJA/wDPE2JwhngHwbcIbc4cDz6\n" - + "cG0S3FmgiKGuuSqUy35v/k3y7zMHQSdx7DSR2tzhH/bBL/9qGvpT71KKrxPtaxS0\n" - + "bAqPcEkKWDo7IMlGGW7LaAWfGg8CAwEAAaOCASswggEnMAwGA1UdEwEB/wQCMAAw\n" - + "DgYDVR0PAQH/BAQDAgXgMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMCMIHPBgNVHSME\n" - + "gccwgcSAFBeH36Ba62YSp9XQ+LoSRTy3KwCcoYGVpIGSMIGPMRMwEQYKCZImiZPy\n" - + "LGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQRXhh\n" - + "bXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENB\n" - + "MSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0GCFHfkrz782p+T9k0G\n" - + "xGeM4+BrehWKMB0GA1UdDgQWBBSjMS8tgguX/V7KSGLoGg7K6XMzIDANBgkqhkiG\n" - + "9w0BAQsFAAOCAQEANMwD1JYlwAh82yG1gU3WSdh/tb6gqaSzZK7R6I0L7slaXN9m\n" - + "y2ErUljpTyaHrdiBFmPhU/2Kj2r+fIUXtXdDXzizx/JdmueT0nG9hOixLqzfoC9p\n" - + "fAhZxM62RgtyZoaczQN82k1/geMSwRpEndFe3OH7arkS/HSbIFxQhAIy229eWe5d\n" - + "1bUzP59iu7f3r567I4ob8Vy7PP+Ov35p7Vv4oDHHwgsdRzX6pvL6mmwVrQ3BfVec\n" - + "h9Dqprr+ukYmjho76g6k5cQuRaB6MxqldzUg+2E7IHQP8MCF+co51uZq2nl33mtp\n" - + "RGr6JbdHXc96zsLTL3saJQ8AWEfu1gbTVrwyRA==\n" - + "-----END CERTIFICATE-----" + () -> getCertContent( + List.of( + "-----BEGIN CERTIFICATE-----", + "MIIEmDCCA4CgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iYwDQYJKoZIhvcNAQEL", + "BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt", + "cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl", + "IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v", + "dCBDQTAeFw0yMzA4MjkyMDA2MzdaFw0zMzA4MjYyMDA2MzdaME0xCzAJBgNVBAYT", + "AmRlMQ0wCwYDVQQHDAR0ZXN0MQ8wDQYDVQQKDAZjbGllbnQxDzANBgNVBAsMBmNs", + "aWVudDENMAsGA1UEAwwEa2lyazCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC", + "ggEBAJVcOAQlCiuB9emCljROAXnlsPbG7PE3kNz2sN+BbGuw686Wgyl3uToVHvVs", + "paMmLUqm1KYz9wMSWTIBZgpJ9hYaIbGxD4RBb7qTAJ8Q4ddCV2f7T4lxao/6ixI+", + "O0l/BG9E3mRGo/r0w+jtTQ3aR2p6eoxaOYbVyEMYtFI4QZTkcgGIPGxm05y8xonx", + "vV5pbSW9L7qAVDzQC8EYGQMMI4ccu0NcHKWtmTYJA/wDPE2JwhngHwbcIbc4cDz6", + "cG0S3FmgiKGuuSqUy35v/k3y7zMHQSdx7DSR2tzhH/bBL/9qGvpT71KKrxPtaxS0", + "bAqPcEkKWDo7IMlGGW7LaAWfGg8CAwEAAaOCASswggEnMAwGA1UdEwEB/wQCMAAw", + "DgYDVR0PAQH/BAQDAgXgMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMCMIHPBgNVHSME", + "gccwgcSAFBeH36Ba62YSp9XQ+LoSRTy3KwCcoYGVpIGSMIGPMRMwEQYKCZImiZPy", + "LGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQRXhh", + "bXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENB", + "MSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0GCFHfkrz782p+T9k0G", + "xGeM4+BrehWKMB0GA1UdDgQWBBSjMS8tgguX/V7KSGLoGg7K6XMzIDANBgkqhkiG", + "9w0BAQsFAAOCAQEANMwD1JYlwAh82yG1gU3WSdh/tb6gqaSzZK7R6I0L7slaXN9m", + "y2ErUljpTyaHrdiBFmPhU/2Kj2r+fIUXtXdDXzizx/JdmueT0nG9hOixLqzfoC9p", + "fAhZxM62RgtyZoaczQN82k1/geMSwRpEndFe3OH7arkS/HSbIFxQhAIy229eWe5d", + "1bUzP59iu7f3r567I4ob8Vy7PP+Ov35p7Vv4oDHHwgsdRzX6pvL6mmwVrQ3BfVec", + "h9Dqprr+ukYmjho76g6k5cQuRaB6MxqldzUg+2E7IHQP8MCF+co51uZq2nl33mtp", + "RGr6JbdHXc96zsLTL3saJQ8AWEfu1gbTVrwyRA==", + "-----END CERTIFICATE-----" + ) + ) ), ADMIN_CERT_KEY( "kirk-key.pem", - "-----BEGIN PRIVATE KEY-----\n" - + "MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCVXDgEJQorgfXp\n" - + "gpY0TgF55bD2xuzxN5Dc9rDfgWxrsOvOloMpd7k6FR71bKWjJi1KptSmM/cDElky\n" - + "AWYKSfYWGiGxsQ+EQW+6kwCfEOHXQldn+0+JcWqP+osSPjtJfwRvRN5kRqP69MPo\n" - + "7U0N2kdqenqMWjmG1chDGLRSOEGU5HIBiDxsZtOcvMaJ8b1eaW0lvS+6gFQ80AvB\n" - + "GBkDDCOHHLtDXBylrZk2CQP8AzxNicIZ4B8G3CG3OHA8+nBtEtxZoIihrrkqlMt+\n" - + "b/5N8u8zB0Encew0kdrc4R/2wS//ahr6U+9Siq8T7WsUtGwKj3BJClg6OyDJRhlu\n" - + "y2gFnxoPAgMBAAECggEAP5TOycDkx+megAWVoHV2fmgvgZXkBrlzQwUG/VZQi7V4\n" - + "ZGzBMBVltdqI38wc5MtbK3TCgHANnnKgor9iq02Z4wXDwytPIiti/ycV9CDRKvv0\n" - + "TnD2hllQFjN/IUh5n4thHWbRTxmdM7cfcNgX3aZGkYbLBVVhOMtn4VwyYu/Mxy8j\n" - + "xClZT2xKOHkxqwmWPmdDTbAeZIbSv7RkIGfrKuQyUGUaWhrPslvYzFkYZ0umaDgQ\n" - + "OAthZew5Bz3OfUGOMPLH61SVPuJZh9zN1hTWOvT65WFWfsPd2yStI+WD/5PU1Doo\n" - + "1RyeHJO7s3ug8JPbtNJmaJwHe9nXBb/HXFdqb976yQKBgQDNYhpu+MYSYupaYqjs\n" - + "9YFmHQNKpNZqgZ4ceRFZ6cMJoqpI5dpEMqToFH7tpor72Lturct2U9nc2WR0HeEs\n" - + "/6tiptyMPTFEiMFb1opQlXF2ae7LeJllntDGN0Q6vxKnQV+7VMcXA0Y8F7tvGDy3\n" - + "qJu5lfvB1mNM2I6y/eMxjBuQhwKBgQC6K41DXMFro0UnoO879pOQYMydCErJRmjG\n" - + "/tZSy3Wj4KA/QJsDSViwGfvdPuHZRaG9WtxdL6kn0w1exM9Rb0bBKl36lvi7o7xv\n" - + "M+Lw9eyXMkww8/F5d7YYH77gIhGo+RITkKI3+5BxeBaUnrGvmHrpmpgRXWmINqr0\n" - + "0jsnN3u0OQKBgCf45vIgItSjQb8zonLz2SpZjTFy4XQ7I92gxnq8X0Q5z3B+o7tQ\n" - + "K/4rNwTju/sGFHyXAJlX+nfcK4vZ4OBUJjP+C8CTjEotX4yTNbo3S6zjMyGQqDI5\n" - + "9aIOUY4pb+TzeUFJX7If5gR+DfGyQubvvtcg1K3GHu9u2l8FwLj87sRzAoGAflQF\n" - + "RHuRiG+/AngTPnZAhc0Zq0kwLkpH2Rid6IrFZhGLy8AUL/O6aa0IGoaMDLpSWUJp\n" - + "nBY2S57MSM11/MVslrEgGmYNnI4r1K25xlaqV6K6ztEJv6n69327MS4NG8L/gCU5\n" - + "3pEm38hkUi8pVYU7in7rx4TCkrq94OkzWJYurAkCgYATQCL/rJLQAlJIGulp8s6h\n" - + "mQGwy8vIqMjAdHGLrCS35sVYBXG13knS52LJHvbVee39AbD5/LlWvjJGlQMzCLrw\n" - + "F7oILW5kXxhb8S73GWcuMbuQMFVHFONbZAZgn+C9FW4l7XyRdkrbR1MRZ2km8YMs\n" - + "/AHmo368d4PSNRMMzLHw8Q==\n" - + "-----END PRIVATE KEY-----" + () -> getCertContent( + List.of( + "-----BEGIN PRIVATE KEY-----", + "MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCVXDgEJQorgfXp", + "gpY0TgF55bD2xuzxN5Dc9rDfgWxrsOvOloMpd7k6FR71bKWjJi1KptSmM/cDElky", + "AWYKSfYWGiGxsQ+EQW+6kwCfEOHXQldn+0+JcWqP+osSPjtJfwRvRN5kRqP69MPo", + "7U0N2kdqenqMWjmG1chDGLRSOEGU5HIBiDxsZtOcvMaJ8b1eaW0lvS+6gFQ80AvB", + "GBkDDCOHHLtDXBylrZk2CQP8AzxNicIZ4B8G3CG3OHA8+nBtEtxZoIihrrkqlMt+", + "b/5N8u8zB0Encew0kdrc4R/2wS//ahr6U+9Siq8T7WsUtGwKj3BJClg6OyDJRhlu", + "y2gFnxoPAgMBAAECggEAP5TOycDkx+megAWVoHV2fmgvgZXkBrlzQwUG/VZQi7V4", + "ZGzBMBVltdqI38wc5MtbK3TCgHANnnKgor9iq02Z4wXDwytPIiti/ycV9CDRKvv0", + "TnD2hllQFjN/IUh5n4thHWbRTxmdM7cfcNgX3aZGkYbLBVVhOMtn4VwyYu/Mxy8j", + "xClZT2xKOHkxqwmWPmdDTbAeZIbSv7RkIGfrKuQyUGUaWhrPslvYzFkYZ0umaDgQ", + "OAthZew5Bz3OfUGOMPLH61SVPuJZh9zN1hTWOvT65WFWfsPd2yStI+WD/5PU1Doo", + "1RyeHJO7s3ug8JPbtNJmaJwHe9nXBb/HXFdqb976yQKBgQDNYhpu+MYSYupaYqjs", + "9YFmHQNKpNZqgZ4ceRFZ6cMJoqpI5dpEMqToFH7tpor72Lturct2U9nc2WR0HeEs", + "/6tiptyMPTFEiMFb1opQlXF2ae7LeJllntDGN0Q6vxKnQV+7VMcXA0Y8F7tvGDy3", + "qJu5lfvB1mNM2I6y/eMxjBuQhwKBgQC6K41DXMFro0UnoO879pOQYMydCErJRmjG", + "/tZSy3Wj4KA/QJsDSViwGfvdPuHZRaG9WtxdL6kn0w1exM9Rb0bBKl36lvi7o7xv", + "M+Lw9eyXMkww8/F5d7YYH77gIhGo+RITkKI3+5BxeBaUnrGvmHrpmpgRXWmINqr0", + "0jsnN3u0OQKBgCf45vIgItSjQb8zonLz2SpZjTFy4XQ7I92gxnq8X0Q5z3B+o7tQ", + "K/4rNwTju/sGFHyXAJlX+nfcK4vZ4OBUJjP+C8CTjEotX4yTNbo3S6zjMyGQqDI5", + "9aIOUY4pb+TzeUFJX7If5gR+DfGyQubvvtcg1K3GHu9u2l8FwLj87sRzAoGAflQF", + "RHuRiG+/AngTPnZAhc0Zq0kwLkpH2Rid6IrFZhGLy8AUL/O6aa0IGoaMDLpSWUJp", + "nBY2S57MSM11/MVslrEgGmYNnI4r1K25xlaqV6K6ztEJv6n69327MS4NG8L/gCU5", + "3pEm38hkUi8pVYU7in7rx4TCkrq94OkzWJYurAkCgYATQCL/rJLQAlJIGulp8s6h", + "mQGwy8vIqMjAdHGLrCS35sVYBXG13knS52LJHvbVee39AbD5/LlWvjJGlQMzCLrw", + "F7oILW5kXxhb8S73GWcuMbuQMFVHFONbZAZgn+C9FW4l7XyRdkrbR1MRZ2km8YMs", + "/AHmo368d4PSNRMMzLHw8Q==", + "-----END PRIVATE KEY-----" + ) + ) ), NODE_CERT( "esnode.pem", - "-----BEGIN CERTIFICATE-----\n" - + "MIIEPDCCAySgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iIwDQYJKoZIhvcNAQEL\n" - + "BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt\n" - + "cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl\n" - + "IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v\n" - + "dCBDQTAeFw0yMzA4MjkwNDIzMTJaFw0zMzA4MjYwNDIzMTJaMFcxCzAJBgNVBAYT\n" - + "AmRlMQ0wCwYDVQQHDAR0ZXN0MQ0wCwYDVQQKDARub2RlMQ0wCwYDVQQLDARub2Rl\n" - + "MRswGQYDVQQDDBJub2RlLTAuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA\n" - + "A4IBDwAwggEKAoIBAQCm93kXteDQHMAvbUPNPW5pyRHKDD42XGWSgq0k1D29C/Ud\n" - + "yL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0\n" - + "HGkn47XVu3EwbfrTENg3jFu+Oem6a/501SzITzJWtS0cn2dIFOBimTVpT/4Zv5qr\n" - + "XA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8n\n" - + "dibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b6l+KLo3IKpfTbAIJXIO+M67FLtWKtttD\n" - + "ao94B069skzKk6FPgW/OZh6PRCD0oxOavV+ld2SjAgMBAAGjgcYwgcMwRwYDVR0R\n" - + "BEAwPogFKgMEBQWCEm5vZGUtMC5leGFtcGxlLmNvbYIJbG9jYWxob3N0hxAAAAAA\n" - + "AAAAAAAAAAAAAAABhwR/AAABMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEF\n" - + "BQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU0/qDQaY10jIo\n" - + "wCjLUpz/HfQXyt8wHwYDVR0jBBgwFoAUF4ffoFrrZhKn1dD4uhJFPLcrAJwwDQYJ\n" - + "KoZIhvcNAQELBQADggEBAD2hkndVih6TWxoe/oOW0i2Bq7ScNO/n7/yHWL04HJmR\n" - + "MaHv/Xjc8zLFLgHuHaRvC02ikWIJyQf5xJt0Oqu2GVbqXH9PBGKuEP2kCsRRyU27\n" - + "zTclAzfQhqmKBTYQ/3lJ3GhRQvXIdYTe+t4aq78TCawp1nSN+vdH/1geG6QjMn5N\n" - + "1FU8tovDd4x8Ib/0dv8RJx+n9gytI8n/giIaDCEbfLLpe4EkV5e5UNpOnRgJjjuy\n" - + "vtZutc81TQnzBtkS9XuulovDE0qI+jQrKkKu8xgGLhgH0zxnPkKtUg2I3Aq6zl1L\n" - + "zYkEOUF8Y25J6WeY88Yfnc0iigI+Pnz5NK8R9GL7TYo=\n" - + "-----END CERTIFICATE-----" + () -> getCertContent( + List.of( + "-----BEGIN CERTIFICATE-----", + "MIIEPDCCAySgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iIwDQYJKoZIhvcNAQEL", + "BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt", + "cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl", + "IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v", + "dCBDQTAeFw0yMzA4MjkwNDIzMTJaFw0zMzA4MjYwNDIzMTJaMFcxCzAJBgNVBAYT", + "AmRlMQ0wCwYDVQQHDAR0ZXN0MQ0wCwYDVQQKDARub2RlMQ0wCwYDVQQLDARub2Rl", + "MRswGQYDVQQDDBJub2RlLTAuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA", + "A4IBDwAwggEKAoIBAQCm93kXteDQHMAvbUPNPW5pyRHKDD42XGWSgq0k1D29C/Ud", + "yL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0", + "HGkn47XVu3EwbfrTENg3jFu+Oem6a/501SzITzJWtS0cn2dIFOBimTVpT/4Zv5qr", + "XA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8n", + "dibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b6l+KLo3IKpfTbAIJXIO+M67FLtWKtttD", + "ao94B069skzKk6FPgW/OZh6PRCD0oxOavV+ld2SjAgMBAAGjgcYwgcMwRwYDVR0R", + "BEAwPogFKgMEBQWCEm5vZGUtMC5leGFtcGxlLmNvbYIJbG9jYWxob3N0hxAAAAAA", + "AAAAAAAAAAAAAAABhwR/AAABMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEF", + "BQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU0/qDQaY10jIo", + "wCjLUpz/HfQXyt8wHwYDVR0jBBgwFoAUF4ffoFrrZhKn1dD4uhJFPLcrAJwwDQYJ", + "KoZIhvcNAQELBQADggEBAD2hkndVih6TWxoe/oOW0i2Bq7ScNO/n7/yHWL04HJmR", + "MaHv/Xjc8zLFLgHuHaRvC02ikWIJyQf5xJt0Oqu2GVbqXH9PBGKuEP2kCsRRyU27", + "zTclAzfQhqmKBTYQ/3lJ3GhRQvXIdYTe+t4aq78TCawp1nSN+vdH/1geG6QjMn5N", + "1FU8tovDd4x8Ib/0dv8RJx+n9gytI8n/giIaDCEbfLLpe4EkV5e5UNpOnRgJjjuy", + "vtZutc81TQnzBtkS9XuulovDE0qI+jQrKkKu8xgGLhgH0zxnPkKtUg2I3Aq6zl1L", + "zYkEOUF8Y25J6WeY88Yfnc0iigI+Pnz5NK8R9GL7TYo=", + "-----END CERTIFICATE-----" + ) + ) ), NODE_KEY( "esnode-key.pem", - "-----BEGIN PRIVATE KEY-----\n" - + "MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCm93kXteDQHMAv\n" - + "bUPNPW5pyRHKDD42XGWSgq0k1D29C/UdyL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0\n" - + "o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0HGkn47XVu3EwbfrTENg3jFu+Oem6a/50\n" - + "1SzITzJWtS0cn2dIFOBimTVpT/4Zv5qrXA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1\n" - + "MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8ndibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b\n" - + "6l+KLo3IKpfTbAIJXIO+M67FLtWKtttDao94B069skzKk6FPgW/OZh6PRCD0oxOa\n" - + "vV+ld2SjAgMBAAECggEAQK1+uAOZeaSZggW2jQut+MaN4JHLi61RH2cFgU3COLgo\n" - + "FIiNjFn8f2KKU3gpkt1It8PjlmprpYut4wHI7r6UQfuv7ZrmncRiPWHm9PB82+ZQ\n" - + "5MXYqj4YUxoQJ62Cyz4sM6BobZDrjG6HHGTzuwiKvHHkbsEE9jQ4E5m7yfbVvM0O\n" - + "zvwrSOM1tkZihKSTpR0j2+taji914tjBssbn12TMZQL5ItGnhR3luY8mEwT9MNkZ\n" - + "xg0VcREoAH+pu9FE0vPUgLVzhJ3be7qZTTSRqv08bmW+y1plu80GbppePcgYhEow\n" - + "dlW4l6XPJaHVSn1lSFHE6QAx6sqiAnBz0NoTPIaLyQKBgQDZqDOlhCRciMRicSXn\n" - + "7yid9rhEmdMkySJHTVFOidFWwlBcp0fGxxn8UNSBcXdSy7GLlUtH41W9PWl8tp9U\n" - + "hQiiXORxOJ7ZcB80uNKXF01hpPj2DpFPWyHFxpDkWiTAYpZl68rOlYujxZUjJIej\n" - + "VvcykBC2BlEOG9uZv2kxcqLyJwKBgQDEYULTxaTuLIa17wU3nAhaainKB3vHxw9B\n" - + "Ksy5p3ND43UNEKkQm7K/WENx0q47TA1mKD9i+BhaLod98mu0YZ+BCUNgWKcBHK8c\n" - + "uXpauvM/pLhFLXZ2jvEJVpFY3J79FSRK8bwE9RgKfVKMMgEk4zOyZowS8WScOqiy\n" - + "hnQn1vKTJQKBgElhYuAnl9a2qXcC7KOwRsJS3rcKIVxijzL4xzOyVShp5IwIPbOv\n" - + "hnxBiBOH/JGmaNpFYBcBdvORE9JfA4KMQ2fx53agfzWRjoPI1/7mdUk5RFI4gRb/\n" - + "A3jZRBoopgFSe6ArCbnyQxzYzToG48/Wzwp19ZxYrtUR4UyJct6f5n27AoGBAJDh\n" - + "KIpQQDOvCdtjcbfrF4aM2DPCfaGPzENJriwxy6oEPzDaX8Bu/dqI5Ykt43i/zQrX\n" - + "GpyLaHvv4+oZVTiI5UIvcVO9U8hQPyiz9f7F+fu0LHZs6f7hyhYXlbe3XFxeop3f\n" - + "5dTKdWgXuTTRF2L9dABkA2deS9mutRKwezWBMQk5AoGBALPtX0FrT1zIosibmlud\n" - + "tu49A/0KZu4PBjrFMYTSEWGNJez3Fb2VsJwylVl6HivwbP61FhlYfyksCzQQFU71\n" - + "+x7Nmybp7PmpEBECr3deoZKQ/acNHn0iwb0It+YqV5+TquQebqgwK6WCLsMuiYKT\n" - + "bg/ch9Rhxbq22yrVgWHh6epp\n" - + "-----END PRIVATE KEY-----" + () -> getCertContent( + List.of( + "-----BEGIN PRIVATE KEY-----", + "MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCm93kXteDQHMAv", + "bUPNPW5pyRHKDD42XGWSgq0k1D29C/UdyL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0", + "o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0HGkn47XVu3EwbfrTENg3jFu+Oem6a/50", + "1SzITzJWtS0cn2dIFOBimTVpT/4Zv5qrXA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1", + "MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8ndibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b", + "6l+KLo3IKpfTbAIJXIO+M67FLtWKtttDao94B069skzKk6FPgW/OZh6PRCD0oxOa", + "vV+ld2SjAgMBAAECggEAQK1+uAOZeaSZggW2jQut+MaN4JHLi61RH2cFgU3COLgo", + "FIiNjFn8f2KKU3gpkt1It8PjlmprpYut4wHI7r6UQfuv7ZrmncRiPWHm9PB82+ZQ", + "5MXYqj4YUxoQJ62Cyz4sM6BobZDrjG6HHGTzuwiKvHHkbsEE9jQ4E5m7yfbVvM0O", + "zvwrSOM1tkZihKSTpR0j2+taji914tjBssbn12TMZQL5ItGnhR3luY8mEwT9MNkZ", + "xg0VcREoAH+pu9FE0vPUgLVzhJ3be7qZTTSRqv08bmW+y1plu80GbppePcgYhEow", + "dlW4l6XPJaHVSn1lSFHE6QAx6sqiAnBz0NoTPIaLyQKBgQDZqDOlhCRciMRicSXn", + "7yid9rhEmdMkySJHTVFOidFWwlBcp0fGxxn8UNSBcXdSy7GLlUtH41W9PWl8tp9U", + "hQiiXORxOJ7ZcB80uNKXF01hpPj2DpFPWyHFxpDkWiTAYpZl68rOlYujxZUjJIej", + "VvcykBC2BlEOG9uZv2kxcqLyJwKBgQDEYULTxaTuLIa17wU3nAhaainKB3vHxw9B", + "Ksy5p3ND43UNEKkQm7K/WENx0q47TA1mKD9i+BhaLod98mu0YZ+BCUNgWKcBHK8c", + "uXpauvM/pLhFLXZ2jvEJVpFY3J79FSRK8bwE9RgKfVKMMgEk4zOyZowS8WScOqiy", + "hnQn1vKTJQKBgElhYuAnl9a2qXcC7KOwRsJS3rcKIVxijzL4xzOyVShp5IwIPbOv", + "hnxBiBOH/JGmaNpFYBcBdvORE9JfA4KMQ2fx53agfzWRjoPI1/7mdUk5RFI4gRb/", + "A3jZRBoopgFSe6ArCbnyQxzYzToG48/Wzwp19ZxYrtUR4UyJct6f5n27AoGBAJDh", + "KIpQQDOvCdtjcbfrF4aM2DPCfaGPzENJriwxy6oEPzDaX8Bu/dqI5Ykt43i/zQrX", + "GpyLaHvv4+oZVTiI5UIvcVO9U8hQPyiz9f7F+fu0LHZs6f7hyhYXlbe3XFxeop3f", + "5dTKdWgXuTTRF2L9dABkA2deS9mutRKwezWBMQk5AoGBALPtX0FrT1zIosibmlud", + "tu49A/0KZu4PBjrFMYTSEWGNJez3Fb2VsJwylVl6HivwbP61FhlYfyksCzQQFU71", + "+x7Nmybp7PmpEBECr3deoZKQ/acNHn0iwb0It+YqV5+TquQebqgwK6WCLsMuiYKT", + "bg/ch9Rhxbq22yrVgWHh6epp", + "-----END PRIVATE KEY-----" + ) + ) ), ROOT_CA( "root-ca.pem", - "-----BEGIN CERTIFICATE-----\n" - + "MIIExjCCA66gAwIBAgIUd+SvPvzan5P2TQbEZ4zj4Gt6FYowDQYJKoZIhvcNAQEL\n" - + "BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt\n" - + "cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl\n" - + "IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v\n" - + "dCBDQTAeFw0yMzA4MjkwNDIwMDNaFw0yMzA5MjgwNDIwMDNaMIGPMRMwEQYKCZIm\n" - + "iZPyLGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQ\n" - + "RXhhbXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290\n" - + "IENBMSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0EwggEiMA0GCSqG\n" - + "SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEPyN7J9VGPyJcQmCBl5TGwfSzvVdWwoQU\n" - + "j9aEsdfFJ6pBCDQSsj8Lv4RqL0dZra7h7SpZLLX/YZcnjikrYC+rP5OwsI9xEE/4\n" - + "U98CsTBPhIMgqFK6SzNE5494BsAk4cL72dOOc8tX19oDS/PvBULbNkthQ0aAF1dg\n" - + "vbrHvu7hq7LisB5ZRGHVE1k/AbCs2PaaKkn2jCw/b+U0Ml9qPuuEgz2mAqJDGYoA\n" - + "WSR4YXrOcrmPuRqbws464YZbJW898/0Pn/U300ed+4YHiNYLLJp51AMkR4YEw969\n" - + "VRPbWIvLrd0PQBooC/eLrL6rvud/GpYhdQEUx8qcNCKd4bz3OaQ5AgMBAAGjggEW\n" - + "MIIBEjAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQU\n" - + "F4ffoFrrZhKn1dD4uhJFPLcrAJwwgc8GA1UdIwSBxzCBxIAUF4ffoFrrZhKn1dD4\n" - + "uhJFPLcrAJyhgZWkgZIwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJ\n" - + "k/IsZAEZFgdleGFtcGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYD\n" - + "VQQLDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUg\n" - + "Q29tIEluYy4gUm9vdCBDQYIUd+SvPvzan5P2TQbEZ4zj4Gt6FYowDQYJKoZIhvcN\n" - + "AQELBQADggEBAIopqco/k9RSjouTeKP4z0EVUxdD4qnNh1GLSRqyAVe0aChyKF5f\n" - + "qt1Bd1XCY8D16RgekkKGHDpJhGCpel+vtIoXPBxUaGQNYxmJCf5OzLMODlcrZk5i\n" - + "jHIcv/FMeK02NBcz/WQ3mbWHVwXLhmwqa2zBsF4FmPCJAbFLchLhkAv1HJifHbnD\n" - + "jQzlKyl5jxam/wtjWxSm0iyso0z2TgyzY+MESqjEqB1hZkCFzD1xtUOCxbXgtKae\n" - + "dgfHVFuovr3fNLV3GvQk0s9okDwDUcqV7DSH61e5bUMfE84o3of8YA7+HUoPV5Du\n" - + "8sTOKRf7ncGXdDRA8aofW268pTCuIu3+g/Y=\n" - + "-----END CERTIFICATE-----" + () -> getCertContent( + List.of( + "-----BEGIN CERTIFICATE-----", + "MIIExjCCA66gAwIBAgIUd+SvPvzan5P2TQbEZ4zj4Gt6FYowDQYJKoZIhvcNAQEL", + "BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt", + "cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl", + "IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v", + "dCBDQTAeFw0yMzA4MjkwNDIwMDNaFw0yMzA5MjgwNDIwMDNaMIGPMRMwEQYKCZIm", + "iZPyLGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQ", + "RXhhbXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290", + "IENBMSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0EwggEiMA0GCSqG", + "SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEPyN7J9VGPyJcQmCBl5TGwfSzvVdWwoQU", + "j9aEsdfFJ6pBCDQSsj8Lv4RqL0dZra7h7SpZLLX/YZcnjikrYC+rP5OwsI9xEE/4", + "U98CsTBPhIMgqFK6SzNE5494BsAk4cL72dOOc8tX19oDS/PvBULbNkthQ0aAF1dg", + "vbrHvu7hq7LisB5ZRGHVE1k/AbCs2PaaKkn2jCw/b+U0Ml9qPuuEgz2mAqJDGYoA", + "WSR4YXrOcrmPuRqbws464YZbJW898/0Pn/U300ed+4YHiNYLLJp51AMkR4YEw969", + "VRPbWIvLrd0PQBooC/eLrL6rvud/GpYhdQEUx8qcNCKd4bz3OaQ5AgMBAAGjggEW", + "MIIBEjAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQU", + "F4ffoFrrZhKn1dD4uhJFPLcrAJwwgc8GA1UdIwSBxzCBxIAUF4ffoFrrZhKn1dD4", + "uhJFPLcrAJyhgZWkgZIwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJ", + "k/IsZAEZFgdleGFtcGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYD", + "VQQLDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUg", + "Q29tIEluYy4gUm9vdCBDQYIUd+SvPvzan5P2TQbEZ4zj4Gt6FYowDQYJKoZIhvcN", + "AQELBQADggEBAIopqco/k9RSjouTeKP4z0EVUxdD4qnNh1GLSRqyAVe0aChyKF5f", + "qt1Bd1XCY8D16RgekkKGHDpJhGCpel+vtIoXPBxUaGQNYxmJCf5OzLMODlcrZk5i", + "jHIcv/FMeK02NBcz/WQ3mbWHVwXLhmwqa2zBsF4FmPCJAbFLchLhkAv1HJifHbnD", + "jQzlKyl5jxam/wtjWxSm0iyso0z2TgyzY+MESqjEqB1hZkCFzD1xtUOCxbXgtKae", + "dgfHVFuovr3fNLV3GvQk0s9okDwDUcqV7DSH61e5bUMfE84o3of8YA7+HUoPV5Du", + "8sTOKRf7ncGXdDRA8aofW268pTCuIu3+g/Y=", + "-----END CERTIFICATE-----" + ) + ) ); private final String fileName; - private final String content; + private final Supplier contentSupplier; - Certificates(String fileName, String content) { + Certificates(String fileName, Supplier contentSupplier) { this.fileName = fileName; - this.content = content; + this.contentSupplier = contentSupplier; } public String getFileName() { @@ -180,6 +203,10 @@ public String getFileName() { } public String getContent() { - return content; + return contentSupplier.get(); + } + + private static String getCertContent(List certLines) { + return String.join(System.lineSeparator(), certLines); } } diff --git a/src/main/java/org/opensearch/security/tools/democonfig/Installer.java b/src/main/java/org/opensearch/security/tools/democonfig/Installer.java index 0b166ad580..500c65f825 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/Installer.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/Installer.java @@ -23,47 +23,83 @@ import java.util.Scanner; import java.util.Set; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + /** * This class installs demo configuration for security plugin */ public class Installer { - static boolean assumeyes = false; - static boolean initsecurity = false; - static boolean cluster_mode = false; - static int skip_updates = -1; - static String SCRIPT_DIR; - static String BASE_DIR; - static String OPENSEARCH_CONF_FILE; - static String OPENSEARCH_BIN_DIR; - static String OPENSEARCH_PLUGINS_DIR; - static String OPENSEARCH_LIB_PATH; - static String OPENSEARCH_INSTALL_TYPE; - static String OPENSEARCH_CONF_DIR; - static String OPENSEARCH_VERSION; - static String SECURITY_VERSION; + // Singleton Pattern + private static Installer instance; - static ExecutionEnvironment environment = ExecutionEnvironment.DEMO; + private static SecuritySettingsConfigurer securitySettingsConfigurer; - static final String OS = System.getProperty("os.name") + " " + System.getProperty("os.version") + " " + System.getProperty("os.arch"); + private static CertificateGenerator certificateGenerator; - static final String FILE_EXTENSION = OS.toLowerCase().contains("win") ? ".bat" : ".sh"; + boolean assumeyes = false; + boolean initsecurity = false; + boolean cluster_mode = false; + int skip_updates = -1; + String SCRIPT_DIR; + String BASE_DIR; + String OPENSEARCH_CONF_FILE; + String OPENSEARCH_BIN_DIR; + String OPENSEARCH_PLUGINS_DIR; + String OPENSEARCH_LIB_PATH; + String OPENSEARCH_INSTALL_TYPE; + String OPENSEARCH_CONF_DIR; + String OPENSEARCH_VERSION; + String SECURITY_VERSION; - static final String SYSTEM_INDICES = ".plugins-ml-config, .plugins-ml-connector, .plugins-ml-model-group, .plugins-ml-model, " - + ".plugins-ml-task, .plugins-ml-conversation-meta, .plugins-ml-conversation-interactions, .opendistro-alerting-config, .opendistro-alerting-alert*, " - + ".opendistro-anomaly-results*, .opendistro-anomaly-detector*, .opendistro-anomaly-checkpoints, .opendistro-anomaly-detection-state, " - + ".opendistro-reports-*, .opensearch-notifications-*, .opensearch-notebooks, .opensearch-observability, .ql-datasources, " - + ".opendistro-asynchronous-search-response*, .replication-metadata-store, .opensearch-knn-models, .geospatial-ip2geo-data*"; + ExecutionEnvironment environment = ExecutionEnvironment.DEMO; - static SecuritySettingsConfigurer securitySettingsConfigurer; - static CertificateGenerator certificateGenerator; + String OS; - public static void main(String[] options) { - securitySettingsConfigurer = new SecuritySettingsConfigurer(); - certificateGenerator = new CertificateGenerator(); + final String FILE_EXTENSION; - printScriptHeaders(); + static File RPM_DEB_OPENSEARCH_HOME = new File("/usr/share/opensearch"); + + private final Options options; + + // To print help information for this script + private final HelpFormatter formatter = new HelpFormatter(); + + /** + * We do not want this class to be instantiated more than once, + * as we are following Singleton Factory pattern + */ + private Installer() { + this.OS = System.getProperty("os.name") + " " + System.getProperty("os.version") + " " + System.getProperty("os.arch"); + FILE_EXTENSION = OS.toLowerCase().contains("win") ? ".bat" : ".sh"; + options = new Options(); + } + + /** + * Returns a singleton instance of this class + * @return an existing instance OR a new instance if there was no existing instance + */ + public static Installer getInstance() { + if (instance == null) { + instance = new Installer(); + securitySettingsConfigurer = new SecuritySettingsConfigurer(instance); + certificateGenerator = new CertificateGenerator(instance); + } + return instance; + } + + /** + * Installs the demo security configuration + * @param options the options passed to the script + */ + public void installDemoConfiguration(String[] options) { readOptions(options); + printScriptHeaders(); gatherUserInputs(); initializeVariables(); printVariables(); @@ -72,6 +108,44 @@ public static void main(String[] options) { finishScriptExecution(); } + public static void main(String[] options) { + Installer installer = Installer.getInstance(); + installer.buildOptions(); + installer.installDemoConfiguration(options); + } + + /** + * Builds options supported by this tool + */ + void buildOptions() { + options.addOption("h", "show-help", false, "Shows help for this tool."); + options.addOption("y", "answer-yes-to-all-prompts", false, "Confirm all installation dialogues automatically."); + options.addOption( + "i", + "initialize-security", + false, + "Initialize Security plugin with default configuration (default is to ask if -y is not given)." + ); + options.addOption( + "c", + "enable-cluster-mode", + false, + "Enable cluster mode by binding to all network interfaces (default is to ask if -y is not given)." + ); + options.addOption( + "s", + "skip-updates-when-already-configured", + false, + "Skip updates if config is already applied to opensearch.yml." + ); + options.addOption( + "t", + "test-execution-environment", + false, + "Set the execution environment to `test` to skip password validation. Should be used only for testing. (default is set to `demo`)" + ); + } + /** * Prints headers that indicate the start of script execution */ @@ -82,52 +156,37 @@ static void printScriptHeaders() { /** * Reads the options passed to the script - * @param options an array of strings containing options passed to the script + * @param args an array of strings containing options passed to the script */ - static void readOptions(String[] options) { + void readOptions(String[] args) { // set script execution dir - SCRIPT_DIR = options[0]; - - for (int i = 1; i < options.length; i++) { - switch (options[i]) { - case "-y": - assumeyes = true; - break; - case "-i": - initsecurity = true; - break; - case "-c": - cluster_mode = true; - break; - case "-s": - skip_updates = 0; - break; - case "-t": - environment = ExecutionEnvironment.TEST; - break; - case "-h": - case "-?": - showHelp(); - return; - default: - System.out.println("Invalid option: " + options[i]); + SCRIPT_DIR = args[0]; + + CommandLineParser parser = new DefaultParser(); + try { + CommandLine line = parser.parse(options, args); + + if (line.hasOption("h")) { + showHelp(); + return; } + assumeyes = line.hasOption("y"); + initsecurity = line.hasOption("i"); + cluster_mode = line.hasOption("c"); + skip_updates = line.hasOption("s") ? 0 : -1; + environment = line.hasOption("t") ? ExecutionEnvironment.TEST : environment; + + } catch (ParseException exp) { + System.out.println("ERR: Parsing failed. Reason: " + exp.getMessage()); + System.exit(-1); } } /** * Prints the help menu when -h option is passed */ - static void showHelp() { - System.out.println("install_demo_configuration.sh [-y] [-i] [-c]"); - System.out.println(" -h show help"); - System.out.println(" -y confirm all installation dialogues automatically"); - System.out.println(" -i initialize Security plugin with default configuration (default is to ask if -y is not given)"); - System.out.println(" -c enable cluster mode by binding to all network interfaces (default is to ask if -y is not given)"); - System.out.println(" -s skip updates if config is already applied to opensearch.yml"); - System.out.println( - " -t set the execution environment to `test` to skip password validation. Should be used only for testing. (default is set to `demo`)" - ); + void showHelp() { + formatter.printHelp("install_demo_configuration.sh", options, true); System.exit(0); } @@ -135,7 +194,7 @@ static void showHelp() { * Prompt the user and collect user inputs * Input collection will be skipped if -y option was passed */ - static void gatherUserInputs() { + void gatherUserInputs() { if (!assumeyes) { try (Scanner scanner = new Scanner(System.in, StandardCharsets.UTF_8)) { @@ -149,7 +208,7 @@ static void gatherUserInputs() { if (!cluster_mode) { System.out.println("Cluster mode requires additional setup of:"); - System.out.println(" - Virtual memory (vm.max_map_count)\n"); + System.out.println(" - Virtual memory (vm.max_map_count)" + System.lineSeparator()); cluster_mode = confirmAction(scanner, "Enable cluster mode?"); } } @@ -165,7 +224,7 @@ static void gatherUserInputs() { * @param message prompt question * @return true or false based on user input */ - static boolean confirmAction(Scanner scanner, String message) { + boolean confirmAction(Scanner scanner, String message) { System.out.print(message + " [y/N] "); String response = scanner.nextLine(); return response.equalsIgnoreCase("yes") || response.equalsIgnoreCase("y"); @@ -174,7 +233,7 @@ static boolean confirmAction(Scanner scanner, String message) { /** * Initialize all class level variables required */ - static void initializeVariables() { + void initializeVariables() { setBaseDir(); setOpenSearchVariables(); setSecurityVariables(); @@ -183,7 +242,7 @@ static void initializeVariables() { /** * Sets the base directory to be used by the script */ - static void setBaseDir() { + void setBaseDir() { File baseDirFile = new File(SCRIPT_DIR).getParentFile().getParentFile().getParentFile(); BASE_DIR = baseDirFile != null ? baseDirFile.getAbsolutePath() : null; @@ -198,50 +257,62 @@ static void setBaseDir() { /** * Sets the variables for items at OpenSearch level */ - static void setOpenSearchVariables() { + void setOpenSearchVariables() { OPENSEARCH_CONF_FILE = BASE_DIR + "config" + File.separator + "opensearch.yml"; OPENSEARCH_BIN_DIR = BASE_DIR + "bin" + File.separator; OPENSEARCH_PLUGINS_DIR = BASE_DIR + "plugins" + File.separator; OPENSEARCH_LIB_PATH = BASE_DIR + "lib" + File.separator; OPENSEARCH_INSTALL_TYPE = determineInstallType(); - if (!(new File(OPENSEARCH_CONF_FILE).exists())) { - System.out.println("Unable to determine OpenSearch config directory. Quit."); + Set errorMessages = validatePaths(); + + if (!errorMessages.isEmpty()) { + errorMessages.forEach(System.out::println); System.exit(-1); } + OPENSEARCH_CONF_DIR = new File(OPENSEARCH_CONF_FILE).getParent(); + OPENSEARCH_CONF_DIR = new File(OPENSEARCH_CONF_DIR).getAbsolutePath() + File.separator; + } + + /** + * Helper method + * Returns a set of error messages for the paths that didn't contain files/directories + * @return a set containing error messages if any, empty otherwise + */ + private Set validatePaths() { + Set errorMessages = new HashSet<>(); + if (!(new File(OPENSEARCH_CONF_FILE).exists())) { + errorMessages.add("Unable to determine OpenSearch config file. Quit."); + } + if (!(new File(OPENSEARCH_BIN_DIR).exists())) { - System.out.println("Unable to determine OpenSearch bin directory. Quit."); - System.exit(-1); + errorMessages.add("Unable to determine OpenSearch bin directory. Quit."); } if (!(new File(OPENSEARCH_PLUGINS_DIR).exists())) { - System.out.println("Unable to determine OpenSearch plugins directory. Quit."); - System.exit(-1); + errorMessages.add("Unable to determine OpenSearch plugins directory. Quit."); } if (!(new File(OPENSEARCH_LIB_PATH).exists())) { - System.out.println("Unable to determine OpenSearch lib directory. Quit."); - System.exit(-1); + errorMessages.add("Unable to determine OpenSearch lib directory. Quit."); } - - OPENSEARCH_CONF_DIR = new File(OPENSEARCH_CONF_FILE).getParent(); - OPENSEARCH_CONF_DIR = new File(OPENSEARCH_CONF_DIR).getAbsolutePath() + File.separator; + return errorMessages; } /** * Returns the installation type based on the underlying operating system * @return will be one of `.zip`, `.tar.gz` or `rpm/deb` */ - static String determineInstallType() { + String determineInstallType() { // windows (.bat execution) if (OS.toLowerCase().contains("win")) { return ".zip"; } // other OS (.sh execution) - if (new File("/usr/share/opensearch").equals(new File(BASE_DIR))) { - OPENSEARCH_CONF_FILE = "/usr/share/opensearch/config/opensearch.yml"; + if (RPM_DEB_OPENSEARCH_HOME.exists() && RPM_DEB_OPENSEARCH_HOME.equals(new File(BASE_DIR))) { + OPENSEARCH_CONF_FILE = RPM_DEB_OPENSEARCH_HOME.getAbsolutePath() + "/config/opensearch.yml"; if (!new File(OPENSEARCH_CONF_FILE).exists()) { OPENSEARCH_CONF_FILE = "/etc/opensearch/opensearch.yml"; } @@ -253,7 +324,7 @@ static String determineInstallType() { /** * Sets the path variables for items at OpenSearch security plugin level */ - static void setSecurityVariables() { + void setSecurityVariables() { if (!(new File(OPENSEARCH_PLUGINS_DIR + "opensearch-security").exists())) { System.out.println("OpenSearch Security plugin not installed. Quit."); System.exit(-1); @@ -261,11 +332,11 @@ static void setSecurityVariables() { // Extract OpenSearch version and Security version File[] opensearchLibFiles = new File(OPENSEARCH_LIB_PATH).listFiles( - pathname -> pathname.getName().startsWith("opensearch-") && pathname.getName().endsWith(".jar") + pathname -> pathname.getName().matches("opensearch-core-(.*).jar") ); if (opensearchLibFiles != null && opensearchLibFiles.length > 0) { - OPENSEARCH_VERSION = opensearchLibFiles[0].getName().replaceAll("opensearch-(.*).jar", "$1"); + OPENSEARCH_VERSION = opensearchLibFiles[0].getName().replaceAll("opensearch-core-(.*).jar", "$1"); } File[] securityFiles = new File(OPENSEARCH_PLUGINS_DIR + "opensearch-security").listFiles( @@ -280,7 +351,7 @@ static void setSecurityVariables() { /** * Prints the initialized variables */ - static void printVariables() { + void printVariables() { System.out.println("OpenSearch install type: " + OPENSEARCH_INSTALL_TYPE + " on " + OS); System.out.println("OpenSearch config dir: " + OPENSEARCH_CONF_DIR); System.out.println("OpenSearch config file: " + OPENSEARCH_CONF_FILE); @@ -294,7 +365,7 @@ static void printVariables() { /** * Prints end of script execution message and creates security admin demo file. */ - static void finishScriptExecution() { + void finishScriptExecution() { System.out.println("### Success"); System.out.println("### Execute this script now on all your nodes and then start all nodes"); @@ -356,7 +427,11 @@ static void finishScriptExecution() { System.out.println("### To use the Security Plugin ConfigurationGUI"); } - System.out.println("### To access your secured cluster open https://: and log in with admin/admin."); + System.out.println( + "### To access your secured cluster open https://: and log in with admin/" + + SecuritySettingsConfigurer.ADMIN_PASSWORD + + "." + ); System.out.println("### (Ignore the SSL certificate warning because we installed self-signed demo certificates)"); } catch (Exception e) { diff --git a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java index a5daa579dd..ac9b0651fd 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java @@ -21,20 +21,62 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.databind.JsonNode; import org.opensearch.common.settings.Settings; +import org.opensearch.security.DefaultObjectMapper; import org.opensearch.security.dlic.rest.validation.PasswordValidator; import org.opensearch.security.dlic.rest.validation.RequestContentValidator; import org.opensearch.security.tools.Hasher; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.Yaml; + import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_PASSWORD_MIN_LENGTH; import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_PASSWORD_VALIDATION_REGEX; -import static org.opensearch.security.user.UserService.generatePassword; /** * This class updates the security related configuration, as needed. */ -public class SecuritySettingsConfigurer extends Installer { +public class SecuritySettingsConfigurer { + + static final List REST_ENABLED_ROLES = List.of("all_access", "security_rest_api_access"); + static final List SYSTEM_INDICES = List.of( + ".plugins-ml-config", + ".plugins-ml-connector", + ".plugins-ml-model-group", + ".plugins-ml-model", + ".plugins-ml-task", + ".plugins-ml-conversation-meta", + ".plugins-ml-conversation-interactions", + ".opendistro-alerting-config", + ".opendistro-alerting-alert*", + ".opendistro-anomaly-results*", + ".opendistro-anomaly-detector*", + ".opendistro-anomaly-checkpoints", + ".opendistro-anomaly-detection-state", + ".opendistro-reports-*", + ".opensearch-notifications-*", + ".opensearch-notebooks", + ".opensearch-observability", + ".ql-datasources", + ".opendistro-asynchronous-search-response*", + ".replication-metadata-store", + ".opensearch-knn-models", + ".geospatial-ip2geo-data*" + ); + static String ADMIN_PASSWORD = ""; + static String ADMIN_USERNAME = "admin"; + + private final Installer installer; + + public SecuritySettingsConfigurer(Installer installer) { + this.installer = installer; + } /** * Configures security related changes to the opensearch configuration @@ -51,12 +93,11 @@ public void configureSecuritySettings() { /** * Replaces the admin password in internal_users.yml with the custom or generated password */ - static void updateAdminPassword() { - String ADMIN_PASSWORD = ""; - String initialAdminPassword = System.getenv("initialAdminPassword"); - String ADMIN_PASSWORD_FILE_PATH = OPENSEARCH_CONF_DIR + "initialAdminPassword.txt"; - String INTERNAL_USERS_FILE_PATH = OPENSEARCH_CONF_DIR + "opensearch-security" + File.separator + "internal_users.yml"; - boolean shouldValidatePassword = environment.equals(ExecutionEnvironment.DEMO); + void updateAdminPassword() { + String initialAdminPassword = System.getenv().get("initialAdminPassword"); + String ADMIN_PASSWORD_FILE_PATH = installer.OPENSEARCH_CONF_DIR + "initialAdminPassword.txt"; + String INTERNAL_USERS_FILE_PATH = installer.OPENSEARCH_CONF_DIR + "opensearch-security" + File.separator + "internal_users.yml"; + boolean shouldValidatePassword = installer.environment.equals(ExecutionEnvironment.DEMO); try { final PasswordValidator passwordValidator = PasswordValidator.of( Settings.builder() @@ -83,19 +124,15 @@ static void updateAdminPassword() { // If script execution environment is set to demo, validate custom password, else if set to test, skip validation if (shouldValidatePassword && !ADMIN_PASSWORD.isEmpty() - && passwordValidator.validate("admin", ADMIN_PASSWORD) != RequestContentValidator.ValidationError.NONE) { + && passwordValidator.validate(ADMIN_USERNAME, ADMIN_PASSWORD) != RequestContentValidator.ValidationError.NONE) { System.out.println("Password " + ADMIN_PASSWORD + " is weak. Please re-try with a stronger password."); System.exit(-1); } - // if ADMIN_PASSWORD is still an empty string, it implies no custom password was provided. We proceed with generating a new one. + // if ADMIN_PASSWORD is still an empty string, it implies no custom password was provided. We exit the setup. if (ADMIN_PASSWORD.isEmpty()) { - System.out.println("No custom admin password found. Generating a new password now."); - // generate a new random password - // We always validate a generated password - while (passwordValidator.validate("admin", ADMIN_PASSWORD) != RequestContentValidator.ValidationError.NONE) { - ADMIN_PASSWORD = generatePassword(); - } + System.out.println("No custom admin password found. Please provide a password."); + System.exit(-1); } // print the password to the logs @@ -106,7 +143,7 @@ static void updateAdminPassword() { writePasswordToInternalUsersFile(ADMIN_PASSWORD, INTERNAL_USERS_FILE_PATH); } catch (IOException e) { - System.out.println("Exception: " + e.getMessage()); + System.out.println("Exception updating the admin password : " + e.getMessage()); System.exit(-1); } } @@ -117,7 +154,7 @@ static void updateAdminPassword() { * @param internalUsersFile the file path string to internal_users.yml file * @throws IOException while reading, writing to files */ - static void writePasswordToInternalUsersFile(String adminPassword, String internalUsersFile) throws IOException { + void writePasswordToInternalUsersFile(String adminPassword, String internalUsersFile) throws IOException { String hashedAdminPassword = Hasher.hash(adminPassword.toCharArray()); if (hashedAdminPassword.isEmpty()) { @@ -151,15 +188,15 @@ static void writePasswordToInternalUsersFile(String adminPassword, String intern /** * Checks if security plugin is already configured. If so, the script execution will not continue. */ - static void checkIfSecurityPluginIsAlreadyConfigured() { + void checkIfSecurityPluginIsAlreadyConfigured() { // Check if the configuration file contains the 'plugins.security' string - if (OPENSEARCH_CONF_FILE != null && new File(OPENSEARCH_CONF_FILE).exists()) { - try (BufferedReader br = new BufferedReader(new FileReader(OPENSEARCH_CONF_FILE, StandardCharsets.UTF_8))) { + if (installer.OPENSEARCH_CONF_FILE != null && new File(installer.OPENSEARCH_CONF_FILE).exists()) { + try (BufferedReader br = new BufferedReader(new FileReader(installer.OPENSEARCH_CONF_FILE, StandardCharsets.UTF_8))) { String line; while ((line = br.readLine()) != null) { if (line.toLowerCase().contains("plugins.security")) { - System.out.println(OPENSEARCH_CONF_FILE + " seems to be already configured for Security. Quit."); - System.exit(skip_updates); + System.out.println(installer.OPENSEARCH_CONF_FILE + " seems to be already configured for Security. Quit."); + System.exit(installer.skip_updates); } } } catch (IOException e) { @@ -175,66 +212,75 @@ static void checkIfSecurityPluginIsAlreadyConfigured() { /** * Update opensearch.yml with security configuration information */ - static void writeSecurityConfigToOpenSearchYML() { - String securityConfig = buildSecurityConfigString(); - - try (FileWriter writer = new FileWriter(OPENSEARCH_CONF_FILE, StandardCharsets.UTF_8, true)) { - writer.write(securityConfig); + void writeSecurityConfigToOpenSearchYML() { + String configHeader = System.lineSeparator() + + System.lineSeparator() + + "######## Start OpenSearch Security Demo Configuration ########" + + System.lineSeparator() + + "# WARNING: revise all the lines below before you go into production" + + System.lineSeparator(); + String configFooter = "######## End OpenSearch Security Demo Configuration ########" + System.lineSeparator(); + + Map securityConfigAsMap = buildSecurityConfigMap(); + + try (FileWriter writer = new FileWriter(installer.OPENSEARCH_CONF_FILE, StandardCharsets.UTF_8, true)) { + writer.write(configHeader); + Yaml yaml = new Yaml(); + DumperOptions options = new DumperOptions(); + options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + String yamlString = yaml.dump(securityConfigAsMap); + writer.write(yamlString); + writer.write(configFooter); } catch (IOException e) { - System.err.println("Exception writing security configuration to opensearch.yml."); + System.err.println("Exception writing security configuration to opensearch.yml : " + e.getMessage()); System.exit(-1); } } /** * Helper method to build security configuration to append to opensearch.yml - * @return the configuration string to be written to opensearch.yml + * @return the configuration map to be written to opensearch.yml */ - static String buildSecurityConfigString() { - StringBuilder securityConfigLines = new StringBuilder(); - - securityConfigLines.append("\n") - .append("######## Start OpenSearch Security Demo Configuration ########\n") - .append("# WARNING: revise all the lines below before you go into production\n") - .append("plugins.security.ssl.transport.pemcert_filepath: esnode.pem\n") - .append("plugins.security.ssl.transport.pemkey_filepath: esnode-key.pem\n") - .append("plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem\n") - .append("plugins.security.ssl.transport.enforce_hostname_verification: false\n") - .append("plugins.security.ssl.http.enabled: true\n") - .append("plugins.security.ssl.http.pemcert_filepath: esnode.pem\n") - .append("plugins.security.ssl.http.pemkey_filepath: esnode-key.pem\n") - .append("plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem\n") - .append("plugins.security.allow_unsafe_democertificates: true\n"); - - if (initsecurity) { - securityConfigLines.append("plugins.security.allow_default_init_securityindex: true\n"); + Map buildSecurityConfigMap() { + Map configMap = new LinkedHashMap<>(); + + configMap.put("plugins.security.ssl.transport.pemcert_filepath", Certificates.NODE_CERT.getFileName()); + configMap.put("plugins.security.ssl.transport.pemkey_filepath", Certificates.NODE_KEY.getFileName()); + configMap.put("plugins.security.ssl.transport.pemtrustedcas_filepath", Certificates.ROOT_CA.getFileName()); + configMap.put("plugins.security.ssl.transport.enforce_hostname_verification", false); + configMap.put("plugins.security.ssl.http.enabled", true); + configMap.put("plugins.security.ssl.http.pemcert_filepath", Certificates.NODE_CERT.getFileName()); + configMap.put("plugins.security.ssl.http.pemkey_filepath", Certificates.NODE_KEY.getFileName()); + configMap.put("plugins.security.ssl.http.pemtrustedcas_filepath", Certificates.ROOT_CA.getFileName()); + configMap.put("plugins.security.allow_unsafe_democertificates", true); + + if (installer.initsecurity) { + configMap.put("plugins.security.allow_default_init_securityindex", true); } - securityConfigLines.append("plugins.security.authcz.admin_dn:\n - CN=kirk,OU=client,O=client,L=test, C=de\n\n"); + configMap.put("plugins.security.authcz.admin_dn", List.of("CN=kirk,OU=client,O=client,L=test,C=de")); - securityConfigLines.append("plugins.security.audit.type: internal_opensearch\n"); - securityConfigLines.append("plugins.security.enable_snapshot_restore_privilege: true\n"); - securityConfigLines.append("plugins.security.check_snapshot_restore_write_privileges: true\n"); - securityConfigLines.append("plugins.security.restapi.roles_enabled: [\"all_access\", \"security_rest_api_access\"]\n"); + configMap.put("plugins.security.audit.type", "internal_opensearch"); + configMap.put("plugins.security.enable_snapshot_restore_privilege", true); + configMap.put("plugins.security.check_snapshot_restore_write_privileges", true); + configMap.put("plugins.security.restapi.roles_enabled", REST_ENABLED_ROLES); - securityConfigLines.append("plugins.security.system_indices.enabled: true\n"); - securityConfigLines.append("plugins.security.system_indices.indices: [").append(SYSTEM_INDICES).append("]\n"); + configMap.put("plugins.security.system_indices.enabled", true); + configMap.put("plugins.security.system_indices.indices", SYSTEM_INDICES); - if (!isNetworkHostAlreadyPresent(OPENSEARCH_CONF_FILE)) { - if (cluster_mode) { - securityConfigLines.append("network.host: 0.0.0.0\n"); - securityConfigLines.append("node.name: smoketestnode\n"); - securityConfigLines.append("cluster.initial_cluster_manager_nodes: smoketestnode\n"); + if (!isNetworkHostAlreadyPresent(installer.OPENSEARCH_CONF_FILE)) { + if (installer.cluster_mode) { + configMap.put("network.host", "0.0.0.0"); + configMap.put("node.name", "smoketestnode"); + configMap.put("cluster.initial_cluster_manager_nodes", "smoketestnode"); } } - if (!isNodeMaxLocalStorageNodesAlreadyPresent(OPENSEARCH_CONF_FILE)) { - securityConfigLines.append("node.max_local_storage_nodes: 3\n"); + if (!isNodeMaxLocalStorageNodesAlreadyPresent(installer.OPENSEARCH_CONF_FILE)) { + configMap.put("node.max_local_storage_nodes", 3); } - securityConfigLines.append("######## End OpenSearch Security Demo Configuration ########\n"); - - return securityConfigLines.toString(); + return configMap; } /** @@ -244,8 +290,8 @@ static String buildSecurityConfigString() { */ static boolean isNetworkHostAlreadyPresent(String filePath) { try { - String searchString = "^network.host"; - return isStringAlreadyPresentInFile(filePath, searchString); + String searchString = "network.host"; + return isKeyPresentInYMLFile(filePath, searchString); } catch (IOException e) { return false; } @@ -258,30 +304,29 @@ static boolean isNetworkHostAlreadyPresent(String filePath) { */ static boolean isNodeMaxLocalStorageNodesAlreadyPresent(String filePath) { try { - String searchString = "^node.max_local_storage_nodes"; - return isStringAlreadyPresentInFile(filePath, searchString); + String searchString = "node.max_local_storage_nodes"; + return isKeyPresentInYMLFile(filePath, searchString); } catch (IOException e) { return false; } } /** - * Checks if given string is already present in the file - * @param filePath path to file in which given string should be searched - * @param searchString the string to be searched for - * @return true if string is present, false otherwise + * Checks if the given key is present in the yml file + * @param filePath path to yml file in which given key should be searched + * @param key the key to be searched for + * @return true if the key is present, false otherwise * @throws IOException if there was exception reading the file */ - static boolean isStringAlreadyPresentInFile(String filePath, String searchString) throws IOException { - try (BufferedReader reader = new BufferedReader(new FileReader(filePath, StandardCharsets.UTF_8))) { - String line; - while ((line = reader.readLine()) != null) { - if (line.matches(searchString)) { - return true; - } - } + static boolean isKeyPresentInYMLFile(String filePath, String key) throws IOException { + JsonNode node; + try { + node = DefaultObjectMapper.YAML_MAPPER.readTree(new File(filePath)); + } catch (IOException e) { + throw new RuntimeException(e); } - return false; + + return node.has(key); } /** @@ -291,33 +336,40 @@ static boolean isStringAlreadyPresentInFile(String filePath, String searchString * @throws IOException if there was error reading/writing the file */ void createSecurityAdminDemoScript(String securityAdminScriptPath, String securityAdminDemoScriptPath) throws IOException { - String[] securityAdminCommands; + String[] securityAdminCommands = getSecurityAdminCommands(securityAdminScriptPath); + // Write securityadmin_demo script + FileWriter writer = new FileWriter(securityAdminDemoScriptPath, StandardCharsets.UTF_8); + for (String command : securityAdminCommands) { + writer.write(command + System.lineSeparator()); + } + writer.close(); + } + + /** + * Return the command to be added to securityadmin_demo script + * @param securityAdminScriptPath the path to securityadmin.(sh|bat) + * @return the command string + */ + String[] getSecurityAdminCommands(String securityAdminScriptPath) { String securityAdminExecutionPath = securityAdminScriptPath + "\" -cd \"" - + OPENSEARCH_CONF_DIR + + installer.OPENSEARCH_CONF_DIR + "opensearch-security\" -icl -key \"" - + OPENSEARCH_CONF_DIR + + installer.OPENSEARCH_CONF_DIR + Certificates.ADMIN_CERT_KEY.getFileName() + "\" -cert \"" - + OPENSEARCH_CONF_DIR + + installer.OPENSEARCH_CONF_DIR + Certificates.ADMIN_CERT.getFileName() + "\" -cacert \"" - + OPENSEARCH_CONF_DIR + + installer.OPENSEARCH_CONF_DIR + Certificates.ROOT_CA.getFileName() + "\" -nhnv"; - if (OS.toLowerCase().contains("win")) { - securityAdminCommands = new String[] { "@echo off", "call \"" + securityAdminExecutionPath }; - } else { - securityAdminCommands = new String[] { "#!/bin/bash", "sudo" + " \"" + securityAdminExecutionPath }; + if (installer.OS.toLowerCase().contains("win")) { + return new String[] { "@echo off", "call \"" + securityAdminExecutionPath }; } - // Write securityadmin_demo script - FileWriter writer = new FileWriter(securityAdminDemoScriptPath, StandardCharsets.UTF_8); - for (String command : securityAdminCommands) { - writer.write(command + "\n"); - } - writer.close(); + return new String[] { "#!/bin/bash", "sudo" + " \"" + securityAdminExecutionPath }; } } diff --git a/tools/install_demo_configuration.bat b/tools/install_demo_configuration.bat index 5767166b26..5cf4d715fa 100755 --- a/tools/install_demo_configuration.bat +++ b/tools/install_demo_configuration.bat @@ -1,14 +1,29 @@ @echo off set DIR=%~dp0 -if defined OPENSEARCH_JAVA_HOME ( - set BIN_PATH="%OPENSEARCH_JAVA_HOME%\bin\java.exe" -) else if defined JAVA_HOME ( - set BIN_PATH="%JAVA_HOME%\bin\java.exe" +set CUR_DIR=%DIR% + +rem set opensearch home for instances when using bundled jdk +if not defined OPENSEARCH_HOME ( + for %%I in ("%DIR%..\..\..") do set "OPENSEARCH_HOME=%%~dpfI" +) +cd %CUR_DIR% + +if not "%OPENSEARCH_JAVA_HOME%" == "" ( + set "JAVA=%OPENSEARCH_JAVA_HOME%\bin\java.exe" + set JAVA_TYPE=OPENSEARCH_JAVA_HOME +) else if not "%JAVA_HOME%" == "" ( + set "JAVA=%JAVA_HOME%\bin\java.exe" + set JAVA_TYPE=JAVA_HOME ) else ( - echo Unable to find java runtime - echo OPENSEARCH_JAVA_HOME or JAVA_HOME must be defined + set "JAVA=%OPENSEARCH_HOME%\jdk\bin\java.exe" + set "JAVA_HOME=%OPENSEARCH_HOME%\jdk" + set JAVA_TYPE=bundled jdk +) + +if not exist "%JAVA%" ( + echo "could not find java in %JAVA_TYPE% at %JAVA%" >&2 exit /b 1 ) -%BIN_PATH% -Dorg.apache.logging.log4j.simplelog.StatusLogger.level=OFF -cp "%DIR%\..\*;%DIR%\..\..\..\lib\*;%DIR%\..\deps\*" org.opensearch.security.tools.democonfig.Installer %DIR% %* 2> nul \ No newline at end of file +"%JAVA%" -Dorg.apache.logging.log4j.simplelog.StatusLogger.level=OFF -cp "%DIR%\..\*;%DIR%\..\..\..\lib\*;%DIR%\..\deps\*" org.opensearch.security.tools.democonfig.Installer %DIR% %* 2> nul diff --git a/tools/install_demo_configuration.sh b/tools/install_demo_configuration.sh index 7835f7c675..d3a3ae8f75 100755 --- a/tools/install_demo_configuration.sh +++ b/tools/install_demo_configuration.sh @@ -1,6 +1,15 @@ #!/bin/bash #install_demo_configuration.sh [-y] +UNAME=$(uname -s) +if [ "$UNAME" = "FreeBSD" ]; then + OS="freebsd" +elif [ "$UNAME" = "Darwin" ]; then + OS="darwin" +else + OS="other" +fi + SCRIPT_PATH="${BASH_SOURCE[0]}" if ! [ -x "$(command -v realpath)" ]; then if [ -L "$SCRIPT_PATH" ]; then @@ -16,17 +25,40 @@ else DIR="$( cd "$( dirname "$(realpath "$SCRIPT_PATH")" )" && pwd -P)" fi -BIN_PATH="java" -# now set the path to java: first OPENSEARCH_JAVA_HOME, then JAVA_HOME +if [ -z "$OPENSEARCH_HOME" ]; then + # move to opensearch root folder and set the variable + OPENSEARCH_HOME=`cd "$DIR/../../.."; pwd` +fi + +# now set the path to java: OPENSEARCH_JAVA_HOME -> JAVA_HOME -> bundled JRE -> bundled JDK if [ -n "$OPENSEARCH_JAVA_HOME" ]; then - BIN_PATH="$OPENSEARCH_JAVA_HOME/bin/java" + JAVA="$OPENSEARCH_JAVA_HOME/bin/java" + JAVA_TYPE="OPENSEARCH_JAVA_HOME" elif [ -n "$JAVA_HOME" ]; then - BIN_PATH="$JAVA_HOME/bin/java" + JAVA="$JAVA_HOME/bin/java" + JAVA_TYPE="JAVA_HOME" else - echo "Unable to find java runtime" - echo "OPENSEARCH_JAVA_HOME or JAVA_HOME must be defined" + if [ "$OS" = "darwin" ]; then + # macOS bundled Java + JAVA="$OPENSEARCH_HOME/jdk.app/Contents/Home/bin/java" + JAVA_TYPE="bundled jdk" + elif [ "$OS" = "freebsd" ]; then + # using FreeBSD default java from ports if JAVA_HOME is not set + JAVA="/usr/local/bin/java" + JAVA_TYPE="bundled jdk" + elif [ -d "$OPENSEARCH_HOME/jre" ]; then + JAVA="$OPENSEARCH_HOME/jre/bin/java" + JAVA_TYPE="bundled jre" + else + JAVA="$OPENSEARCH_HOME/jdk/bin/java" + JAVA_TYPE="bundled jdk" + fi +fi + +if [ ! -x "$JAVA" ]; then + echo "could not find java in $JAVA_TYPE at $JAVA" >&2 exit 1 fi -"$BIN_PATH" $JAVA_OPTS -Dorg.apache.logging.log4j.simplelog.StatusLogger.level=OFF -cp "$DIR/../*:$DIR/../../../lib/*:$DIR/../deps/*" org.opensearch.security.tools.democonfig.Installer "$DIR" "$@" 2>/dev/null +"$JAVA" -Dorg.apache.logging.log4j.simplelog.StatusLogger.level=OFF -cp "$DIR/../*:$DIR/../../../lib/*:$DIR/../deps/*" org.opensearch.security.tools.democonfig.Installer "$DIR" "$@" 2>/dev/null From 676dacef288366084a14ff582d589547840c57cc Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 12 Dec 2023 16:35:25 -0600 Subject: [PATCH 046/204] Add do not fail on forbidden test cases around the stats API (#3825) Signed-off-by: Peter Nied --- .../security/DefaultConfigurationTests.java | 2 +- .../security/DoNotFailOnForbiddenTests.java | 46 ++++++++++++++++++- .../security/PointInTimeOperationTest.java | 4 +- .../security/SecurityConfigurationTests.java | 10 ++-- .../org/opensearch/security/SslOnlyTests.java | 2 +- .../http/CommonProxyAuthenticationTests.java | 2 +- .../http/ExtendedProxyAuthenticationTest.java | 4 +- .../framework/cluster/TestRestClient.java | 15 +----- 8 files changed, 58 insertions(+), 27 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/DefaultConfigurationTests.java b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationTests.java index 043d3908e9..8bb5b96145 100644 --- a/src/integrationTest/java/org/opensearch/security/DefaultConfigurationTests.java +++ b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationTests.java @@ -69,7 +69,7 @@ public void shouldLoadDefaultConfiguration() { } try (TestRestClient client = cluster.getRestClient(ADMIN_USER_NAME, DEFAULT_PASSWORD)) { client.confirmCorrectCredentials(ADMIN_USER_NAME); - HttpResponse response = client.get("/_plugins/_security/api/internalusers"); + HttpResponse response = client.get("_plugins/_security/api/internalusers"); response.assertStatusCode(200); Map users = response.getBodyAs(Map.class); assertThat(users, allOf(aMapWithSize(3), hasKey(ADMIN_USER_NAME), hasKey(NEW_USER), hasKey(LIMITED_USER))); diff --git a/src/integrationTest/java/org/opensearch/security/DoNotFailOnForbiddenTests.java b/src/integrationTest/java/org/opensearch/security/DoNotFailOnForbiddenTests.java index 207564daaa..3a50a4b1f6 100644 --- a/src/integrationTest/java/org/opensearch/security/DoNotFailOnForbiddenTests.java +++ b/src/integrationTest/java/org/opensearch/security/DoNotFailOnForbiddenTests.java @@ -39,10 +39,14 @@ import org.opensearch.client.Response; import org.opensearch.client.RestHighLevelClient; import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.TestSecurityConfig.Role; import org.opensearch.test.framework.TestSecurityConfig.User; import org.opensearch.test.framework.cluster.ClusterManager; import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; +import static org.apache.http.HttpStatus.SC_CREATED; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.allOf; @@ -51,6 +55,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; import static org.opensearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions.Type.ADD; import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; @@ -127,13 +132,17 @@ public class DoNotFailOnForbiddenTests { .on(MARVELOUS_SONGS) ); + private static final User STATS_USER = new User("stats_user").roles( + new Role("test_role").clusterPermissions("cluster:monitor/*").indexPermissions("read", "indices:monitor/*").on("hi1") + ); + private static final String BOTH_INDEX_ALIAS = "both-indices"; private static final String FORBIDDEN_INDEX_ALIAS = "forbidden-index"; @ClassRule public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS) .authc(AUTHC_HTTPBASIC_INTERNAL) - .users(ADMIN_USER, LIMITED_USER) + .users(ADMIN_USER, LIMITED_USER, STATS_USER) .anonymousAuth(false) .doNotFailOnForbidden(true) .build(); @@ -434,4 +443,39 @@ public void shouldPerformCatIndices_positive() throws IOException { } } + @Test + public void checkStatsApi() { + // As admin creates 2 documents in different indices, can find both indices in search, cat indice & stats APIs + try (final TestRestClient client = cluster.getRestClient(ADMIN_USER.getName(), ADMIN_USER.getPassword())) { + final HttpResponse createDoc1 = client.postJson("hi1/_doc?refresh=true", "{\"hi\":\"Hello1\"}"); + createDoc1.assertStatusCode(SC_CREATED); + final HttpResponse createDoc2 = client.postJson("hi2/_doc?refresh=true", "{\"hi\":\"Hello2\"}"); + createDoc2.assertStatusCode(SC_CREATED); + + final HttpResponse search = client.postJson("hi*/_search", "{}"); + assertThat("Unexpected document results in search:" + search.getBody(), search.getBody(), containsString("2")); + + final HttpResponse catIndices = client.get("_cat/indices"); + assertThat("Expected cat indices: " + catIndices.getBody(), catIndices.getBody(), containsString("hi1")); + assertThat("Expected cat indices: " + catIndices.getBody(), catIndices.getBody(), containsString("hi2")); + + final HttpResponse stats = client.get("hi*/_stats?filter_path=indices.*.uuid"); + assertThat("Expected stats indices: " + stats.getBody(), stats.getBody(), containsString("hi1")); + assertThat("Expected stats indices: " + stats.getBody(), stats.getBody(), containsString("hi2")); + } + + // As user who can only see the index "hi1" make sure that DNFOF is filtering out "hi2" + try (final TestRestClient client = cluster.getRestClient(STATS_USER.getName(), STATS_USER.getPassword())) { + final HttpResponse search = client.postJson("hi*/_search", "{}"); + assertThat("Unexpected document results in search:" + search.getBody(), search.getBody(), containsString("1")); + + final HttpResponse catIndices = client.get("_cat/indices"); + assertThat("Expected cat indices: " + catIndices.getBody(), catIndices.getBody(), containsString("hi1")); + assertThat("Unexpected cat indices: " + catIndices.getBody(), catIndices.getBody(), not(containsString("hi2"))); + + final HttpResponse stats = client.get("hi*/_stats?filter_path=indices.*.uuid"); + assertThat("Expected stats indices: " + stats.getBody(), stats.getBody(), containsString("hi1")); + assertThat("Unexpected stats indices: " + stats.getBody(), stats.getBody(), not(containsString("hi2"))); + } + } } diff --git a/src/integrationTest/java/org/opensearch/security/PointInTimeOperationTest.java b/src/integrationTest/java/org/opensearch/security/PointInTimeOperationTest.java index 3d634c4a5d..ce934a8e16 100644 --- a/src/integrationTest/java/org/opensearch/security/PointInTimeOperationTest.java +++ b/src/integrationTest/java/org/opensearch/security/PointInTimeOperationTest.java @@ -381,7 +381,7 @@ public void listPitSegmentsCreatedWithIndexAlias_negative() throws IOException { @Test public void listAllPitSegments_positive() { try (TestRestClient restClient = cluster.getRestClient(POINT_IN_TIME_USER)) { - HttpResponse response = restClient.get("/_cat/pit_segments/_all"); + HttpResponse response = restClient.get("_cat/pit_segments/_all"); response.assertStatusCode(OK.getStatus()); } @@ -390,7 +390,7 @@ public void listAllPitSegments_positive() { @Test public void listAllPitSegments_negative() { try (TestRestClient restClient = cluster.getRestClient(LIMITED_POINT_IN_TIME_USER)) { - HttpResponse response = restClient.get("/_cat/pit_segments/_all"); + HttpResponse response = restClient.get("_cat/pit_segments/_all"); response.assertStatusCode(FORBIDDEN.getStatus()); } diff --git a/src/integrationTest/java/org/opensearch/security/SecurityConfigurationTests.java b/src/integrationTest/java/org/opensearch/security/SecurityConfigurationTests.java index cc95f191f7..76ea02494e 100644 --- a/src/integrationTest/java/org/opensearch/security/SecurityConfigurationTests.java +++ b/src/integrationTest/java/org/opensearch/security/SecurityConfigurationTests.java @@ -119,7 +119,7 @@ public void shouldCreateUserViaRestApi_failure() { @Test public void shouldAuthenticateAsAdminWithCertificate_positive() { try (TestRestClient client = cluster.getRestClient(cluster.getAdminCertificate())) { - HttpResponse httpResponse = client.get("/_plugins/_security/whoami"); + HttpResponse httpResponse = client.get("_plugins/_security/whoami"); httpResponse.assertStatusCode(200); assertThat(httpResponse.getTextFromJsonBody("/is_admin"), equalTo("true")); @@ -130,7 +130,7 @@ public void shouldAuthenticateAsAdminWithCertificate_positive() { public void shouldAuthenticateAsAdminWithCertificate_negativeSelfSignedCertificate() { TestCertificates testCertificates = cluster.getTestCertificates(); try (TestRestClient client = cluster.getRestClient(testCertificates.createSelfSignedCertificate("CN=bond"))) { - HttpResponse httpResponse = client.get("/_plugins/_security/whoami"); + HttpResponse httpResponse = client.get("_plugins/_security/whoami"); httpResponse.assertStatusCode(200); assertThat(httpResponse.getTextFromJsonBody("/is_admin"), equalTo("false")); @@ -141,7 +141,7 @@ public void shouldAuthenticateAsAdminWithCertificate_negativeSelfSignedCertifica public void shouldAuthenticateAsAdminWithCertificate_negativeIncorrectDn() { TestCertificates testCertificates = cluster.getTestCertificates(); try (TestRestClient client = cluster.getRestClient(testCertificates.createAdminCertificate("CN=non_admin"))) { - HttpResponse httpResponse = client.get("/_plugins/_security/whoami"); + HttpResponse httpResponse = client.get("_plugins/_security/whoami"); httpResponse.assertStatusCode(200); assertThat(httpResponse.getTextFromJsonBody("/is_admin"), equalTo("false")); @@ -199,7 +199,7 @@ public void shouldStillWorkAfterUpdateOfSecurityConfig() { @Test public void shouldAccessIndexWithPlaceholder_positive() { try (TestRestClient client = cluster.getRestClient(LIMITED_USER)) { - HttpResponse httpResponse = client.get("/" + LIMITED_USER_INDEX + "/_doc/" + ID_1); + HttpResponse httpResponse = client.get(LIMITED_USER_INDEX + "/_doc/" + ID_1); httpResponse.assertStatusCode(200); } @@ -208,7 +208,7 @@ public void shouldAccessIndexWithPlaceholder_positive() { @Test public void shouldAccessIndexWithPlaceholder_negative() { try (TestRestClient client = cluster.getRestClient(LIMITED_USER)) { - HttpResponse httpResponse = client.get("/" + PROHIBITED_INDEX + "/_doc/" + ID_2); + HttpResponse httpResponse = client.get(PROHIBITED_INDEX + "/_doc/" + ID_2); httpResponse.assertStatusCode(403); } diff --git a/src/integrationTest/java/org/opensearch/security/SslOnlyTests.java b/src/integrationTest/java/org/opensearch/security/SslOnlyTests.java index 25feffb2b4..2ea5b4c0b2 100644 --- a/src/integrationTest/java/org/opensearch/security/SslOnlyTests.java +++ b/src/integrationTest/java/org/opensearch/security/SslOnlyTests.java @@ -59,7 +59,7 @@ public void shouldGetIndicesWithoutAuthentication() { try (TestRestClient client = cluster.getRestClient()) { // request does not contains credential - HttpResponse response = client.get("/_cat/indices"); + HttpResponse response = client.get("_cat/indices"); // successful response is returned because the security plugin in SSL only mode // does not perform authentication and authorization diff --git a/src/integrationTest/java/org/opensearch/security/http/CommonProxyAuthenticationTests.java b/src/integrationTest/java/org/opensearch/security/http/CommonProxyAuthenticationTests.java index 49ded4f2a9..d98acf8895 100644 --- a/src/integrationTest/java/org/opensearch/security/http/CommonProxyAuthenticationTests.java +++ b/src/integrationTest/java/org/opensearch/security/http/CommonProxyAuthenticationTests.java @@ -31,7 +31,7 @@ */ abstract class CommonProxyAuthenticationTests { - protected static final String RESOURCE_AUTH_INFO = "/_opendistro/_security/authinfo"; + protected static final String RESOURCE_AUTH_INFO = "_opendistro/_security/authinfo"; protected static final TestSecurityConfig.User USER_ADMIN = new TestSecurityConfig.User("admin").roles(ALL_ACCESS); protected static final String ATTRIBUTE_DEPARTMENT = "department"; diff --git a/src/integrationTest/java/org/opensearch/security/http/ExtendedProxyAuthenticationTest.java b/src/integrationTest/java/org/opensearch/security/http/ExtendedProxyAuthenticationTest.java index 6fcc7eac83..7c361828d6 100644 --- a/src/integrationTest/java/org/opensearch/security/http/ExtendedProxyAuthenticationTest.java +++ b/src/integrationTest/java/org/opensearch/security/http/ExtendedProxyAuthenticationTest.java @@ -230,7 +230,7 @@ public void shouldRetrieveUserRolesAndAttributesSoThatAccessToPersonalIndexIsPos .header(HEADER_DEPARTMENT, DEPARTMENT_BRIDGE); try (TestRestClient client = cluster.createGenericClientRestClient(testRestClientConfiguration)) { - HttpResponse response = client.get("/" + PERSONAL_INDEX_NAME_SPOCK + "/_search"); + HttpResponse response = client.get(PERSONAL_INDEX_NAME_SPOCK + "/_search"); response.assertStatusCode(200); assertThat(response.getLongFromJsonBody(POINTER_TOTAL_HITS), equalTo(1L)); @@ -251,7 +251,7 @@ public void shouldRetrieveUserRolesAndAttributesSoThatAccessToPersonalIndexIsPos .header(HEADER_DEPARTMENT, DEPARTMENT_BRIDGE); try (TestRestClient client = cluster.createGenericClientRestClient(testRestClientConfiguration)) { - HttpResponse response = client.get("/" + PERSONAL_INDEX_NAME_KIRK + "/_search"); + HttpResponse response = client.get(PERSONAL_INDEX_NAME_KIRK + "/_search"); response.assertStatusCode(403); } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java index 34eb8983cd..c2e01bb338 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -31,8 +31,6 @@ import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; -import java.net.URI; -import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -61,10 +59,8 @@ import org.apache.hc.client5.http.routing.HttpRoutePlanner; import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpEntity; -import org.apache.hc.core5.http.NameValuePair; import org.apache.hc.core5.http.io.entity.StringEntity; import org.apache.hc.core5.http.message.BasicHeader; -import org.apache.hc.core5.net.URIBuilder; import org.apache.http.HttpHeaders; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -109,17 +105,8 @@ public TestRestClient(InetSocketAddress nodeHttpAddress, List
headers, S this.sourceInetAddress = sourceInetAddress; } - public HttpResponse get(String path, List queryParameters, Header... headers) { - try { - URI uri = new URIBuilder(getHttpServerUri()).setPath(path).addParameters(queryParameters).build(); - return executeRequest(new HttpGet(uri), headers); - } catch (URISyntaxException ex) { - throw new RuntimeException("Incorrect URI syntax", ex); - } - } - public HttpResponse get(String path, Header... headers) { - return get(path, Collections.emptyList(), headers); + return executeRequest(new HttpGet(getHttpServerUri() + "/" + path), headers); } public HttpResponse getAuthInfo(Header... headers) { From c4c3c4c49f4ef9e52abc1c42f025c1ede83912a7 Mon Sep 17 00:00:00 2001 From: David Osorno <48450162+davidosorno@users.noreply.github.com> Date: Tue, 12 Dec 2023 14:58:33 -0800 Subject: [PATCH 047/204] Enhance SelfRefreshingKeySet test coverage (#3509) Signed-off-by: David Osorno Signed-off-by: David Osorno <48450162+davidosorno@users.noreply.github.com> Co-authored-by: Craig Perkins --- .../ConfigurationRepository.java | 4 + .../keybyoidc/SelfRefreshingKeySetTest.java | 135 ++++++++---------- .../ConfigurationRepositoryTest.java | 114 +++++++++++++++ .../SecurityDynamicConfigurationTest.java | 52 +++++++ .../SecuritySSLNettyTransportTests.java | 117 +++++++++++++++ 5 files changed, 350 insertions(+), 72 deletions(-) create mode 100644 src/test/java/org/opensearch/security/configuration/ConfigurationRepositoryTest.java create mode 100644 src/test/java/org/opensearch/security/securityconf/impl/SecurityDynamicConfigurationTest.java create mode 100644 src/test/java/org/opensearch/security/ssl/transport/SecuritySSLNettyTransportTests.java diff --git a/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java b/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java index 81e9f47370..e7f375bef4 100644 --- a/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java +++ b/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java @@ -488,4 +488,8 @@ private static String formatDate(long date) { public static int getDefaultConfigVersion() { return ConfigurationRepository.DEFAULT_CONFIG_VERSION; } + + public AtomicBoolean getInstallDefaultConfig() { + return installDefaultConfig; + } } diff --git a/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/SelfRefreshingKeySetTest.java b/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/SelfRefreshingKeySetTest.java index bab23c5fc4..ef53ba5ec0 100644 --- a/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/SelfRefreshingKeySetTest.java +++ b/src/test/java/com/amazon/dlic/auth/http/jwt/keybyoidc/SelfRefreshingKeySetTest.java @@ -11,66 +11,94 @@ package com.amazon.dlic.auth.http.jwt.keybyoidc; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; -import com.nimbusds.jose.jwk.JWK; import com.nimbusds.jose.jwk.JWKSet; import com.nimbusds.jose.jwk.OctetSequenceKey; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.fail; + public class SelfRefreshingKeySetTest { - @Test - public void basicTest() throws AuthenticatorUnavailableException, BadCredentialsException { - SelfRefreshingKeySet selfRefreshingKeySet = new SelfRefreshingKeySet(new MockKeySetProvider()); - - OctetSequenceKey key1 = (OctetSequenceKey) selfRefreshingKeySet.getKey("kid/a"); - Assert.assertEquals(TestJwk.OCT_1_K, key1.getKeyValue().decodeToString()); - Assert.assertEquals(1, selfRefreshingKeySet.getRefreshCount()); - - OctetSequenceKey key2 = (OctetSequenceKey) selfRefreshingKeySet.getKey("kid/b"); - Assert.assertEquals(TestJwk.OCT_2_K, key2.getKeyValue().decodeToString()); - Assert.assertEquals(1, selfRefreshingKeySet.getRefreshCount()); - - try { - selfRefreshingKeySet.getKey("kid/X"); - Assert.fail("Expected a BadCredentialsException"); - } catch (BadCredentialsException e) { - Assert.assertEquals(2, selfRefreshingKeySet.getRefreshCount()); - } + private SelfRefreshingKeySet selfRefreshingKeySet; + private String keyForKidA; + private String keyForKidB; + private int numThreads = 10; + @Before + public void setUp() throws AuthenticatorUnavailableException, BadCredentialsException { + selfRefreshingKeySet = new SelfRefreshingKeySet(new MockKeySetProvider()); + keyForKidA = TestJwk.OCT_1_K; + keyForKidB = TestJwk.OCT_2_K; } - @Test(timeout = 10000) - public void twoThreadedTest() throws Exception { - BlockingMockKeySetProvider provider = new BlockingMockKeySetProvider(); + @Test + public void getKey_withKidShouldReturnValidKey() throws AuthenticatorUnavailableException, BadCredentialsException { - final SelfRefreshingKeySet selfRefreshingKeySet = new SelfRefreshingKeySet(provider); + OctetSequenceKey key = (OctetSequenceKey) selfRefreshingKeySet.getKey("kid/a"); + assertThat(keyForKidA, is(equalTo(key.getKeyValue().decodeToString()))); + } - ExecutorService executorService = Executors.newCachedThreadPool(); + @Test + public void getKey__withNullOrInvalidKidShouldThrowAnException() throws AuthenticatorUnavailableException, BadCredentialsException { - Future f1 = executorService.submit(() -> selfRefreshingKeySet.getKey("kid/a")); + Assert.assertThrows(AuthenticatorUnavailableException.class, () -> selfRefreshingKeySet.getKey(null)); + Assert.assertThrows(BadCredentialsException.class, () -> selfRefreshingKeySet.getKey("kid/X")); + } - provider.waitForCalled(); + @Test + public void getKeyAfterRefresh_withKidShouldReturnKey() throws AuthenticatorUnavailableException, BadCredentialsException { - Future f2 = executorService.submit(() -> selfRefreshingKeySet.getKey("kid/b")); + OctetSequenceKey key = (OctetSequenceKey) selfRefreshingKeySet.getKeyAfterRefresh("kid/b"); + assertThat(keyForKidB, is(equalTo(key.getKeyValue().decodeToString()))); + } - while (selfRefreshingKeySet.getQueuedGetCount() == 0) { - Thread.sleep(10); + @Test + public void getKeyAfterRefresh_withMultipleCallsShouldIncreaseQueueCount() throws InterruptedException, ExecutionException { + ExecutorService executor = Executors.newFixedThreadPool(numThreads); + String[] keys = new String[] { "kid/a", "kid/b" }; + for (int i = 0; i < numThreads; i++) { + // Using executor to make multiple asynchronous calls to method getKeyAfterRefresh, so queuedGetCount gets increased. + // Without executor block, getKeyAfterRefresh method would be called once on each iteration in the main thread and wait for the + // task to complete before continuing the loop, so queuedGetCount would not have pending tasks. + executor.execute(() -> { + try { + int indexKey = (int) (Math.random() * 2); + String keyToCompare = indexKey == 0 ? keyForKidA : keyForKidB; + OctetSequenceKey key = (OctetSequenceKey) selfRefreshingKeySet.getKeyAfterRefresh(keys[indexKey]); + + assertThat(key, is(notNullValue())); + assertThat(keyToCompare, is(equalTo(key.getKeyValue().decodeToString()))); + } catch (AuthenticatorUnavailableException | BadCredentialsException e) { + fail("No exception was expected but found: " + e.getMessage()); + } + }); } - provider.unblock(); + executor.shutdown(); + executor.awaitTermination(1, TimeUnit.SECONDS); - Assert.assertEquals(TestJwk.OCT_1_K, ((OctetSequenceKey) f1.get()).getKeyValue().decodeToString()); - Assert.assertEquals(TestJwk.OCT_2_K, ((OctetSequenceKey) f2.get()).getKeyValue().decodeToString()); + assertThat((int) selfRefreshingKeySet.getRefreshCount(), is(greaterThan(0))); + assertThat((int) selfRefreshingKeySet.getQueuedGetCount(), is(greaterThan(0))); + } - Assert.assertEquals(1, selfRefreshingKeySet.getRefreshCount()); - Assert.assertEquals(1, selfRefreshingKeySet.getQueuedGetCount()); + @Test + public void getKeyAfterRefresh_withNullOrInvalidKidShouldThrowBadCredentialsException() { + Assert.assertThrows(BadCredentialsException.class, () -> selfRefreshingKeySet.getKeyAfterRefresh(null)); + Assert.assertThrows(BadCredentialsException.class, () -> selfRefreshingKeySet.getKeyAfterRefresh("kid/X")); } static class MockKeySetProvider implements KeySetProvider { @@ -79,42 +107,5 @@ static class MockKeySetProvider implements KeySetProvider { public JWKSet get() throws AuthenticatorUnavailableException { return TestJwk.OCT_1_2_3; } - - } - - static class BlockingMockKeySetProvider extends MockKeySetProvider { - private boolean blocked = true; - private boolean called = false; - - @Override - public synchronized JWKSet get() throws AuthenticatorUnavailableException { - - called = true; - notifyAll(); - - waitForUnblock(); - - return super.get(); - } - - public synchronized void unblock() { - blocked = false; - notifyAll(); - } - - public synchronized void waitForCalled() throws InterruptedException { - while (!called) { - wait(); - } - } - - private synchronized void waitForUnblock() { - while (blocked) { - try { - wait(); - } catch (InterruptedException e) {} - - } - } } } diff --git a/src/test/java/org/opensearch/security/configuration/ConfigurationRepositoryTest.java b/src/test/java/org/opensearch/security/configuration/ConfigurationRepositoryTest.java new file mode 100644 index 0000000000..c8f41433e0 --- /dev/null +++ b/src/test/java/org/opensearch/security/configuration/ConfigurationRepositoryTest.java @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.configuration; + +import java.io.IOException; +import java.nio.file.Path; + +import org.junit.Before; +import org.junit.Test; + +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.security.auditlog.AuditLog; +import org.opensearch.security.securityconf.impl.CType; +import org.opensearch.security.securityconf.impl.SecurityDynamicConfiguration; +import org.opensearch.security.support.ConfigConstants; +import org.opensearch.security.transport.SecurityInterceptorTests; +import org.opensearch.threadpool.ThreadPool; + +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; + +public class ConfigurationRepositoryTest { + + @Mock + private Client localClient; + @Mock + private AuditLog auditLog; + @Mock + private Path path; + @Mock + private ClusterService clusterService; + + private ThreadPool threadPool; + + @Before + public void setUp() { + MockitoAnnotations.openMocks(this); + + Settings settings = Settings.builder() + .put("node.name", SecurityInterceptorTests.class.getSimpleName()) + .put("request.headers.default", "1") + .build(); + + threadPool = new ThreadPool(settings); + } + + private ConfigurationRepository createConfigurationRepository(Settings settings) { + + return ConfigurationRepository.create(settings, path, threadPool, localClient, clusterService, auditLog); + } + + @Test + public void create_shouldReturnConfigurationRepository() { + ConfigurationRepository configRepository = createConfigurationRepository(Settings.EMPTY); + + assertThat(configRepository, is(notNullValue())); + assertThat(configRepository, instanceOf(ConfigurationRepository.class)); + } + + @Test + public void initOnNodeStart_withSecurityIndexCreationEnabledShouldSetInstallDefaultConfigTrue() { + Settings settings = Settings.builder().put(ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, true).build(); + + ConfigurationRepository configRepository = createConfigurationRepository(settings); + + configRepository.initOnNodeStart(); + + assertThat(configRepository.getInstallDefaultConfig().get(), is(true)); + } + + @Test + public void initOnNodeStart_withSecurityIndexNotCreatedShouldNotSetInstallDefaultConfig() { + Settings settings = Settings.builder().put(ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, false).build(); + + ConfigurationRepository configRepository = createConfigurationRepository(settings); + + configRepository.initOnNodeStart(); + + assertThat(configRepository.getInstallDefaultConfig().get(), is(false)); + } + + @Test + public void getConfiguration_withInvalidConfigurationShouldReturnNewEmptyConfigurationObject() throws IOException { + ConfigurationRepository configRepository = createConfigurationRepository(Settings.EMPTY); + + SecurityDynamicConfiguration config = configRepository.getConfiguration(CType.CONFIG); + SecurityDynamicConfiguration emptyConfig = SecurityDynamicConfiguration.empty(); + + assertThat(config, is(instanceOf(SecurityDynamicConfiguration.class))); + assertThat(config.getCEntries().size(), is(equalTo(0))); + assertThat(config.getVersion(), is(equalTo(emptyConfig.getVersion()))); + assertThat(config.getCType(), is(equalTo(emptyConfig.getCType()))); + assertThat(config.getSeqNo(), is(equalTo(emptyConfig.getSeqNo()))); + assertThat(config, is(not(equalTo(emptyConfig)))); + } +} diff --git a/src/test/java/org/opensearch/security/securityconf/impl/SecurityDynamicConfigurationTest.java b/src/test/java/org/opensearch/security/securityconf/impl/SecurityDynamicConfigurationTest.java new file mode 100644 index 0000000000..c554784581 --- /dev/null +++ b/src/test/java/org/opensearch/security/securityconf/impl/SecurityDynamicConfigurationTest.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.securityconf.impl; + +import java.io.IOException; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.Before; +import org.junit.Test; + +import org.opensearch.security.DefaultObjectMapper; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; + +public class SecurityDynamicConfigurationTest { + + private SecurityDynamicConfiguration securityDynamicConfiguration; + private ObjectMapper objectMapper = DefaultObjectMapper.objectMapper; + private ObjectNode objectNode = objectMapper.createObjectNode(); + + @Before + public void setUp() throws JsonProcessingException, IOException { + objectNode.set("_meta", objectMapper.createObjectNode().put("type", CType.ROLES.toLCString()).put("config_version", 2)); + securityDynamicConfiguration = SecurityDynamicConfiguration.fromJson( + objectMapper.writeValueAsString(objectNode), + CType.ROLES, + 2, + 1, + 1 + ); + } + + @Test + public void deepClone_shouldReturnNewObject() { + SecurityDynamicConfiguration securityDeepClone = securityDynamicConfiguration.deepClone(); + assertThat(securityDeepClone, is(not(equalTo(securityDynamicConfiguration)))); + } +} diff --git a/src/test/java/org/opensearch/security/ssl/transport/SecuritySSLNettyTransportTests.java b/src/test/java/org/opensearch/security/ssl/transport/SecuritySSLNettyTransportTests.java new file mode 100644 index 0000000000..27705988d8 --- /dev/null +++ b/src/test/java/org/opensearch/security/ssl/transport/SecuritySSLNettyTransportTests.java @@ -0,0 +1,117 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.ssl.transport; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import org.opensearch.Version; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.common.network.NetworkService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.PageCacheRecycler; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.indices.breaker.CircuitBreakerService; +import org.opensearch.security.ssl.SecurityKeyStore; +import org.opensearch.security.ssl.SslExceptionHandler; +import org.opensearch.security.ssl.transport.SecuritySSLNettyTransport.SSLClientChannelInitializer; +import org.opensearch.security.ssl.transport.SecuritySSLNettyTransport.SSLServerChannelInitializer; +import org.opensearch.telemetry.tracing.Tracer; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.SharedGroupFactory; + +import io.netty.channel.ChannelHandler; +import org.mockito.Mock; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +public class SecuritySSLNettyTransportTests { + + @Mock + private Version version; + @Mock + private ThreadPool threadPool; + @Mock + private NetworkService networkService; + @Mock + private PageCacheRecycler pageCacheRecycler; + @Mock + private NamedWriteableRegistry namedWriteableRegistry; + @Mock + private CircuitBreakerService circuitBreakerService; + @Mock + private SharedGroupFactory sharedGroupFactory; + @Mock + private Tracer trace; + @Mock + private SecurityKeyStore ossks; + @Mock + private SslExceptionHandler sslExceptionHandler; + @Mock + private DiscoveryNode discoveryNode; + + private SSLConfig sslConfig; + private SecuritySSLNettyTransport securitySSLNettyTransport; + + @Before + public void setup() { + + sslConfig = new SSLConfig(Settings.EMPTY); + + securitySSLNettyTransport = new SecuritySSLNettyTransport( + Settings.EMPTY, + version, + threadPool, + networkService, + pageCacheRecycler, + namedWriteableRegistry, + circuitBreakerService, + ossks, + sslExceptionHandler, + sharedGroupFactory, + sslConfig, + trace + ); + } + + @Test + public void OnException_withNullChannelShouldThrowException() { + + NullPointerException exception = new NullPointerException("Test Exception"); + + Assert.assertThrows(NullPointerException.class, () -> securitySSLNettyTransport.onException(null, exception)); + + } + + @Test + public void getServerChannelInitializer_shouldReturnValidServerChannel() { + + ChannelHandler channelHandler = securitySSLNettyTransport.getServerChannelInitializer("test-server-channel"); + + assertThat(channelHandler, is(notNullValue())); + assertThat(channelHandler, is(instanceOf(SSLServerChannelInitializer.class))); + } + + @Test + public void getClientChannelInitializer_shouldReturnValidClientChannel() { + + ChannelHandler channelHandler = securitySSLNettyTransport.getClientChannelInitializer(discoveryNode); + + assertThat(channelHandler, is(notNullValue())); + assertThat(channelHandler, is(instanceOf(SSLClientChannelInitializer.class))); + } + +} From 1846fd157847ee08ffea88a849d5c43c46269335 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 13 Dec 2023 10:28:32 -0500 Subject: [PATCH 048/204] Fix issue when scroll is created with LDAP user (#3805) Signed-off-by: Craig Perkins --- .../security/http/LdapAuthenticationTest.java | 74 +++++++++++++++++++ .../org/opensearch/security/user/User.java | 2 +- 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/src/integrationTest/java/org/opensearch/security/http/LdapAuthenticationTest.java b/src/integrationTest/java/org/opensearch/security/http/LdapAuthenticationTest.java index 299b2cc7d2..dbb1724a55 100644 --- a/src/integrationTest/java/org/opensearch/security/http/LdapAuthenticationTest.java +++ b/src/integrationTest/java/org/opensearch/security/http/LdapAuthenticationTest.java @@ -10,8 +10,10 @@ package org.opensearch.security.http; import java.util.List; +import java.util.Map; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.apache.hc.core5.http.message.BasicHeader; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.junit.ClassRule; @@ -20,7 +22,12 @@ import org.junit.rules.RuleChain; import org.junit.runner.RunWith; +import org.opensearch.security.support.ConfigConstants; +import org.opensearch.test.framework.AuthorizationBackend; +import org.opensearch.test.framework.AuthzDomain; import org.opensearch.test.framework.LdapAuthenticationConfigBuilder; +import org.opensearch.test.framework.LdapAuthorizationConfigBuilder; +import org.opensearch.test.framework.RolesMapping; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain; import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AuthenticationBackend; @@ -32,13 +39,20 @@ import org.opensearch.test.framework.ldap.EmbeddedLDAPServer; import org.opensearch.test.framework.log.LogsRule; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.opensearch.security.http.CertificateAuthenticationTest.POINTER_BACKEND_ROLES; +import static org.opensearch.security.http.DirectoryInformationTrees.CN_GROUP_ADMIN; import static org.opensearch.security.http.DirectoryInformationTrees.DN_CAPTAIN_SPOCK_PEOPLE_TEST_ORG; +import static org.opensearch.security.http.DirectoryInformationTrees.DN_GROUPS_TEST_ORG; import static org.opensearch.security.http.DirectoryInformationTrees.DN_OPEN_SEARCH_PEOPLE_TEST_ORG; import static org.opensearch.security.http.DirectoryInformationTrees.DN_PEOPLE_TEST_ORG; import static org.opensearch.security.http.DirectoryInformationTrees.LDIF_DATA; +import static org.opensearch.security.http.DirectoryInformationTrees.PASSWORD_KIRK; import static org.opensearch.security.http.DirectoryInformationTrees.PASSWORD_OPEN_SEARCH; import static org.opensearch.security.http.DirectoryInformationTrees.PASSWORD_SPOCK; import static org.opensearch.security.http.DirectoryInformationTrees.USERNAME_ATTRIBUTE; +import static org.opensearch.security.http.DirectoryInformationTrees.USER_KIRK; import static org.opensearch.security.http.DirectoryInformationTrees.USER_SEARCH; import static org.opensearch.security.http.DirectoryInformationTrees.USER_SPOCK; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; @@ -54,6 +68,8 @@ public class LdapAuthenticationTest { private static final Logger log = LogManager.getLogger(LdapAuthenticationTest.class); + private static final String HEADER_NAME_IMPERSONATE = "opendistro_security_impersonate_as"; + private static final TestSecurityConfig.User ADMIN_USER = new TestSecurityConfig.User("admin").roles(ALL_ACCESS); private static final TestCertificates TEST_CERTIFICATES = new TestCertificates(); @@ -67,6 +83,7 @@ public class LdapAuthenticationTest { public static LocalCluster cluster = new LocalCluster.Builder().testCertificates(TEST_CERTIFICATES) .clusterManager(ClusterManager.SINGLENODE) .anonymousAuth(false) + .nodeSettings(Map.of(ConfigConstants.SECURITY_AUTHCZ_REST_IMPERSONATION_USERS + "." + ADMIN_USER.getName(), List.of(USER_KIRK))) .authc( new AuthcDomain("ldap", BASIC_AUTH_DOMAIN_ORDER + 1, true).httpAuthenticator(new HttpAuthenticator("basic").challenge(false)) .backend( @@ -89,6 +106,29 @@ public class LdapAuthenticationTest { ) .authc(AUTHC_HTTPBASIC_INTERNAL) .users(ADMIN_USER) + .rolesMapping(new RolesMapping(ALL_ACCESS).backendRoles(CN_GROUP_ADMIN)) + .authz( + new AuthzDomain("ldap_roles").httpEnabled(true) + .transportEnabled(true) + .authorizationBackend( + new AuthorizationBackend("ldap").config( + () -> new LdapAuthorizationConfigBuilder().hosts(List.of("localhost:" + embeddedLDAPServer.getLdapNonTlsPort())) + .enableSsl(false) + .bindDn(DN_OPEN_SEARCH_PEOPLE_TEST_ORG) + .password(PASSWORD_OPEN_SEARCH) + .userBase(DN_PEOPLE_TEST_ORG) + .userSearch(USER_SEARCH) + .usernameAttribute(USERNAME_ATTRIBUTE) + .roleBase(DN_GROUPS_TEST_ORG) + .roleSearch("(uniqueMember={0})") + .userRoleAttribute(null) + .userRoleName("disabled") + .roleName("cn") + .resolveNestedRoles(true) + .build() + ) + ) + ) .build(); @ClassRule @@ -117,4 +157,38 @@ public void shouldAuthenticateUserWithLdap_negativeWhenIncorrectPassword() { logsRule.assertThatStackTraceContain(expectedStackTraceFragment); } } + + @Test + public void testShouldCreateScrollWithLdapUserAndImpersonateWithAdmin() { + try (TestRestClient client = cluster.getRestClient(ADMIN_USER)) { + TestRestClient.HttpResponse response = client.put("movies"); + + response.assertStatusCode(200); + } + + String scrollId; + + try (TestRestClient client = cluster.getRestClient(USER_KIRK, PASSWORD_KIRK)) { + TestRestClient.HttpResponse authinfo = client.getAuthInfo(); + + List backendRoles = authinfo.getTextArrayFromJsonBody(POINTER_BACKEND_ROLES); + assertThat(backendRoles, contains(CN_GROUP_ADMIN)); + + TestRestClient.HttpResponse response = client.getWithJsonBody("movies/_search?scroll=10m", "{\"size\": 1}"); + + response.assertStatusCode(200); + + scrollId = response.getTextFromJsonBody("/_scroll_id"); + } + + try (TestRestClient client = cluster.getRestClient(ADMIN_USER)) { + TestRestClient.HttpResponse scrollResponse = client.getWithJsonBody( + "_search/scroll", + "{\"scroll\": \"10m\", \"scroll_id\": \"" + scrollId + "\"}", + new BasicHeader(HEADER_NAME_IMPERSONATE, USER_KIRK) + ); + + scrollResponse.assertStatusCode(200); + } + } } diff --git a/src/main/java/org/opensearch/security/user/User.java b/src/main/java/org/opensearch/security/user/User.java index aa9c09a469..6abba3d734 100644 --- a/src/main/java/org/opensearch/security/user/User.java +++ b/src/main/java/org/opensearch/security/user/User.java @@ -229,7 +229,7 @@ public final boolean equals(final Object obj) { if (obj == null) { return false; } - if (getClass() != obj.getClass()) { + if (!(obj instanceof User)) { return false; } final User other = (User) obj; From e69831505a13fc79dab14c0743c8a12864cdb94d Mon Sep 17 00:00:00 2001 From: Darshit Chanpura <35282393+DarshitChanpura@users.noreply.github.com> Date: Wed, 13 Dec 2023 11:32:36 -0500 Subject: [PATCH 049/204] Adds tests for demo configuration java tool (#3807) ### Description This PR adds tests associated with the demo-configuration tool. ## Issues Resolved: - Resolves https://github.com/opensearch-project/security/issues/3636 ### Testing - Unit Testing - Integration Testing ### Check List - [x] New functionality includes testing ~- [ ] New functionality has been documented~ - [x] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: Darshit Chanpura --- .../security/tools/democonfig/Installer.java | 10 +- .../sanity/tests/InvalidAdminPasswordIT.java | 50 ++ .../sanity/tests/SingleClusterSanityIT.java | 11 + .../democonfig/CertificateGeneratorTests.java | 178 ++++++ .../tools/democonfig/InstallerTests.java | 519 ++++++++++++++++++ .../SecuritySettingsConfigurerTests.java | 326 +++++++++++ .../democonfig/util/DemoConfigHelperUtil.java | 54 ++ .../util/NoExitSecurityManager.java | 26 + 8 files changed, 1173 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/opensearch/security/sanity/tests/InvalidAdminPasswordIT.java create mode 100644 src/test/java/org/opensearch/security/tools/democonfig/CertificateGeneratorTests.java create mode 100644 src/test/java/org/opensearch/security/tools/democonfig/InstallerTests.java create mode 100644 src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java create mode 100644 src/test/java/org/opensearch/security/tools/democonfig/util/DemoConfigHelperUtil.java create mode 100644 src/test/java/org/opensearch/security/tools/democonfig/util/NoExitSecurityManager.java diff --git a/src/main/java/org/opensearch/security/tools/democonfig/Installer.java b/src/main/java/org/opensearch/security/tools/democonfig/Installer.java index 500c65f825..68bc4d7cbf 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/Installer.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/Installer.java @@ -186,7 +186,7 @@ void readOptions(String[] args) { * Prints the help menu when -h option is passed */ void showHelp() { - formatter.printHelp("install_demo_configuration.sh", options, true); + formatter.printHelp("install_demo_configuration" + FILE_EXTENSION, options, true); System.exit(0); } @@ -438,4 +438,12 @@ void finishScriptExecution() { System.out.println(e.getMessage()); } } + + /** + * FOR TESTS ONLY + * resets the installer state to allow testing with fresh instance for the next test. + */ + static void resetInstance() { + instance = null; + } } diff --git a/src/test/java/org/opensearch/security/sanity/tests/InvalidAdminPasswordIT.java b/src/test/java/org/opensearch/security/sanity/tests/InvalidAdminPasswordIT.java new file mode 100644 index 0000000000..60d2eee138 --- /dev/null +++ b/src/test/java/org/opensearch/security/sanity/tests/InvalidAdminPasswordIT.java @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.sanity.tests; + +import org.hamcrest.MatcherAssert; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class InvalidAdminPasswordIT extends SecurityRestTestCase { + + static String currentPasswordVariable = System.getProperty("password"); + + @BeforeClass + public static void setUpAdminAsPasswordVariable() { + System.setProperty("password", "admin"); + } + + @AfterClass + public static void restorePasswordProperty() { + System.setProperty("password", currentPasswordVariable); + } + + @Test + public void testAdminCredentials_adminAsPassword_shouldFail() throws Exception { + try { + client().performRequest(new Request("GET", "")); + } catch (ResponseException e) { + Response res = e.getResponse(); + MatcherAssert.assertThat(res.getStatusLine().getStatusCode(), is(equalTo(401))); + MatcherAssert.assertThat(res.getStatusLine().getReasonPhrase(), is(equalTo("Unauthorized"))); + } + } +} diff --git a/src/test/java/org/opensearch/security/sanity/tests/SingleClusterSanityIT.java b/src/test/java/org/opensearch/security/sanity/tests/SingleClusterSanityIT.java index 8987744d58..97937a2c52 100644 --- a/src/test/java/org/opensearch/security/sanity/tests/SingleClusterSanityIT.java +++ b/src/test/java/org/opensearch/security/sanity/tests/SingleClusterSanityIT.java @@ -19,7 +19,11 @@ import org.hamcrest.MatcherAssert; import org.junit.Test; +import org.opensearch.client.Request; +import org.opensearch.client.Response; + import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; @@ -34,6 +38,13 @@ public void testSecurityPluginInstallation() throws Exception { verifyPluginInstallationOnAllNodes(); } + @Test + public void testAdminCredentials_validAdminPassword_shouldSucceed() throws Exception { + Response response = client().performRequest(new Request("GET", "")); + MatcherAssert.assertThat(response.getStatusLine().getStatusCode(), is(equalTo(200))); + MatcherAssert.assertThat(response.getStatusLine().getReasonPhrase(), is(equalTo("OK"))); + } + private void verifyPluginInstallationOnAllNodes() throws Exception { Map> nodesInCluster = (Map>) getAsMapByAdmin("_nodes").get("nodes"); diff --git a/src/test/java/org/opensearch/security/tools/democonfig/CertificateGeneratorTests.java b/src/test/java/org/opensearch/security/tools/democonfig/CertificateGeneratorTests.java new file mode 100644 index 0000000000..58cf6d1368 --- /dev/null +++ b/src/test/java/org/opensearch/security/tools/democonfig/CertificateGeneratorTests.java @@ -0,0 +1,178 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.tools.democonfig; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileReader; +import java.security.KeyFactory; +import java.security.PrivateKey; +import java.security.cert.Certificate; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.security.spec.PKCS8EncodedKeySpec; +import java.time.Instant; +import java.time.LocalDate; +import java.time.Period; +import java.util.Base64; +import java.util.Date; +import java.util.TimeZone; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import org.opensearch.security.tools.democonfig.util.NoExitSecurityManager; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; +import static org.opensearch.security.tools.democonfig.util.DemoConfigHelperUtil.createDirectory; +import static org.opensearch.security.tools.democonfig.util.DemoConfigHelperUtil.deleteDirectoryRecursive; +import static org.junit.Assert.fail; + +public class CertificateGeneratorTests { + + private static Installer installer; + + @Before + public void setUp() { + installer = Installer.getInstance(); + installer.buildOptions(); + installer.OPENSEARCH_CONF_DIR = System.getProperty("user.dir") + File.separator + "test-conf"; + createDirectory(installer.OPENSEARCH_CONF_DIR); + } + + @After + public void tearDown() { + deleteDirectoryRecursive(installer.OPENSEARCH_CONF_DIR); + Installer.resetInstance(); + } + + @Test + public void testCreateDemoCertificates() throws Exception { + CertificateGenerator certificateGenerator = new CertificateGenerator(installer); + Certificates[] certificatesArray = Certificates.values(); + + certificateGenerator.createDemoCertificates(); + + // root-ca.pem, esnode.pem, esnode-key.pem, kirk.pem, kirk-key.pem + int expectedNumberOfCertificateFiles = 5; + + int certsFound = 0; + + for (Certificates cert : certificatesArray) { + String certFilePath = installer.OPENSEARCH_CONF_DIR + File.separator + cert.getFileName(); + File certFile = new File(certFilePath); + assertThat(certFile.exists(), is(equalTo(true))); + assertThat(certFile.canRead(), is(equalTo(true))); + + if (certFilePath.endsWith("-key.pem")) { + checkPrivateKeyValidity(certFilePath); + } else { + checkCertificateValidity(certFilePath); + } + + // increment a count since a valid certificate was found + certsFound++; + } + + assertThat(certsFound, equalTo(expectedNumberOfCertificateFiles)); + } + + @Test + public void testCreateDemoCertificates_invalidPath() { + installer.OPENSEARCH_CONF_DIR = "invalidPath"; + CertificateGenerator certificateGenerator = new CertificateGenerator(installer); + try { + System.setSecurityManager(new NoExitSecurityManager()); + certificateGenerator.createDemoCertificates(); + } catch (SecurityException e) { + assertThat(e.getMessage(), equalTo("System.exit(-1) blocked to allow print statement testing.")); + } finally { + System.setSecurityManager(null); + } + } + + private static void checkCertificateValidity(String certPath) throws Exception { + try (FileInputStream certInputStream = new FileInputStream(certPath)) { + CertificateFactory cf = CertificateFactory.getInstance("X.509"); + Certificate certificate = cf.generateCertificate(certInputStream); + + if (certificate instanceof X509Certificate) { + X509Certificate x509Certificate = (X509Certificate) certificate; + Date expiryDate = x509Certificate.getNotAfter(); + Instant expiry = expiryDate.toInstant(); + + Period duration = getPeriodBetween(x509Certificate.getNotBefore().toInstant(), expiry); + if (certPath.endsWith("-ca.pem")) { + // root-ca.pem is already expired as the validity is only 30 days from generation + // so we just check interval to be of 30 days + assertThat(duration.getDays(), equalTo(30)); + return; + } + + // we check that cert is valid for total of ~10 yrs + // we don't check days as leaps years may cause flaky-ness + assertThat(duration.getYears(), equalTo(9)); + assertThat(duration.getMonths(), equalTo(11)); + + x509Certificate.checkValidity(); + verifyExpiryAtLeastAYearFromNow(expiry); + + assertThat(x509Certificate.getSigAlgName(), is(equalTo("SHA256withRSA"))); + } + } + } + + private static void verifyExpiryAtLeastAYearFromNow(Instant expiry) { + Period gap = getPeriodBetween(Instant.now(), expiry); + assertThat(gap.getYears(), greaterThanOrEqualTo(1)); + } + + private static Period getPeriodBetween(Instant start, Instant end) { + LocalDate startDate = LocalDate.ofInstant(start, TimeZone.getTimeZone("EDT").toZoneId()); + LocalDate endDate = LocalDate.ofInstant(end, TimeZone.getTimeZone("EDT").toZoneId()); + + return Period.between(startDate, endDate); + } + + private void checkPrivateKeyValidity(String keyPath) { + try { + String pemContent = readPEMFile(keyPath); + + String base64Data = pemContent.replaceAll("-----BEGIN PRIVATE KEY-----|-----END PRIVATE KEY-----", "").replaceAll("\\s", ""); + + byte[] keyBytes = Base64.getDecoder().decode(base64Data); + KeyFactory kf = KeyFactory.getInstance("RSA"); + PrivateKey key = kf.generatePrivate(new PKCS8EncodedKeySpec(keyBytes)); + assertThat(key.getFormat(), is(equalTo("PKCS#8"))); + assertThat(key.getAlgorithm(), is(equalTo("RSA"))); + assertThat(key.isDestroyed(), is(equalTo(false))); + } catch (Exception e) { + fail("Error checking key validity: " + e.getMessage()); + } + } + + private static String readPEMFile(String pemFilePath) throws Exception { + StringBuilder pemContent = new StringBuilder(); + try (BufferedReader reader = new BufferedReader(new FileReader(pemFilePath))) { + String line; + while ((line = reader.readLine()) != null) { + pemContent.append(line).append("\n"); + } + } + return pemContent.toString(); + } +} diff --git a/src/test/java/org/opensearch/security/tools/democonfig/InstallerTests.java b/src/test/java/org/opensearch/security/tools/democonfig/InstallerTests.java new file mode 100644 index 0000000000..4e64fdea72 --- /dev/null +++ b/src/test/java/org/opensearch/security/tools/democonfig/InstallerTests.java @@ -0,0 +1,519 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.tools.democonfig; + +// CS-SUPPRESS-SINGLE: RegexpSingleline extension key-word is used in file ext variable +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.PosixFilePermission; +import java.util.HashSet; +import java.util.Set; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import org.opensearch.security.tools.democonfig.util.NoExitSecurityManager; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.opensearch.security.tools.democonfig.Installer.RPM_DEB_OPENSEARCH_HOME; +import static org.opensearch.security.tools.democonfig.Installer.printScriptHeaders; +import static org.opensearch.security.tools.democonfig.util.DemoConfigHelperUtil.createDirectory; +import static org.opensearch.security.tools.democonfig.util.DemoConfigHelperUtil.createFile; +import static org.opensearch.security.tools.democonfig.util.DemoConfigHelperUtil.deleteDirectoryRecursive; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.fail; + +public class InstallerTests { + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final InputStream originalIn = System.in; + + private static Installer installer; + + @Before + public void setUpStreams() { + System.setOut(new PrintStream(outContent)); + installer = Installer.getInstance(); + installer.buildOptions(); + } + + @After + public void restoreStreams() { + System.setOut(originalOut); + System.setIn(originalIn); + Installer.resetInstance(); + } + + @Test + public void testPrintScriptHeaders() { + printScriptHeaders(); + + String expectedOutput = "### OpenSearch Security Demo Installer" + + System.lineSeparator() + + "### ** Warning: Do not use on production or public reachable systems **" + + System.lineSeparator(); + assertThat(outContent.toString(), equalTo(expectedOutput)); + } + + @Test + public void testReadOptions_withoutHelpOption() { + // All options except Help `-h` + String[] validOptions = { "/scriptDir", "-y", "-i", "-c", "-s", "-t" }; + installer.readOptions(validOptions); + + assertThat(installer.SCRIPT_DIR, equalTo("/scriptDir")); + assertThat(installer.assumeyes, is(true)); + assertThat(installer.initsecurity, is(true)); + assertThat(installer.cluster_mode, is(true)); + assertThat(installer.skip_updates, equalTo(0)); + assertThat(installer.environment, equalTo(ExecutionEnvironment.TEST)); + } + + @Test + public void testReadOptions_help() { + try { + System.setSecurityManager(new NoExitSecurityManager()); + String[] helpOption = { "/scriptDir", "-h" }; + installer.readOptions(helpOption); + } catch (SecurityException e) { + // if help text printed correctly then exit code 0 is expected + assertThat(e.getMessage(), equalTo("System.exit(0) blocked to allow print statement testing.")); + } finally { + System.setSecurityManager(null); + } + + verifyStdOutContainsString("usage: install_demo_configuration" + installer.FILE_EXTENSION + " [-c] [-h] [-i] [-s] [-t] [-y]"); + } + + @Test + public void testGatherUserInputs_withoutAssumeYes() { + // -i & -c option is not passed + String[] validOptions = { "/scriptDir" }; + installer.readOptions(validOptions); + assertThat(installer.assumeyes, is(false)); + assertThat(installer.initsecurity, is(false)); + assertThat(installer.cluster_mode, is(false)); + + // set initsecurity and cluster_mode to no + readInputStream("y" + System.lineSeparator() + "n" + System.lineSeparator() + "n" + System.lineSeparator()); // pass all 3 inputs as + // y + installer.gatherUserInputs(); + + verifyStdOutContainsString("Install demo certificates?"); + verifyStdOutContainsString("Initialize Security Modules?"); + verifyStdOutContainsString("Cluster mode requires additional setup of:"); + verifyStdOutContainsString(" - Virtual memory (vm.max_map_count)" + System.lineSeparator()); + verifyStdOutContainsString("Enable cluster mode?"); + + assertThat(installer.initsecurity, is(false)); + assertThat(installer.cluster_mode, is(false)); + + outContent.reset(); + + // set initsecurity and cluster_mode to no + readInputStream("y" + System.lineSeparator() + "y" + System.lineSeparator() + "y" + System.lineSeparator()); // pass all 3 inputs as + // y + installer.gatherUserInputs(); + + verifyStdOutContainsString("Install demo certificates?"); + verifyStdOutContainsString("Initialize Security Modules?"); + verifyStdOutContainsString("Cluster mode requires additional setup of:"); + verifyStdOutContainsString(" - Virtual memory (vm.max_map_count)" + System.lineSeparator()); + verifyStdOutContainsString("Enable cluster mode?"); + + assertThat(installer.initsecurity, is(true)); + assertThat(installer.cluster_mode, is(true)); + + outContent.reset(); + + // no to demo certificates + try { + System.setSecurityManager(new NoExitSecurityManager()); + + readInputStream("n" + System.lineSeparator() + "n" + System.lineSeparator() + "n" + System.lineSeparator()); + installer.gatherUserInputs(); + } catch (SecurityException e) { + assertThat(e.getMessage(), equalTo("System.exit(0) blocked to allow print statement testing.")); + } finally { + System.setSecurityManager(null); + } + verifyStdOutContainsString("Install demo certificates?"); + verifyStdOutDoesNotContainString("Initialize Security Modules?"); + verifyStdOutDoesNotContainString("Cluster mode requires additional setup of:"); + verifyStdOutDoesNotContainString(" - Virtual memory (vm.max_map_count)" + System.lineSeparator()); + verifyStdOutDoesNotContainString("Enable cluster mode?"); + + outContent.reset(); + + // pass initsecurity and cluster_mode options + String[] validOptionsIC = { "/scriptDir", "-i", "-c" }; + installer.readOptions(validOptionsIC); + assertThat(installer.assumeyes, is(false)); + assertThat(installer.initsecurity, is(true)); + assertThat(installer.cluster_mode, is(true)); + + readInputStream("y" + System.lineSeparator() + "y" + System.lineSeparator() + "y" + System.lineSeparator()); // pass all 3 inputs as + // y + installer.gatherUserInputs(); + + verifyStdOutContainsString("Install demo certificates?"); + verifyStdOutDoesNotContainString("Initialize Security Modules?"); + verifyStdOutDoesNotContainString("Enable cluster mode?"); + + assertThat(installer.initsecurity, is(true)); + assertThat(installer.cluster_mode, is(true)); + } + + @Test + public void testGatherInputs_withAssumeYes() { + String[] validOptionsYes = { "/scriptDir", "-y" }; + installer.readOptions(validOptionsYes); + assertThat(installer.assumeyes, is(true)); + + installer.gatherUserInputs(); + + assertThat(installer.initsecurity, is(true)); + assertThat(installer.cluster_mode, is(true)); + } + + @Test + public void testInitializeVariables_setBaseDir_invalidPath() { + String[] invalidScriptDirPath = { "/scriptDir", "-y" }; + installer.readOptions(invalidScriptDirPath); + + assertThrows("Expected NullPointerException to be thrown", NullPointerException.class, installer::initializeVariables); + + String[] invalidScriptDirPath2 = { "/opensearch/plugins/opensearch-security/tools", "-y" }; + installer.readOptions(invalidScriptDirPath2); + + try { + System.setSecurityManager(new NoExitSecurityManager()); + installer.initializeVariables(); + } catch (SecurityException e) { + assertThat(e.getMessage(), equalTo("System.exit(-1) blocked to allow print statement testing.")); + } finally { + System.setSecurityManager(null); + } + + verifyStdOutContainsString("DEBUG: basedir does not exist"); + } + + @Test + public void testSetBaseDir_valid() { + String currentDir = System.getProperty("user.dir"); + + String[] validBaseDir = { currentDir, "-y" }; + installer.readOptions(validBaseDir); + + installer.setBaseDir(); + + String expectedBaseDirValue = new File(currentDir).getParentFile().getParentFile().getParentFile().getAbsolutePath() + + File.separator; + assertThat(installer.BASE_DIR, equalTo(expectedBaseDirValue)); + } + + @Test + public void testSetOpenSearchVariables_invalidPath() { + String currentDir = System.getProperty("user.dir"); + + String[] validBaseDir = { currentDir, "-y" }; + installer.readOptions(validBaseDir); + + try { + System.setSecurityManager(new NoExitSecurityManager()); + installer.setBaseDir(); + installer.setOpenSearchVariables(); + } catch (SecurityException e) { + assertThat(e.getMessage(), equalTo("System.exit(-1) blocked to allow print statement testing.")); + } finally { + System.setSecurityManager(null); + + } + verifyStdOutContainsString("Unable to determine OpenSearch config file. Quit."); + verifyStdOutContainsString("Unable to determine OpenSearch bin directory. Quit."); + verifyStdOutContainsString("Unable to determine OpenSearch plugins directory. Quit."); + verifyStdOutContainsString("Unable to determine OpenSearch lib directory. Quit."); + + String expectedBaseDirValue = new File(currentDir).getParentFile().getParentFile().getParentFile().getAbsolutePath() + + File.separator; + String expectedOpensearchConfFilePath = expectedBaseDirValue + "config" + File.separator + "opensearch.yml"; + String expectedOpensearchBinDirPath = expectedBaseDirValue + "bin" + File.separator; + String expectedOpensearchPluginDirPath = expectedBaseDirValue + "plugins" + File.separator; + String expectedOpensearchLibDirPath = expectedBaseDirValue + "lib" + File.separator; + String expectedOpensearchInstallType = installer.determineInstallType(); + + assertThat(installer.OPENSEARCH_CONF_FILE, equalTo(expectedOpensearchConfFilePath)); + assertThat(installer.OPENSEARCH_BIN_DIR, equalTo(expectedOpensearchBinDirPath)); + assertThat(installer.OPENSEARCH_PLUGINS_DIR, equalTo(expectedOpensearchPluginDirPath)); + assertThat(installer.OPENSEARCH_LIB_PATH, equalTo(expectedOpensearchLibDirPath)); + assertThat(installer.OPENSEARCH_INSTALL_TYPE, equalTo(expectedOpensearchInstallType)); + + } + + @Test + public void testDetermineInstallType_windows() { + installer.OS = "Windows"; + + String installType = installer.determineInstallType(); + + assertThat(installType, equalTo(".zip")); + } + + @Test + public void testDetermineInstallType_rpm_deb() { + installer.OS = "Linux"; + String dir = System.getProperty("user.dir"); + installer.BASE_DIR = dir; + RPM_DEB_OPENSEARCH_HOME = new File(dir); + + String installType = installer.determineInstallType(); + + assertThat(installType, equalTo("rpm/deb")); + } + + @Test + public void testDetermineInstallType_default() { + installer.OS = "Anything else"; + installer.BASE_DIR = "/random-dir"; + String installType = installer.determineInstallType(); + + assertThat(installType, equalTo(".tar.gz")); + } + + @Test + public void testSetSecurityVariables() { + setUpSecurityDirectories(); + installer.setSecurityVariables(); + + assertThat(installer.OPENSEARCH_VERSION, is(equalTo("osVersion"))); + assertThat(installer.SECURITY_VERSION, is(equalTo("version"))); + tearDownSecurityDirectories(); + } + + @Test + public void testSetSecurityVariables_noSecurityPlugin() { + try { + System.setSecurityManager(new NoExitSecurityManager()); + + installer.setSecurityVariables(); + fail("Expected System.exit(-1) to be called"); + } catch (SecurityException e) { + assertThat(e.getMessage(), equalTo("System.exit(-1) blocked to allow print statement testing.")); + } finally { + System.setSecurityManager(null); + } + } + + @Test + public void testPrintVariables() { + installer.OPENSEARCH_INSTALL_TYPE = "installType"; + installer.OS = "OS"; + installer.OPENSEARCH_CONF_DIR = "confDir"; + installer.OPENSEARCH_CONF_FILE = "confFile"; + installer.OPENSEARCH_BIN_DIR = "/bin"; + installer.OPENSEARCH_PLUGINS_DIR = "/plugins"; + installer.OPENSEARCH_LIB_PATH = "/lib"; + installer.OPENSEARCH_VERSION = "osVersion"; + installer.SECURITY_VERSION = "version"; + + installer.printVariables(); + + String expectedOutput = "OpenSearch install type: installType on OS" + + System.lineSeparator() + + "OpenSearch config dir: confDir" + + System.lineSeparator() + + "OpenSearch config file: confFile" + + System.lineSeparator() + + "OpenSearch bin dir: /bin" + + System.lineSeparator() + + "OpenSearch plugins dir: /plugins" + + System.lineSeparator() + + "OpenSearch lib dir: /lib" + + System.lineSeparator() + + "Detected OpenSearch Version: osVersion" + + System.lineSeparator() + + "Detected OpenSearch Security Version: version" + + System.lineSeparator(); + + assertThat(outContent.toString(), equalTo(expectedOutput)); + } + + @Test + public void testFinishScriptExecution() { + setUpSecurityDirectories(); + SecuritySettingsConfigurer.ADMIN_PASSWORD = "ble"; + + installer.finishScriptExecution(); + + String securityAdminScriptPath = installer.OPENSEARCH_PLUGINS_DIR + + "opensearch-security" + + File.separator + + "tools" + + File.separator + + "securityadmin" + + installer.FILE_EXTENSION; + String securityAdminDemoScriptPath = installer.OPENSEARCH_CONF_DIR + "securityadmin_demo" + installer.FILE_EXTENSION; + setWritePermissions(securityAdminDemoScriptPath); + + SecuritySettingsConfigurer securitySettingsConfigurer = new SecuritySettingsConfigurer(installer); + String lastLine = securitySettingsConfigurer.getSecurityAdminCommands(securityAdminScriptPath)[1]; + + String expectedOutput = "### Success" + + System.lineSeparator() + + "### Execute this script now on all your nodes and then start all nodes" + + System.lineSeparator() + + "### After the whole cluster is up execute: " + + System.lineSeparator() + + lastLine + + System.lineSeparator() + + "### or run ." + + File.separator + + "securityadmin_demo" + + installer.FILE_EXTENSION + + System.lineSeparator() + + "### After that you can also use the Security Plugin ConfigurationGUI" + + System.lineSeparator() + + "### To access your secured cluster open https://: and log in with admin/" + + SecuritySettingsConfigurer.ADMIN_PASSWORD + + "." + + System.lineSeparator() + + "### (Ignore the SSL certificate warning because we installed self-signed demo certificates)" + + System.lineSeparator(); + + assertThat(outContent.toString(), equalTo(expectedOutput)); + + tearDownSecurityDirectories(); + } + + @Test + public void testFinishScriptExecution_withInitSecurityEnabled() { + setUpSecurityDirectories(); + installer.initsecurity = true; + SecuritySettingsConfigurer.ADMIN_PASSWORD = "ble"; + + installer.finishScriptExecution(); + + String securityAdminScriptPath = installer.OPENSEARCH_PLUGINS_DIR + + "opensearch-security" + + File.separator + + "tools" + + File.separator + + "securityadmin" + + installer.FILE_EXTENSION; + String securityAdminDemoScriptPath = installer.OPENSEARCH_CONF_DIR + "securityadmin_demo" + installer.FILE_EXTENSION; + setWritePermissions(securityAdminDemoScriptPath); + + SecuritySettingsConfigurer securitySettingsConfigurer = new SecuritySettingsConfigurer(installer); + String lastLine = securitySettingsConfigurer.getSecurityAdminCommands(securityAdminScriptPath)[1]; + + String expectedOutput = "### Success" + + System.lineSeparator() + + "### Execute this script now on all your nodes and then start all nodes" + + System.lineSeparator() + + "### OpenSearch Security will be automatically initialized." + + System.lineSeparator() + + "### If you like to change the runtime configuration " + + System.lineSeparator() + + "### change the files in .." + + File.separator + + ".." + + File.separator + + ".." + + File.separator + + "config" + + File.separator + + "opensearch-security and execute: " + + System.lineSeparator() + + lastLine + + System.lineSeparator() + + "### or run ." + + File.separator + + "securityadmin_demo" + + installer.FILE_EXTENSION + + System.lineSeparator() + + "### To use the Security Plugin ConfigurationGUI" + + System.lineSeparator() + + "### To access your secured cluster open https://: and log in with admin/" + + SecuritySettingsConfigurer.ADMIN_PASSWORD + + "." + + System.lineSeparator() + + "### (Ignore the SSL certificate warning because we installed self-signed demo certificates)" + + System.lineSeparator(); + + assertThat(outContent.toString(), equalTo(expectedOutput)); + + tearDownSecurityDirectories(); + } + + private void readInputStream(String input) { + System.setIn(new ByteArrayInputStream(input.getBytes())); + } + + public void setUpSecurityDirectories() { + String currentDir = System.getProperty("user.dir"); + + String[] validBaseDir = { currentDir, "-y" }; + installer.readOptions(validBaseDir); + installer.setBaseDir(); + installer.OPENSEARCH_PLUGINS_DIR = installer.BASE_DIR + "plugins" + File.separator; + installer.OPENSEARCH_LIB_PATH = installer.BASE_DIR + "lib" + File.separator; + installer.OPENSEARCH_CONF_DIR = installer.BASE_DIR + "test-conf" + File.separator; + + createDirectory(installer.OPENSEARCH_PLUGINS_DIR); + createDirectory(installer.OPENSEARCH_LIB_PATH); + createDirectory(installer.OPENSEARCH_CONF_DIR); + createDirectory(installer.OPENSEARCH_PLUGINS_DIR + "opensearch-security"); + createFile(installer.OPENSEARCH_LIB_PATH + "opensearch-core-osVersion.jar"); + createFile(installer.OPENSEARCH_PLUGINS_DIR + "opensearch-security" + File.separator + "opensearch-security-version.jar"); + createFile(installer.OPENSEARCH_CONF_DIR + File.separator + "securityadmin_demo.sh"); + } + + public void tearDownSecurityDirectories() { + // Clean up testing directories or files + deleteDirectoryRecursive(installer.OPENSEARCH_PLUGINS_DIR); + deleteDirectoryRecursive(installer.OPENSEARCH_LIB_PATH); + deleteDirectoryRecursive(installer.OPENSEARCH_CONF_DIR); + } + + static void setWritePermissions(String filePath) { + if (!installer.OS.toLowerCase().contains("win")) { + Path file = Paths.get(filePath); + Set perms = new HashSet<>(); + perms.add(PosixFilePermission.OWNER_WRITE); + try { + Files.setPosixFilePermissions(file, perms); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + private void verifyStdOutContainsString(String s) { + assertThat(outContent.toString(), containsString(s)); + } + + private void verifyStdOutDoesNotContainString(String s) { + assertThat(outContent.toString(), not(containsString(s))); + } +} diff --git a/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java b/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java new file mode 100644 index 0000000000..caba8b44d0 --- /dev/null +++ b/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java @@ -0,0 +1,326 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.tools.democonfig; + +// CS-SUPPRESS-SINGLE: RegexpSingleline extension key-word is used in file ext variable +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.security.tools.democonfig.util.NoExitSecurityManager; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.opensearch.security.tools.democonfig.SecuritySettingsConfigurer.REST_ENABLED_ROLES; +import static org.opensearch.security.tools.democonfig.SecuritySettingsConfigurer.SYSTEM_INDICES; +import static org.opensearch.security.tools.democonfig.SecuritySettingsConfigurer.isKeyPresentInYMLFile; +import static org.opensearch.security.tools.democonfig.util.DemoConfigHelperUtil.createDirectory; +import static org.opensearch.security.tools.democonfig.util.DemoConfigHelperUtil.createFile; +import static org.opensearch.security.tools.democonfig.util.DemoConfigHelperUtil.deleteDirectoryRecursive; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +public class SecuritySettingsConfigurerTests { + + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; + private final InputStream originalIn = System.in; + + private final String adminPasswordKey = "initialAdminPassword"; + + private static SecuritySettingsConfigurer securitySettingsConfigurer; + + private static Installer installer; + + @Before + public void setUp() { + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(outContent)); + installer = Installer.getInstance(); + installer.buildOptions(); + securitySettingsConfigurer = new SecuritySettingsConfigurer(installer); + setUpConf(); + } + + @After + public void tearDown() throws NoSuchFieldException, IllegalAccessException { + outContent.reset(); + System.setOut(originalOut); + System.setErr(originalErr); + System.setIn(originalIn); + deleteDirectoryRecursive(installer.OPENSEARCH_CONF_DIR); + unsetEnv(adminPasswordKey); + Installer.resetInstance(); + } + + @Test + public void testUpdateAdminPasswordWithCustomPassword() throws NoSuchFieldException, IllegalAccessException { + String customPassword = "myStrongPassword123"; // generateStrongPassword(); + setEnv(adminPasswordKey, customPassword); + + securitySettingsConfigurer.updateAdminPassword(); + + assertThat(customPassword, is(equalTo(SecuritySettingsConfigurer.ADMIN_PASSWORD))); + + verifyStdOutContainsString("ADMIN PASSWORD SET TO: " + customPassword); + } + + @Test + public void testUpdateAdminPasswordWithFilePassword() throws IOException { + String customPassword = "myStrongPassword123"; + String initialAdminPasswordTxt = installer.OPENSEARCH_CONF_DIR + adminPasswordKey + ".txt"; + createFile(initialAdminPasswordTxt); + + try (BufferedWriter writer = new BufferedWriter(new FileWriter(initialAdminPasswordTxt, StandardCharsets.UTF_8))) { + writer.write(customPassword); + } catch (IOException e) { + throw new IOException("Unable to update the internal users file with the hashed password."); + } + + securitySettingsConfigurer.updateAdminPassword(); + + assertEquals(customPassword, SecuritySettingsConfigurer.ADMIN_PASSWORD); + verifyStdOutContainsString("ADMIN PASSWORD SET TO: " + customPassword); + } + + @Test + public void testUpdateAdminPassword_noPasswordSupplied() { + try { + System.setSecurityManager(new NoExitSecurityManager()); + securitySettingsConfigurer.updateAdminPassword(); + } catch (SecurityException e) { + assertThat(e.getMessage(), equalTo("System.exit(-1) blocked to allow print statement testing.")); + } finally { + System.setSecurityManager(null); + } + + verifyStdOutContainsString("No custom admin password found. Please provide a password."); + } + + @Test + public void testUpdateAdminPasswordWithWeakPassword() throws NoSuchFieldException, IllegalAccessException { + + setEnv(adminPasswordKey, "weakpassword"); + try { + System.setSecurityManager(new NoExitSecurityManager()); + securitySettingsConfigurer.updateAdminPassword(); + } catch (SecurityException e) { + assertThat(e.getMessage(), equalTo("System.exit(-1) blocked to allow print statement testing.")); + } finally { + System.setSecurityManager(null); + } + + verifyStdOutContainsString("Password weakpassword is weak. Please re-try with a stronger password."); + } + + @Test + public void testUpdateAdminPasswordWithWeakPassword_skipPasswordValidation() throws NoSuchFieldException, IllegalAccessException { + setEnv(adminPasswordKey, "weakpassword"); + installer.environment = ExecutionEnvironment.TEST; + securitySettingsConfigurer.updateAdminPassword(); + + assertThat("weakpassword", is(equalTo(SecuritySettingsConfigurer.ADMIN_PASSWORD))); + verifyStdOutContainsString("ADMIN PASSWORD SET TO: weakpassword"); + } + + @Test + public void testSecurityPluginAlreadyConfigured() { + securitySettingsConfigurer.writeSecurityConfigToOpenSearchYML(); + String expectedMessage = installer.OPENSEARCH_CONF_FILE + " seems to be already configured for Security. Quit."; + try { + System.setSecurityManager(new NoExitSecurityManager()); + securitySettingsConfigurer.checkIfSecurityPluginIsAlreadyConfigured(); + } catch (SecurityException e) { + assertThat(e.getMessage(), equalTo("System.exit(-1) blocked to allow print statement testing.")); + } finally { + System.setSecurityManager(null); + } + verifyStdOutContainsString(expectedMessage); + } + + @Test + public void testSecurityPluginNotConfigured() { + try { + securitySettingsConfigurer.checkIfSecurityPluginIsAlreadyConfigured(); + } catch (Exception e) { + fail("Expected checkIfSecurityPluginIsAlreadyConfigured to succeed without any errors."); + } + } + + @Test + public void testConfigFileDoesNotExist() { + installer.OPENSEARCH_CONF_FILE = "path/to/nonexistentfile"; + String expectedMessage = "OpenSearch configuration file does not exist. Quit."; + try { + System.setSecurityManager(new NoExitSecurityManager()); + securitySettingsConfigurer.checkIfSecurityPluginIsAlreadyConfigured(); + } catch (SecurityException e) { + assertThat(e.getMessage(), equalTo("System.exit(-1) blocked to allow print statement testing.")); + } finally { + System.setSecurityManager(null); + } + + verifyStdOutContainsString(expectedMessage); + + // reset the file pointer + installer.OPENSEARCH_CONF_FILE = installer.OPENSEARCH_CONF_DIR + "opensearch.yml"; + } + + @Test + public void testBuildSecurityConfigMap() { + Map actual = securitySettingsConfigurer.buildSecurityConfigMap(); + + assertThat(actual.size(), is(17)); + assertThat(actual.get("plugins.security.ssl.transport.pemcert_filepath"), is(equalTo(Certificates.NODE_CERT.getFileName()))); + assertThat(actual.get("plugins.security.ssl.transport.pemkey_filepath"), is(equalTo(Certificates.NODE_KEY.getFileName()))); + assertThat(actual.get("plugins.security.ssl.transport.pemtrustedcas_filepath"), is(equalTo(Certificates.ROOT_CA.getFileName()))); + assertThat(actual.get("plugins.security.ssl.transport.enforce_hostname_verification"), is(equalTo(false))); + assertThat(actual.get("plugins.security.ssl.http.enabled"), is(equalTo(true))); + assertThat(actual.get("plugins.security.ssl.http.pemcert_filepath"), is(equalTo(Certificates.NODE_CERT.getFileName()))); + assertThat(actual.get("plugins.security.ssl.http.pemkey_filepath"), is(equalTo(Certificates.NODE_KEY.getFileName()))); + assertThat(actual.get("plugins.security.ssl.http.pemtrustedcas_filepath"), is(equalTo(Certificates.ROOT_CA.getFileName()))); + assertThat(actual.get("plugins.security.allow_unsafe_democertificates"), is(equalTo(true))); + assertThat(actual.get("plugins.security.authcz.admin_dn"), is(equalTo(List.of("CN=kirk,OU=client,O=client,L=test,C=de")))); + assertThat(actual.get("plugins.security.audit.type"), is(equalTo("internal_opensearch"))); + assertThat(actual.get("plugins.security.enable_snapshot_restore_privilege"), is(equalTo(true))); + assertThat(actual.get("plugins.security.check_snapshot_restore_write_privileges"), is(equalTo(true))); + assertThat(actual.get("plugins.security.restapi.roles_enabled"), is(equalTo(REST_ENABLED_ROLES))); + assertThat(actual.get("plugins.security.system_indices.enabled"), is(equalTo(true))); + assertThat(actual.get("plugins.security.system_indices.indices"), is(equalTo(SYSTEM_INDICES))); + assertThat(actual.get("node.max_local_storage_nodes"), is(equalTo(3))); + + installer.initsecurity = true; + actual = securitySettingsConfigurer.buildSecurityConfigMap(); + assertThat(actual.get("plugins.security.allow_default_init_securityindex"), is(equalTo(true))); + + installer.cluster_mode = true; + actual = securitySettingsConfigurer.buildSecurityConfigMap(); + assertThat(actual.get("network.host"), is(equalTo("0.0.0.0"))); + assertThat(actual.get("node.name"), is(equalTo("smoketestnode"))); + assertThat(actual.get("cluster.initial_cluster_manager_nodes"), is(equalTo("smoketestnode"))); + } + + @Test + public void testIsStringAlreadyPresentInFile_isNotPresent() throws IOException { + String str1 = "network.host"; + String str2 = "some.random.config"; + + installer.initsecurity = true; + securitySettingsConfigurer.writeSecurityConfigToOpenSearchYML(); + + assertThat(isKeyPresentInYMLFile(installer.OPENSEARCH_CONF_FILE, str1), is(equalTo(false))); + assertThat(isKeyPresentInYMLFile(installer.OPENSEARCH_CONF_FILE, str2), is(equalTo(false))); + } + + @Test + public void testIsStringAlreadyPresentInFile_isPresent() throws IOException { + String str1 = "network.host"; + String str2 = "some.random.config"; + + installer.initsecurity = true; + installer.cluster_mode = true; + securitySettingsConfigurer.writeSecurityConfigToOpenSearchYML(); + + assertThat(isKeyPresentInYMLFile(installer.OPENSEARCH_CONF_FILE, str1), is(equalTo(true))); + assertThat(isKeyPresentInYMLFile(installer.OPENSEARCH_CONF_FILE, str2), is(equalTo(false))); + } + + @Test + public void testCreateSecurityAdminDemoScriptAndGetSecurityAdminCommands() throws IOException { + String demoPath = installer.OPENSEARCH_CONF_DIR + "securityadmin_demo" + installer.FILE_EXTENSION; + securitySettingsConfigurer.createSecurityAdminDemoScript("scriptPath", demoPath); + + assertThat(new File(demoPath).exists(), is(equalTo(true))); + + String[] commands = securitySettingsConfigurer.getSecurityAdminCommands("scriptPath"); + + try (BufferedReader reader = new BufferedReader(new FileReader(demoPath, StandardCharsets.UTF_8))) { + assertThat(reader.readLine(), is(commands[0])); + assertThat(reader.readLine(), is(equalTo(commands[1]))); + } + } + + @Test + public void testCreateSecurityAdminDemoScript_invalidPath() { + String demoPath = null; + try { + securitySettingsConfigurer.createSecurityAdminDemoScript("scriptPath", demoPath); + fail("Expected to throw Exception"); + } catch (IOException | NullPointerException e) { + // expected + } + } + + @SuppressWarnings("unchecked") + public static void setEnv(String key, String value) throws NoSuchFieldException, IllegalAccessException { + Class[] classes = Collections.class.getDeclaredClasses(); + Map env = System.getenv(); + for (Class cl : classes) { + if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) { + Field field = cl.getDeclaredField("m"); + field.setAccessible(true); + Object obj = field.get(env); + Map map = (Map) obj; + map.clear(); + map.put(key, value); + } + } + } + + @SuppressWarnings("unchecked") + public static void unsetEnv(String key) throws NoSuchFieldException, IllegalAccessException { + Class[] classes = Collections.class.getDeclaredClasses(); + Map env = System.getenv(); + for (Class cl : classes) { + if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) { + Field field = cl.getDeclaredField("m"); + field.setAccessible(true); + Object obj = field.get(env); + Map map = (Map) obj; + map.remove(key); + } + } + } + + void setUpConf() { + installer.OPENSEARCH_CONF_DIR = System.getProperty("user.dir") + File.separator + "test-conf" + File.separator; + installer.OPENSEARCH_CONF_FILE = installer.OPENSEARCH_CONF_DIR + "opensearch.yml"; + String securityConfDir = installer.OPENSEARCH_CONF_DIR + "opensearch-security" + File.separator; + createDirectory(securityConfDir); + createFile(securityConfDir + "internal_users.yml"); + createFile(installer.OPENSEARCH_CONF_FILE); + } + + private void verifyStdOutContainsString(String s) { + assertThat(outContent.toString(), containsString(s)); + } +} diff --git a/src/test/java/org/opensearch/security/tools/democonfig/util/DemoConfigHelperUtil.java b/src/test/java/org/opensearch/security/tools/democonfig/util/DemoConfigHelperUtil.java new file mode 100644 index 0000000000..7fd4c3330d --- /dev/null +++ b/src/test/java/org/opensearch/security/tools/democonfig/util/DemoConfigHelperUtil.java @@ -0,0 +1,54 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.tools.democonfig.util; + +import java.io.File; + +public class DemoConfigHelperUtil { + public static void createDirectory(String path) { + File directory = new File(path); + if (!directory.exists() && !directory.mkdirs()) { + throw new RuntimeException("Failed to create directory: " + path); + } + } + + public static void createFile(String path) { + try { + File file = new File(path); + if (!file.exists() && !file.createNewFile()) { + throw new RuntimeException("Failed to create file: " + path); + } + } catch (Exception e) { + // without this the catch, we would need to throw exception, + // which would then require modifying caller method signature + throw new RuntimeException("Failed to create file: " + path, e); + } + } + + public static void deleteDirectoryRecursive(String path) { + File directory = new File(path); + if (directory.exists()) { + File[] files = directory.listFiles(); + if (files != null) { + for (File file : files) { + if (file.isDirectory()) { + deleteDirectoryRecursive(file.getAbsolutePath()); + } else { + file.delete(); + } + } + } + // Delete the empty directory after all its content is deleted + directory.delete(); + } + } +} diff --git a/src/test/java/org/opensearch/security/tools/democonfig/util/NoExitSecurityManager.java b/src/test/java/org/opensearch/security/tools/democonfig/util/NoExitSecurityManager.java new file mode 100644 index 0000000000..16bb61f169 --- /dev/null +++ b/src/test/java/org/opensearch/security/tools/democonfig/util/NoExitSecurityManager.java @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.tools.democonfig.util; + +public class NoExitSecurityManager extends SecurityManager { + @Override + public void checkPermission(java.security.Permission perm) { + // Allow everything except System.exit code 0 &b -1 + if (perm instanceof java.lang.RuntimePermission && ("exitVM.0".equals(perm.getName()) || "exitVM.-1".equals(perm.getName()))) { + StringBuilder sb = new StringBuilder(); + sb.append("System.exit("); + sb.append(perm.getName().contains("0") ? 0 : -1); + sb.append(") blocked to allow print statement testing."); + throw new SecurityException(sb.toString()); + } + } +} From 9d11524a2f86fb964671a7496b42b9aa6f9cc5fc Mon Sep 17 00:00:00 2001 From: Darshit Chanpura <35282393+DarshitChanpura@users.noreply.github.com> Date: Wed, 13 Dec 2023 16:56:22 -0500 Subject: [PATCH 050/204] Renames initialAdminPassword to OPENSEARCH_INITIAL_ADMIN_PASSWORD to be compliant with opensearch naming convention (#3843) ### Issues Resolved - Resolves https://github.com/opensearch-project/security/issues/3842 ### Testing - automated tests ### Check List - [x] New functionality includes testing ~- [ ] New functionality has been documented~ - [x] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: Darshit Chanpura --- .../actions/start-opensearch-with-one-plugin/action.yml | 4 ++-- .../org/opensearch/security/support/ConfigConstants.java | 4 ++++ .../tools/democonfig/SecuritySettingsConfigurer.java | 9 ++++++--- .../democonfig/SecuritySettingsConfigurerTests.java | 3 ++- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/actions/start-opensearch-with-one-plugin/action.yml b/.github/actions/start-opensearch-with-one-plugin/action.yml index 642264f4ec..8513e64fdf 100644 --- a/.github/actions/start-opensearch-with-one-plugin/action.yml +++ b/.github/actions/start-opensearch-with-one-plugin/action.yml @@ -71,9 +71,9 @@ runs: 'y' | .\opensearch-${{ inputs.opensearch-version }}-SNAPSHOT\bin\opensearch-plugin.bat install file:$(pwd)\${{ inputs.plugin-name }}.zip shell: pwsh - - name: Write password to initialAdminPassword location + - name: Write password to opensearch_initial_admin_password txt run: - echo ${{ inputs.admin-password }} >> ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/config/initialAdminPassword.txt + echo ${{ inputs.admin-password }} >> ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/config/opensearch_initial_admin_password.txt shell: bash # Run any configuration scripts diff --git a/src/main/java/org/opensearch/security/support/ConfigConstants.java b/src/main/java/org/opensearch/security/support/ConfigConstants.java index 1f5728edfb..f106466984 100644 --- a/src/main/java/org/opensearch/security/support/ConfigConstants.java +++ b/src/main/java/org/opensearch/security/support/ConfigConstants.java @@ -334,6 +334,10 @@ public enum RolesMappingResolution { public static final boolean EXTENSIONS_BWC_PLUGIN_MODE_DEFAULT = false; // CS-ENFORCE-SINGLE + // Variables for initial admin password support + public static final String OPENSEARCH_INITIAL_ADMIN_PASSWORD = "OPENSEARCH_INITIAL_ADMIN_PASSWORD"; + public static final String OPENSEARCH_INITIAL_ADMIN_PASSWORD_TXT = "opensearch_initial_admin_password.txt"; + public static Set getSettingAsSet( final Settings settings, final String key, diff --git a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java index ac9b0651fd..28da25c592 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java @@ -31,6 +31,7 @@ import org.opensearch.security.DefaultObjectMapper; import org.opensearch.security.dlic.rest.validation.PasswordValidator; import org.opensearch.security.dlic.rest.validation.RequestContentValidator; +import org.opensearch.security.support.ConfigConstants; import org.opensearch.security.tools.Hasher; import org.yaml.snakeyaml.DumperOptions; @@ -94,8 +95,8 @@ public void configureSecuritySettings() { * Replaces the admin password in internal_users.yml with the custom or generated password */ void updateAdminPassword() { - String initialAdminPassword = System.getenv().get("initialAdminPassword"); - String ADMIN_PASSWORD_FILE_PATH = installer.OPENSEARCH_CONF_DIR + "initialAdminPassword.txt"; + String initialAdminPassword = System.getenv().get(ConfigConstants.OPENSEARCH_INITIAL_ADMIN_PASSWORD); + String ADMIN_PASSWORD_FILE_PATH = installer.OPENSEARCH_CONF_DIR + ConfigConstants.OPENSEARCH_INITIAL_ADMIN_PASSWORD_TXT; String INTERNAL_USERS_FILE_PATH = installer.OPENSEARCH_CONF_DIR + "opensearch-security" + File.separator + "internal_users.yml"; boolean shouldValidatePassword = installer.environment.equals(ExecutionEnvironment.DEMO); try { @@ -115,7 +116,9 @@ void updateAdminPassword() { try (BufferedReader br = new BufferedReader(new FileReader(ADMIN_PASSWORD_FILE_PATH, StandardCharsets.UTF_8))) { ADMIN_PASSWORD = br.readLine(); } catch (IOException e) { - System.out.println("Error reading admin password from initialAdminPassword.txt."); + System.out.println( + "Error reading admin password from " + ConfigConstants.OPENSEARCH_INITIAL_ADMIN_PASSWORD_TXT + "." + ); System.exit(-1); } } diff --git a/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java b/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java index caba8b44d0..cb36ba0d6c 100644 --- a/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java +++ b/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java @@ -32,6 +32,7 @@ import org.junit.Test; import org.junit.runner.RunWith; +import org.opensearch.security.support.ConfigConstants; import org.opensearch.security.tools.democonfig.util.NoExitSecurityManager; import static org.hamcrest.MatcherAssert.assertThat; @@ -55,7 +56,7 @@ public class SecuritySettingsConfigurerTests { private final PrintStream originalErr = System.err; private final InputStream originalIn = System.in; - private final String adminPasswordKey = "initialAdminPassword"; + private final String adminPasswordKey = ConfigConstants.OPENSEARCH_INITIAL_ADMIN_PASSWORD; private static SecuritySettingsConfigurer securitySettingsConfigurer; From 9da4a78f237f238bfebb5ecd2dff58a814927542 Mon Sep 17 00:00:00 2001 From: MaciejMierzwa Date: Thu, 14 Dec 2023 18:34:36 +0100 Subject: [PATCH 051/204] Search operation test flakiness fix (#3602) ### Description test PR ### Issues Resolved - https://github.com/opensearch-project/security/issues/3426 - https://github.com/opensearch-project/security/issues/2141 - https://github.com/opensearch-project/security/issues/2169 Similar to this task: https://github.com/opensearch-project/security/issues/1917 From what I've noticed some logs are duplicated on faster machines. During test creation audit logging results were added based on actual results produced by tests. Now if there are slower moments where logging produces non-duplicated logs, those were marked as failed. By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: Maciej Mierzwa Signed-off-by: MaciejMierzwa --- .../security/SearchOperationTest.java | 242 +++++++++--------- .../opensearch/security/SnapshotSteps.java | 22 +- .../opensearch/security/rest/WhoAmITests.java | 2 +- .../test/framework/audit/AuditLogsRule.java | 42 ++- .../framework/cluster/ClusterManager.java | 9 + 5 files changed, 188 insertions(+), 129 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/SearchOperationTest.java b/src/integrationTest/java/org/opensearch/security/SearchOperationTest.java index 652b0dcf10..cbb5ec11f0 100644 --- a/src/integrationTest/java/org/opensearch/security/SearchOperationTest.java +++ b/src/integrationTest/java/org/opensearch/security/SearchOperationTest.java @@ -13,6 +13,8 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import com.google.common.base.Stopwatch; @@ -23,7 +25,6 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -97,6 +98,7 @@ import org.opensearch.index.reindex.ReindexRequest; import org.opensearch.repositories.RepositoryMissingException; import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.security.auditlog.AuditLog; import org.opensearch.test.framework.AuditCompliance; import org.opensearch.test.framework.AuditConfiguration; import org.opensearch.test.framework.AuditFilters; @@ -359,7 +361,7 @@ public class SearchOperationTest { .on(INDICES_ON_WHICH_USER_CAN_PERFORM_INDEX_OPERATIONS_PREFIX.concat("*")) ); - private static User USER_ALLOWED_TO_CREATE_INDEX = new User("user-allowed-to-create-index").roles( + private static final User USER_ALLOWED_TO_CREATE_INDEX = new User("user-allowed-to-create-index").roles( new Role("create-index-role").indexPermissions("indices:admin/create").on("*") ); @@ -462,7 +464,7 @@ public void cleanData() throws ExecutionException, InterruptedException { if (indicesExistsResponse.isExists()) { DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indexToBeDeleted); indices.delete(deleteIndexRequest).actionGet(); - Awaitility.await().ignoreExceptions().until(() -> indices.exists(indicesExistsRequest).get().isExists() == false); + Awaitility.await().ignoreExceptions().until(() -> !indices.exists(indicesExistsRequest).get().isExists()); } } @@ -976,12 +978,11 @@ public void shouldIndexDocumentInBulkRequest_positive() throws IOException { } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertAtLeast(4, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER));// sometimes 4 or 6 - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest"));// sometimes 2 or 4 + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); + auditLogsRule.assertExactlyOne(auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } - @Ignore("Audit log verification is shown to be flaky in this test") @Test public void shouldIndexDocumentInBulkRequest_partiallyPositive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { @@ -1004,10 +1005,10 @@ public void shouldIndexDocumentInBulkRequest_partiallyPositive() throws IOExcept } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); + auditLogsRule.assertExactlyOne(auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_WRITE_USER, "BulkShardRequest").withIndex(SONG_INDEX_NAME)); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test @@ -1036,7 +1037,6 @@ public void shouldIndexDocumentInBulkRequest_negative() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_WRITE_USER, "BulkShardRequest").withIndex(SONG_INDEX_NAME)); } - @Ignore("Audit log verification is shown to be flaky in this test") @Test public void shouldUpdateDocumentsInBulk_positive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { @@ -1058,13 +1058,10 @@ public void shouldUpdateDocumentsInBulk_positive() throws IOException { } auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); - + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); + auditLogsRule.assertExactlyOne(auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); } - @Ignore("Audit log verification is shown to be flaky in this test") @Test public void shouldUpdateDocumentsInBulk_partiallyPositive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { @@ -1087,10 +1084,10 @@ public void shouldUpdateDocumentsInBulk_partiallyPositive() throws IOException { } auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); + auditLogsRule.assertExactlyOne(auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_WRITE_USER, "BulkShardRequest").withIndex(SONG_INDEX_NAME)); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test @@ -1120,6 +1117,10 @@ public void shouldUpdateDocumentsInBulk_negative() throws IOException { @Test public void shouldDeleteDocumentInBulk_positive() throws IOException { + // create index + Settings sourceIndexSettings = Settings.builder().put("index.number_of_replicas", 2).put("index.number_of_shards", 2).build(); + IndexOperationsHelper.createIndex(cluster, WRITE_SONG_INDEX_NAME, sourceIndexSettings); + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(IMMEDIATE); bulkRequest.add(new IndexRequest(WRITE_SONG_INDEX_NAME).id("one").source(SONGS[0].asMap())); @@ -1144,14 +1145,16 @@ public void shouldDeleteDocumentInBulk_positive() throws IOException { } auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactly(2, auditPredicate(null).withLayer(AuditLog.Origin.TRANSPORT)); + auditLogsRule.assertAtLeastTransportMessages(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); + auditLogsRule.assertAtLeastTransportMessages(4, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); } - @Ignore("Audit log verification is shown to be flaky in this test") @Test public void shouldDeleteDocumentInBulk_partiallyPositive() throws IOException { + Settings indexSettings = Settings.builder().put("index.number_of_replicas", 0).put("index.number_of_shards", 1).build(); + IndexOperationsHelper.createIndex(cluster, WRITE_SONG_INDEX_NAME, indexSettings); + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(IMMEDIATE); bulkRequest.add(new IndexRequest(WRITE_SONG_INDEX_NAME).id("one").source(SONGS[0].asMap())); @@ -1176,10 +1179,8 @@ public void shouldDeleteDocumentInBulk_partiallyPositive() throws IOException { } auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_WRITE_USER, "BulkShardRequest")); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test @@ -1208,7 +1209,6 @@ public void shouldDeleteDocumentInBulk_negative() throws IOException { } - @Ignore("Seems like reindixing isn't completing before the test proceeds") @Test public void shouldReindexDocuments_positive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(REINDEXING_USER)) { @@ -1227,14 +1227,13 @@ public void shouldReindexDocuments_positive() throws IOException { auditLogsRule.assertExactlyOne(grantedPrivilege(REINDEXING_USER, "ReindexRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(REINDEXING_USER, "SearchRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(REINDEXING_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(REINDEXING_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(REINDEXING_USER, "PutMappingRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(REINDEXING_USER, "CreateIndexRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(REINDEXING_USER, "SearchScrollRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(REINDEXING_USER)); + auditLogsRule.assertExactlyOne(auditPredicate(INDEX_EVENT).withEffectiveUser(REINDEXING_USER)); auditLogsRule.assertExactlyOne(missingPrivilege(REINDEXING_USER, "ClearScrollRequest")); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(REINDEXING_USER, "PutMappingRequest")); } - @Ignore("Seems like reindixing isn't completing before the test proceeds") @Test public void shouldReindexDocuments_negativeSource() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(REINDEXING_USER)) { @@ -1249,7 +1248,6 @@ public void shouldReindexDocuments_negativeSource() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(REINDEXING_USER, "SearchRequest")); } - @Ignore("Seems like reindixing isn't completing before the test proceeds") @Test public void shouldReindexDocuments_negativeDestination() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(REINDEXING_USER)) { @@ -1268,7 +1266,6 @@ public void shouldReindexDocuments_negativeDestination() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(REINDEXING_USER, "ClearScrollRequest")); } - @Ignore("Seems like reindixing isn't completing before the test proceeds") @Test public void shouldReindexDocuments_negativeSourceAndDestination() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(REINDEXING_USER)) { @@ -1341,7 +1338,6 @@ public void shouldDeleteDocument_negative() throws IOException { } } - @Ignore("Create alias / delete alias isn't resolving in a timely manner for this test") @Test public void shouldCreateAlias_positive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_READ_USER)) { @@ -1355,11 +1351,10 @@ public void shouldCreateAlias_positive() throws IOException { assertThat(internalClient, clusterContainsDocument(TEMPORARY_ALIAS_NAME, ID_S1)); } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_READ_USER).withRestRequest(POST, "/_aliases")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_READ_USER, "IndicesAliasesRequest")); - auditLogsRule.assertExactly(2, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_READ_USER)); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_READ_USER, "IndicesAliasesRequest")); + auditLogsRule.assertExactlyOne(auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_READ_USER)); } - @Ignore("Create alias / delete alias isn't resolving in a timely manner for this test") @Test public void shouldCreateAlias_negative() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_READ_USER)) { @@ -1377,7 +1372,6 @@ public void shouldCreateAlias_negative() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_READ_USER, "IndicesAliasesRequest")); } - @Ignore("Create alias / delete alias isn't resolving in a timely manner for this test") @Test public void shouldDeleteAlias_positive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_READ_USER)) { @@ -1394,8 +1388,8 @@ public void shouldDeleteAlias_positive() throws IOException { assertThat(internalClient, not(clusterContainsDocument(TEMPORARY_ALIAS_NAME, ID_S1))); } auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_READ_USER).withRestRequest(POST, "/_aliases")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_READ_USER, "IndicesAliasesRequest")); - auditLogsRule.assertExactly(4, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_READ_USER)); + auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_READ_USER, "IndicesAliasesRequest")); + auditLogsRule.assertExactly(2, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_READ_USER)); } @Test @@ -1415,7 +1409,6 @@ public void shouldDeleteAlias_negative() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_READ_USER, "IndicesAliasesRequest")); } - @Ignore("Create alias / delete alias isn't resolving in a timely manner for this test") @Test public void shouldCreateIndexTemplate_positive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { @@ -1439,12 +1432,12 @@ public void shouldCreateIndexTemplate_positive() throws IOException { } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/_template/musical-index-template")); auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/song-transcription-jazz/_doc/0001")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutIndexTemplateRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutIndexTemplateRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "IndexRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(8, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); + auditLogsRule.assertExactly(2, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test @@ -1477,9 +1470,9 @@ public void shouldDeleteTemplate_positive() throws IOException { } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/_template/musical-index-template")); auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(DELETE, "/_template/musical-index-template")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutIndexTemplateRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "DeleteIndexTemplateRequest")); - auditLogsRule.assertExactly(4, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutIndexTemplateRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "DeleteIndexTemplateRequest")); + auditLogsRule.assertExactly(2, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); } @Test @@ -1497,7 +1490,6 @@ public void shouldDeleteTemplate_negative() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_READ_USER, "DeleteIndexTemplateRequest")); } - @Ignore("Create alias / delete alias isn't resolving in a timely manner for this test") @Test public void shouldUpdateTemplate_positive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { @@ -1525,12 +1517,12 @@ public void shouldUpdateTemplate_positive() throws IOException { } auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/_template/musical-index-template")); auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/song-transcription-jazz/_doc/000one")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutIndexTemplateRequest")); + auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutIndexTemplateRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "IndexRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(10, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); + auditLogsRule.assertExactly(3, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test @@ -1620,7 +1612,7 @@ public void shouldCreateSnapshotRepository_positive() throws IOException { assertThat(internalClient, clusterContainsSnapshotRepository(TEST_SNAPSHOT_REPOSITORY_NAME)); } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/_snapshot/test-snapshot-repository")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); } @Test @@ -1656,8 +1648,8 @@ public void shouldDeleteSnapshotRepository_positive() throws IOException { auditLogsRule.assertExactlyOne( userAuthenticated(LIMITED_WRITE_USER).withRestRequest(DELETE, "/_snapshot/test-snapshot-repository") ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "DeleteRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "DeleteRepositoryRequest")); } @Test @@ -1674,9 +1666,10 @@ public void shouldDeleteSnapshotRepository_negative() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_READ_USER, "DeleteRepositoryRequest")); } - @Test // Bug which can be reproduced with the below test: https://github.com/opensearch-project/security/issues/2169 + @Test public void shouldCreateSnapshot_positive() throws IOException { final String snapshotName = "snapshot-positive-test"; + long snapshotGetCount; try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); steps.createSnapshotRepository(TEST_SNAPSHOT_REPOSITORY_NAME, cluster.getSnapshotDirPath(), "fs"); @@ -1685,20 +1678,21 @@ public void shouldCreateSnapshot_positive() throws IOException { assertThat(response, notNullValue()); assertThat(response.status(), equalTo(RestStatus.ACCEPTED)); - steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); + snapshotGetCount = steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); assertThat(internalClient, clusterContainSuccessSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName)); } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/_snapshot/test-snapshot-repository")); auditLogsRule.assertExactlyOne( userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/_snapshot/test-snapshot-repository/snapshot-positive-test") ); - auditLogsRule.assertAtLeast( - 1, - userAuthenticated(LIMITED_WRITE_USER).withRestRequest(GET, "/_snapshot/test-snapshot-repository/snapshot-positive-test") + auditLogsRule.assertExactly( + snapshotGetCount, + userAuthenticated(LIMITED_WRITE_USER).withEffectiveUser(LIMITED_WRITE_USER) + .withRestRequest(GET, "/_snapshot/test-snapshot-repository/snapshot-positive-test") ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); + auditLogsRule.assertAtLeast(snapshotGetCount, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); } @Test @@ -1723,12 +1717,13 @@ public void shouldCreateSnapshot_negative() throws IOException { @Test public void shouldDeleteSnapshot_positive() throws IOException { String snapshotName = "delete-snapshot-positive"; + long snapshotGetCount; try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); restHighLevelClient.snapshot(); steps.createSnapshotRepository(TEST_SNAPSHOT_REPOSITORY_NAME, cluster.getSnapshotDirPath(), "fs"); steps.createSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName, SONG_INDEX_NAME); - steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); + snapshotGetCount = steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); var response = steps.deleteSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); @@ -1742,24 +1737,25 @@ public void shouldDeleteSnapshot_positive() throws IOException { auditLogsRule.assertExactlyOne( userAuthenticated(LIMITED_WRITE_USER).withRestRequest(DELETE, "/_snapshot/test-snapshot-repository/delete-snapshot-positive") ); - auditLogsRule.assertAtLeast( - 1, + auditLogsRule.assertExactly( + snapshotGetCount, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(GET, "/_snapshot/test-snapshot-repository/delete-snapshot-positive") ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "DeleteSnapshotRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "DeleteSnapshotRequest")); + auditLogsRule.assertExactly(snapshotGetCount, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); } @Test public void shouldDeleteSnapshot_negative() throws IOException { String snapshotName = "delete-snapshot-negative"; + long snapshotGetCount; try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); steps.createSnapshotRepository(TEST_SNAPSHOT_REPOSITORY_NAME, cluster.getSnapshotDirPath(), "fs"); steps.createSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName, SONG_INDEX_NAME); - steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); + snapshotGetCount = steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); } try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_READ_USER)) { SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); @@ -1774,23 +1770,27 @@ public void shouldDeleteSnapshot_negative() throws IOException { auditLogsRule.assertExactlyOne( userAuthenticated(LIMITED_READ_USER).withRestRequest(DELETE, "/_snapshot/test-snapshot-repository/delete-snapshot-negative") ); - auditLogsRule.assertAtLeast( - 1, + auditLogsRule.assertExactly( + snapshotGetCount, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(GET, "/_snapshot/test-snapshot-repository/delete-snapshot-negative") ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); - auditLogsRule.assertExactly(1, missingPrivilege(LIMITED_READ_USER, "DeleteSnapshotRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); + auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_READ_USER, "DeleteSnapshotRequest")); + auditLogsRule.assertExactly(snapshotGetCount, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); } - @Ignore("Audit log entries verifcation isn't always consistant") @Test public void shouldRestoreSnapshot_positive() throws IOException { final String snapshotName = "restore-snapshot-positive"; + long snapshotGetCount; + AtomicInteger restoredCount = new AtomicInteger(); try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); // 1. create some documents + Settings indexSettings = Settings.builder().put("index.number_of_replicas", 0).put("index.number_of_shards", 1).build(); + IndexOperationsHelper.createIndex(cluster, WRITE_SONG_INDEX_NAME, indexSettings); + BulkRequest bulkRequest = new BulkRequest(); bulkRequest.add(new IndexRequest(WRITE_SONG_INDEX_NAME).id("Eins").source(SONGS[0].asMap())); bulkRequest.add(new IndexRequest(WRITE_SONG_INDEX_NAME).id("Zwei").source(SONGS[1].asMap())); @@ -1804,7 +1804,7 @@ public void shouldRestoreSnapshot_positive() throws IOException { steps.createSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName, WRITE_SONG_INDEX_NAME); // 4. wait till snapshot is ready - steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); + snapshotGetCount = steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); // 5. introduce some changes bulkRequest = new BulkRequest(); @@ -1824,8 +1824,12 @@ public void shouldRestoreSnapshot_positive() throws IOException { CountRequest countRequest = new CountRequest(RESTORED_SONG_INDEX_NAME); Awaitility.await() .ignoreExceptions() + .pollInterval(100, TimeUnit.MILLISECONDS) .alias("Index contains proper number of documents restored from snapshot.") - .until(() -> restHighLevelClient.count(countRequest, DEFAULT).getCount() == 2); + .until(() -> { + restoredCount.incrementAndGet(); + return restHighLevelClient.count(countRequest, DEFAULT).getCount() == 2; + }); // 8. verify that document are present in restored index assertThat( @@ -1849,28 +1853,33 @@ public void shouldRestoreSnapshot_positive() throws IOException { "/_snapshot/test-snapshot-repository/restore-snapshot-positive/_restore" ) ); - auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/restored_write_song_index/_count")); + auditLogsRule.assertExactly( + restoredCount.get(), + userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/restored_write_song_index/_count") + ); auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); - auditLogsRule.assertAtLeast( - 1, + auditLogsRule.assertExactly( + snapshotGetCount, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(GET, "/_snapshot/test-snapshot-repository/restore-snapshot-positive") ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "RestoreSnapshotRequest")); - auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "SearchRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "RestoreSnapshotRequest")); + auditLogsRule.assertExactly(restoredCount.get(), grantedPrivilege(LIMITED_WRITE_USER, "SearchRequest")); + auditLogsRule.assertExactly(snapshotGetCount, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test public void shouldRestoreSnapshot_failureForbiddenIndex() throws IOException { final String snapshotName = "restore-snapshot-negative-forbidden-index"; String restoreToIndex = "forbidden_index"; + long snapshotGetCount; + Settings indexSettings = Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build(); + IndexOperationsHelper.createIndex(cluster, WRITE_SONG_INDEX_NAME, indexSettings); try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { + SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); // 1. create some documents BulkRequest bulkRequest = new BulkRequest(); @@ -1886,7 +1895,7 @@ public void shouldRestoreSnapshot_failureForbiddenIndex() throws IOException { steps.createSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName, WRITE_SONG_INDEX_NAME); // 4. wait till snapshot is ready - steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); + snapshotGetCount = steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); // 5. restore the snapshot assertThatThrownBy( @@ -1912,27 +1921,28 @@ public void shouldRestoreSnapshot_failureForbiddenIndex() throws IOException { ) ); auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); - auditLogsRule.assertAtLeast( - 1, + auditLogsRule.assertExactly( + snapshotGetCount, userAuthenticated(LIMITED_WRITE_USER).withRestRequest( GET, "/_snapshot/test-snapshot-repository/restore-snapshot-negative-forbidden-index" ) ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "RestoreSnapshotRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); - auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_WRITE_USER, "RestoreSnapshotRequest")); + auditLogsRule.assertExactly(snapshotGetCount, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); + auditLogsRule.assertAtLeastTransportMessages(1, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactlyScanAll(1, missingPrivilege(LIMITED_WRITE_USER, "RestoreSnapshotRequest")); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test public void shouldRestoreSnapshot_failureOperationForbidden() throws IOException { String snapshotName = "restore-snapshot-negative-forbidden-operation"; + long snapshotGetCount; + Settings indexSettings = Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build(); + IndexOperationsHelper.createIndex(cluster, WRITE_SONG_INDEX_NAME, indexSettings); try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); // 1. create some documents @@ -1949,7 +1959,7 @@ public void shouldRestoreSnapshot_failureOperationForbidden() throws IOException steps.createSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName, WRITE_SONG_INDEX_NAME); // 4. wait till snapshot is ready - steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); + snapshotGetCount = steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); } // 5. restore the snapshot try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_READ_USER)) { @@ -1977,21 +1987,19 @@ public void shouldRestoreSnapshot_failureOperationForbidden() throws IOException ) ); auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); - auditLogsRule.assertAtLeast( - 1, + auditLogsRule.assertExactly( + snapshotGetCount, userAuthenticated(LIMITED_WRITE_USER).withRestRequest( GET, "/_snapshot/test-snapshot-repository/restore-snapshot-negative-forbidden-operation" ) ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(1, missingPrivilege(LIMITED_READ_USER, "RestoreSnapshotRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactlyScanAll(1, missingPrivilege(LIMITED_READ_USER, "RestoreSnapshotRequest")); + auditLogsRule.assertExactly(snapshotGetCount, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test @@ -2129,11 +2137,11 @@ public void shouldDeleteIndexByAliasRequest_positive() throws IOException { userAuthenticated(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES).withRestRequest(POST, "/_aliases") ); auditLogsRule.assertExactly( - 2, + 1, grantedPrivilege(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES, "IndicesAliasesRequest") ); auditLogsRule.assertExactly( - 2, + 1, auditPredicate(INDEX_EVENT).withEffectiveUser(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES) ); } @@ -2720,11 +2728,11 @@ public void shouldCreateIndexWithAlias_positive() throws IOException { ) ); auditLogsRule.assertExactly( - 2, + 1, grantedPrivilege(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES, "CreateIndexRequest") ); auditLogsRule.assertExactly( - 2, + 1, auditPredicate(INDEX_EVENT).withEffectiveUser(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES) ); } diff --git a/src/integrationTest/java/org/opensearch/security/SnapshotSteps.java b/src/integrationTest/java/org/opensearch/security/SnapshotSteps.java index 28aa6abd43..a03891ecca 100644 --- a/src/integrationTest/java/org/opensearch/security/SnapshotSteps.java +++ b/src/integrationTest/java/org/opensearch/security/SnapshotSteps.java @@ -11,6 +11,8 @@ import java.io.IOException; import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.awaitility.Awaitility; @@ -58,13 +60,21 @@ public CreateSnapshotResponse createSnapshot(String repositoryName, String snaps return snapshotClient.create(createSnapshotRequest, DEFAULT); } - public void waitForSnapshotCreation(String repositoryName, String snapshotName) { + public int waitForSnapshotCreation(String repositoryName, String snapshotName) { + AtomicInteger count = new AtomicInteger(); GetSnapshotsRequest getSnapshotsRequest = new GetSnapshotsRequest(repositoryName, new String[] { snapshotName }); - Awaitility.await().alias("wait for snapshot creation").ignoreExceptions().until(() -> { - GetSnapshotsResponse snapshotsResponse = snapshotClient.get(getSnapshotsRequest, DEFAULT); - SnapshotInfo snapshotInfo = snapshotsResponse.getSnapshots().get(0); - return SnapshotState.SUCCESS.equals(snapshotInfo.state()); - }); + Awaitility.await() + .pollDelay(250, TimeUnit.MILLISECONDS) + .pollInterval(2, TimeUnit.SECONDS) + .alias("wait for snapshot creation") + .ignoreExceptions() + .until(() -> { + count.incrementAndGet(); + GetSnapshotsResponse snapshotsResponse = snapshotClient.get(getSnapshotsRequest, DEFAULT); + SnapshotInfo snapshotInfo = snapshotsResponse.getSnapshots().get(0); + return SnapshotState.SUCCESS.equals(snapshotInfo.state()); + }); + return count.get(); } // CS-SUPPRESS-SINGLE: RegexpSingleline It is not possible to use phrase "cluster manager" instead of master here diff --git a/src/integrationTest/java/org/opensearch/security/rest/WhoAmITests.java b/src/integrationTest/java/org/opensearch/security/rest/WhoAmITests.java index 2652a5e2a6..c41b5f4cda 100644 --- a/src/integrationTest/java/org/opensearch/security/rest/WhoAmITests.java +++ b/src/integrationTest/java/org/opensearch/security/rest/WhoAmITests.java @@ -172,7 +172,7 @@ public void testAuditLogSimilarityWithTransportLayer() { assertThat(client.get("_cat/indices").getStatusCode(), equalTo(HttpStatus.SC_OK)); // transport layer audit messages - auditLogsRule.assertExactly(2, grantedPrivilege(AUDIT_LOG_VERIFIER, "GetSettingsRequest")); + auditLogsRule.assertExactly(1, grantedPrivilege(AUDIT_LOG_VERIFIER, "GetSettingsRequest")); List grantedPrivilegesMessages = auditLogsRule.getCurrentTestAuditMessages() .stream() diff --git a/src/integrationTest/java/org/opensearch/test/framework/audit/AuditLogsRule.java b/src/integrationTest/java/org/opensearch/test/framework/audit/AuditLogsRule.java index 3d13d731eb..3f9a0ae466 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/audit/AuditLogsRule.java +++ b/src/integrationTest/java/org/opensearch/test/framework/audit/AuditLogsRule.java @@ -39,6 +39,7 @@ public class AuditLogsRule implements TestRule { private static final Logger log = LogManager.getLogger(AuditLogsRule.class); private List currentTestAuditMessages; + private List currentTransportTestAuditMessages; public List getCurrentTestAuditMessages() { return currentTestAuditMessages; @@ -56,6 +57,7 @@ public void waitForAuditLogs() { private void afterWaitingForAuditLogs() { if (log.isDebugEnabled()) { log.debug("Audit records captured during test:\n{}", auditMessagesToString(currentTestAuditMessages)); + log.debug("Audit transport records captured during test:\n{}", auditMessagesToString(currentTransportTestAuditMessages)); } } @@ -63,6 +65,13 @@ public void assertExactlyOne(Predicate predicate) { assertExactly(1, predicate); } + public void assertExactlyScanAll(long expectedNumberOfAuditMessages, Predicate predicate) { + List auditMessages = new ArrayList<>(currentTestAuditMessages); + auditMessages.addAll(currentTransportTestAuditMessages); + assertExactly(exactNumberOfAuditsFulfillPredicate(expectedNumberOfAuditMessages, predicate), auditMessages); + + } + public void assertAuditLogsCount(int from, int to) { int actualCount = currentTestAuditMessages.size(); String message = "Expected audit log count is between " + from + " and " + to + " but was " + actualCount; @@ -70,10 +79,10 @@ public void assertAuditLogsCount(int from, int to) { } public void assertExactly(long expectedNumberOfAuditMessages, Predicate predicate) { - assertExactly(exactNumberOfAuditsFulfillPredicate(expectedNumberOfAuditMessages, predicate)); + assertExactly(exactNumberOfAuditsFulfillPredicate(expectedNumberOfAuditMessages, predicate), currentTestAuditMessages); } - private void assertExactly(Matcher> matcher) { + private void assertExactly(Matcher> matcher, List currentTestAuditMessages) { // pollDelay - initial delay before first evaluation Awaitility.await("Await for audit logs") .atMost(3, TimeUnit.SECONDS) @@ -82,7 +91,11 @@ private void assertExactly(Matcher> matcher) { } public void assertAtLeast(long minCount, Predicate predicate) { - assertExactly(atLeastCertainNumberOfAuditsFulfillPredicate(minCount, predicate)); + assertExactly(atLeastCertainNumberOfAuditsFulfillPredicate(minCount, predicate), currentTestAuditMessages); + } + + public void assertAtLeastTransportMessages(long minCount, Predicate predicate) { + assertExactly(atLeastCertainNumberOfAuditsFulfillPredicate(minCount, predicate), currentTransportTestAuditMessages); } private static String auditMessagesToString(List audits) { @@ -122,16 +135,35 @@ private void whenTimeoutOccurs(String methodName) { private void afterTest() { TestRuleAuditLogSink.unregisterListener(); this.currentTestAuditMessages = null; + this.currentTransportTestAuditMessages = null; } private void beforeTest(String methodName) { log.info("Start collecting audit logs before test {}", methodName); this.currentTestAuditMessages = synchronizedList(new ArrayList<>()); + this.currentTransportTestAuditMessages = synchronizedList(new ArrayList<>()); TestRuleAuditLogSink.registerListener(this); } public void onAuditMessage(AuditMessage auditMessage) { - currentTestAuditMessages.add(auditMessage); - log.debug("New audit message received '{}', total number of audit messages '{}'.", auditMessage, currentTestAuditMessages.size()); + if (auditMessage.getAsMap().keySet().contains("audit_transport_headers")) { + if (log.isDebugEnabled()) { + log.debug( + "New transport audit message received '{}', total number of transport audit messages '{}'.", + auditMessage, + currentTransportTestAuditMessages.size() + ); + } + currentTransportTestAuditMessages.add(auditMessage); + } else { + if (log.isDebugEnabled()) { + log.debug( + "New audit message received '{}', total number of audit messages '{}'.", + auditMessage, + currentTestAuditMessages.size() + ); + } + currentTestAuditMessages.add(auditMessage); + } } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java index b996b3f66f..1a22d13ac7 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java @@ -74,6 +74,15 @@ public enum ClusterManager { new NodeSettings(NodeRole.CLUSTER_MANAGER), new NodeSettings(NodeRole.DATA), new NodeSettings(NodeRole.DATA) + ), + + THREE_CLUSTER_MANAGERS_COORDINATOR( + new NodeSettings(NodeRole.CLUSTER_MANAGER), + new NodeSettings(NodeRole.CLUSTER_MANAGER), + new NodeSettings(NodeRole.CLUSTER_MANAGER), + new NodeSettings(NodeRole.DATA), + new NodeSettings(NodeRole.DATA), + new NodeSettings() ); private List nodeSettings = new LinkedList<>(); From 7498eb00679c307b75db80f54476a01ace48fae9 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 14 Dec 2023 13:06:42 -0500 Subject: [PATCH 052/204] Fix flaky test SlowIntegrationTests.testDelayInSecurityIndexInitialization (#3763) Signed-off-by: Craig Perkins --- DEVELOPER_GUIDE.md | 23 ++++++++- build.gradle | 3 ++ .../security/SlowIntegrationTests.java | 47 ++++++++++++++++++ .../org/opensearch/security/util/Repeat.java | 25 ++++++++++ .../opensearch/security/util/RepeatRule.java | 48 +++++++++++++++++++ 5 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 src/test/java/org/opensearch/security/util/Repeat.java create mode 100644 src/test/java/org/opensearch/security/util/RepeatRule.java diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 5e797c98d1..e4a3834046 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -232,11 +232,30 @@ curl -XGET https://localhost:9200/_plugins/_security/authinfo -u 'admin:admin' - Launch IntelliJ IDEA, choose **Project from Existing Sources**, and select directory with Gradle build script (`build.gradle`). -## Running integration tests +## Running tests Locally these can be run with `./gradlew test` with detailed results being available at `${project-root}/build/reports/tests/test/index.html`. You can also run tests through an IDEs JUnit test runner. -Integration tests are automatically run on all pull requests for all supported versions of the JDK. These must pass for change(s) to be merged. Detailed logs of these test results are available by going to the GitHub Actions workflow summary view and downloading the workflow run of the tests. If you see multiple tests listed with different JDK versions, you can download the version with whichever JDK you are interested in. After extracting the test file on your local machine, integration tests results can be found at `./tests/tests/index.html`. +Tests are automatically run on all pull requests for all supported versions of the JDK. These must pass for change(s) to be merged. Detailed logs of these test results are available by going to the GitHub Actions workflow summary view and downloading the workflow run of the tests. If you see multiple tests listed with different JDK versions, you can download the version with whichever JDK you are interested in. After extracting the test file on your local machine, integration tests results can be found at `./tests/tests/index.html`. + +### Running an individual test multiple times + +This repo has a `@Repeat` annotation which you can import to annotate a test to run many times repeatedly. To use the annotation, add the following code to your test suite. + +``` +@Rule +public RepeatRule repeatRule = new RepeatRule(); + +@Test +@Repeat(10) +public void testMethod() { + ... +} +``` + +## Running tests in the integrationTest package + +Tests in the integrationTest package can be run with `./gradlew integrationTest`. ### Bulk test runs diff --git a/build.gradle b/build.gradle index 4b22f1665f..4f38953d58 100644 --- a/build.gradle +++ b/build.gradle @@ -687,6 +687,9 @@ dependencies { testImplementation "org.springframework:spring-beans:${spring_version}" testImplementation 'org.junit.jupiter:junit-jupiter:5.10.1' testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.1' + testImplementation('org.awaitility:awaitility:4.2.0') { + exclude(group: 'org.hamcrest', module: 'hamcrest') + } // Only osx-x86_64, osx-aarch_64, linux-x86_64, linux-aarch_64, windows-x86_64 are available if (osdetector.classifier in ["osx-x86_64", "osx-aarch_64", "linux-x86_64", "linux-aarch_64", "windows-x86_64"]) { testImplementation "io.netty:netty-tcnative-classes:2.0.61.Final" diff --git a/src/test/java/org/opensearch/security/SlowIntegrationTests.java b/src/test/java/org/opensearch/security/SlowIntegrationTests.java index 0c264ad440..eb147ec422 100644 --- a/src/test/java/org/opensearch/security/SlowIntegrationTests.java +++ b/src/test/java/org/opensearch/security/SlowIntegrationTests.java @@ -26,14 +26,20 @@ package org.opensearch.security; +import java.io.IOException; + import com.google.common.collect.Lists; +import org.awaitility.Awaitility; import org.junit.Assert; import org.junit.Test; import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; import org.opensearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; import org.opensearch.node.Node; import org.opensearch.node.PluginAwareNode; import org.opensearch.security.ssl.util.SSLConfigConstants; @@ -42,8 +48,12 @@ import org.opensearch.security.test.SingleClusterTest; import org.opensearch.security.test.helper.cluster.ClusterConfiguration; import org.opensearch.security.test.helper.file.FileHelper; +import org.opensearch.security.test.helper.rest.RestHelper; import org.opensearch.transport.Netty4ModulePlugin; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertThrows; + public class SlowIntegrationTests extends SingleClusterTest { @Test @@ -210,4 +220,41 @@ public void testNodeClientDisallowedWithNonServerCertificate2() throws Exception Assert.fail(e.toString()); } } + + @Test + public void testDelayInSecurityIndexInitialization() throws Exception { + final Settings settings = Settings.builder() + .put(ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, true) + .put("cluster.routing.allocation.exclude._ip", "127.0.0.1") + .build(); + assertThrows(IOException.class, () -> { + setup(Settings.EMPTY, null, settings, false); + clusterHelper.nodeClient() + .admin() + .indices() + .create(new CreateIndexRequest("test-index").timeout(TimeValue.timeValueSeconds(10))) + .actionGet(); + clusterHelper.waitForCluster(ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(5), ClusterConfiguration.DEFAULT.getNodes()); + }); + // Ideally, we would want to remove this cluster setting, but default settings cannot be removed. So overriding with a reserved + // IP address + clusterHelper.nodeClient() + .admin() + .cluster() + .updateSettings( + new ClusterUpdateSettingsRequest().transientSettings( + Settings.builder().put("cluster.routing.allocation.exclude._ip", "192.0.2.0").build() + ) + ); + this.clusterInfo = clusterHelper.waitForCluster(ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(10), 3); + RestHelper rh = nonSslRestHelper(); + Awaitility.await() + .alias("Wait until Security is initialized") + .until( + () -> rh.executeGetRequest("/_plugins/_security/health", encodeBasicHeader("admin", "admin")) + .getTextFromJsonBody("/status"), + equalTo("UP") + ); + } + } diff --git a/src/test/java/org/opensearch/security/util/Repeat.java b/src/test/java/org/opensearch/security/util/Repeat.java new file mode 100644 index 0000000000..be9c9fddf0 --- /dev/null +++ b/src/test/java/org/opensearch/security/util/Repeat.java @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.util; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.METHOD; + +@Retention(RetentionPolicy.RUNTIME) +@Target({ METHOD, ANNOTATION_TYPE }) +public @interface Repeat { + int value() default 1; +} diff --git a/src/test/java/org/opensearch/security/util/RepeatRule.java b/src/test/java/org/opensearch/security/util/RepeatRule.java new file mode 100644 index 0000000000..387f0c17e5 --- /dev/null +++ b/src/test/java/org/opensearch/security/util/RepeatRule.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.util; + +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; + +public class RepeatRule implements TestRule { + + private static class RepeatStatement extends Statement { + private final Statement statement; + private final int repeat; + + public RepeatStatement(Statement statement, int repeat) { + this.statement = statement; + this.repeat = repeat; + } + + @Override + public void evaluate() throws Throwable { + for (int i = 0; i < repeat; i++) { + statement.evaluate(); + } + } + + } + + @Override + public Statement apply(Statement statement, Description description) { + Statement result = statement; + Repeat repeat = description.getAnnotation(Repeat.class); + if (repeat != null) { + int times = repeat.value(); + result = new RepeatStatement(statement, times); + } + return result; + } +} From 62aed210be664492783a317f6996185e3bb7487b Mon Sep 17 00:00:00 2001 From: Darshit Chanpura <35282393+DarshitChanpura@users.noreply.github.com> Date: Thu, 14 Dec 2023 17:45:54 -0500 Subject: [PATCH 053/204] Removes support for txt file for specifying initial admin password and prevents password from being printed in the logs (#3850) Signed-off-by: Darshit Chanpura --- .../action.yml | 11 ++- .../security/support/ConfigConstants.java | 3 +- .../security/tools/democonfig/Installer.java | 4 +- .../SecuritySettingsConfigurer.java | 76 ++++++++----------- .../tools/democonfig/InstallerTests.java | 8 +- .../SecuritySettingsConfigurerTests.java | 35 ++------- .../util/NoExitSecurityManager.java | 5 +- 7 files changed, 52 insertions(+), 90 deletions(-) diff --git a/.github/actions/start-opensearch-with-one-plugin/action.yml b/.github/actions/start-opensearch-with-one-plugin/action.yml index 8513e64fdf..e8e0f4eb77 100644 --- a/.github/actions/start-opensearch-with-one-plugin/action.yml +++ b/.github/actions/start-opensearch-with-one-plugin/action.yml @@ -71,23 +71,22 @@ runs: 'y' | .\opensearch-${{ inputs.opensearch-version }}-SNAPSHOT\bin\opensearch-plugin.bat install file:$(pwd)\${{ inputs.plugin-name }}.zip shell: pwsh - - name: Write password to opensearch_initial_admin_password txt - run: - echo ${{ inputs.admin-password }} >> ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/config/opensearch_initial_admin_password.txt - shell: bash - # Run any configuration scripts - name: Run Setup Script for Linux if: ${{ runner.os == 'Linux' && inputs.setup-script-name != '' }} run: | echo "running linux setup" + export OPENSEARCH_INITIAL_ADMIN_PASSWORD=${{ inputs.admin-password }} chmod +x ./${{ inputs.setup-script-name }}.sh ./${{ inputs.setup-script-name }}.sh shell: bash - name: Run Setup Script for Windows if: ${{ runner.os == 'Windows' && inputs.setup-script-name != '' }} - run: .\${{ inputs.setup-script-name }}.bat + run: | + echo "running windows setup" + $env:OPENSEARCH_INITIAL_ADMIN_PASSWORD="${{ inputs.admin-password }}" + .\${{ inputs.setup-script-name }}.bat shell: pwsh # Run OpenSearch diff --git a/src/main/java/org/opensearch/security/support/ConfigConstants.java b/src/main/java/org/opensearch/security/support/ConfigConstants.java index f106466984..f10dedade3 100644 --- a/src/main/java/org/opensearch/security/support/ConfigConstants.java +++ b/src/main/java/org/opensearch/security/support/ConfigConstants.java @@ -334,9 +334,8 @@ public enum RolesMappingResolution { public static final boolean EXTENSIONS_BWC_PLUGIN_MODE_DEFAULT = false; // CS-ENFORCE-SINGLE - // Variables for initial admin password support + // Variable for initial admin password support public static final String OPENSEARCH_INITIAL_ADMIN_PASSWORD = "OPENSEARCH_INITIAL_ADMIN_PASSWORD"; - public static final String OPENSEARCH_INITIAL_ADMIN_PASSWORD_TXT = "opensearch_initial_admin_password.txt"; public static Set getSettingAsSet( final Settings settings, diff --git a/src/main/java/org/opensearch/security/tools/democonfig/Installer.java b/src/main/java/org/opensearch/security/tools/democonfig/Installer.java index 68bc4d7cbf..61acd7e4c9 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/Installer.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/Installer.java @@ -428,9 +428,7 @@ void finishScriptExecution() { } System.out.println( - "### To access your secured cluster open https://: and log in with admin/" - + SecuritySettingsConfigurer.ADMIN_PASSWORD - + "." + "### To access your secured cluster open https://: and log in with admin/." ); System.out.println("### (Ignore the SSL certificate warning because we installed self-signed demo certificates)"); diff --git a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java index 28da25c592..b3644e6c4d 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java @@ -28,6 +28,7 @@ import com.fasterxml.jackson.databind.JsonNode; import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.Strings; import org.opensearch.security.DefaultObjectMapper; import org.opensearch.security.dlic.rest.validation.PasswordValidator; import org.opensearch.security.dlic.rest.validation.RequestContentValidator; @@ -91,12 +92,34 @@ public void configureSecuritySettings() { writeSecurityConfigToOpenSearchYML(); } + /** + * Checks if security plugin is already configured. If so, the script execution will exit. + */ + void checkIfSecurityPluginIsAlreadyConfigured() { + // Check if the configuration file contains the 'plugins.security' string + if (installer.OPENSEARCH_CONF_FILE != null && new File(installer.OPENSEARCH_CONF_FILE).exists()) { + try (BufferedReader br = new BufferedReader(new FileReader(installer.OPENSEARCH_CONF_FILE, StandardCharsets.UTF_8))) { + String line; + while ((line = br.readLine()) != null) { + if (line.toLowerCase().contains("plugins.security")) { + System.out.println(installer.OPENSEARCH_CONF_FILE + " seems to be already configured for Security. Quit."); + System.exit(installer.skip_updates); + } + } + } catch (IOException e) { + System.err.println("Error reading configuration file."); + System.exit(-1); + } + } else { + System.err.println("OpenSearch configuration file does not exist. Quit."); + System.exit(-1); + } + } + /** * Replaces the admin password in internal_users.yml with the custom or generated password */ void updateAdminPassword() { - String initialAdminPassword = System.getenv().get(ConfigConstants.OPENSEARCH_INITIAL_ADMIN_PASSWORD); - String ADMIN_PASSWORD_FILE_PATH = installer.OPENSEARCH_CONF_DIR + ConfigConstants.OPENSEARCH_INITIAL_ADMIN_PASSWORD_TXT; String INTERNAL_USERS_FILE_PATH = installer.OPENSEARCH_CONF_DIR + "opensearch-security" + File.separator + "internal_users.yml"; boolean shouldValidatePassword = installer.environment.equals(ExecutionEnvironment.DEMO); try { @@ -107,21 +130,10 @@ void updateAdminPassword() { .build() ); - // Read custom password - if (initialAdminPassword != null && !initialAdminPassword.isEmpty()) { + // Read custom password from environment variable + String initialAdminPassword = System.getenv().get(ConfigConstants.OPENSEARCH_INITIAL_ADMIN_PASSWORD); + if (!Strings.isNullOrEmpty(initialAdminPassword)) { ADMIN_PASSWORD = initialAdminPassword; - } else { - File adminPasswordFile = new File(ADMIN_PASSWORD_FILE_PATH); - if (adminPasswordFile.exists() && adminPasswordFile.length() > 0) { - try (BufferedReader br = new BufferedReader(new FileReader(ADMIN_PASSWORD_FILE_PATH, StandardCharsets.UTF_8))) { - ADMIN_PASSWORD = br.readLine(); - } catch (IOException e) { - System.out.println( - "Error reading admin password from " + ConfigConstants.OPENSEARCH_INITIAL_ADMIN_PASSWORD_TXT + "." - ); - System.exit(-1); - } - } } // If script execution environment is set to demo, validate custom password, else if set to test, skip validation @@ -133,15 +145,13 @@ void updateAdminPassword() { } // if ADMIN_PASSWORD is still an empty string, it implies no custom password was provided. We exit the setup. - if (ADMIN_PASSWORD.isEmpty()) { + if (Strings.isNullOrEmpty(ADMIN_PASSWORD)) { System.out.println("No custom admin password found. Please provide a password."); System.exit(-1); } - // print the password to the logs - System.out.println("\t***************************************************"); - System.out.println("\t\tADMIN PASSWORD SET TO: " + ADMIN_PASSWORD); - System.out.println("\t***************************************************"); + // Print an update to the logs + System.out.println("Admin password set successfully."); writePasswordToInternalUsersFile(ADMIN_PASSWORD, INTERNAL_USERS_FILE_PATH); @@ -188,30 +198,6 @@ void writePasswordToInternalUsersFile(String adminPassword, String internalUsers Files.move(tempFilePath, internalUsersPath, java.nio.file.StandardCopyOption.REPLACE_EXISTING); } - /** - * Checks if security plugin is already configured. If so, the script execution will not continue. - */ - void checkIfSecurityPluginIsAlreadyConfigured() { - // Check if the configuration file contains the 'plugins.security' string - if (installer.OPENSEARCH_CONF_FILE != null && new File(installer.OPENSEARCH_CONF_FILE).exists()) { - try (BufferedReader br = new BufferedReader(new FileReader(installer.OPENSEARCH_CONF_FILE, StandardCharsets.UTF_8))) { - String line; - while ((line = br.readLine()) != null) { - if (line.toLowerCase().contains("plugins.security")) { - System.out.println(installer.OPENSEARCH_CONF_FILE + " seems to be already configured for Security. Quit."); - System.exit(installer.skip_updates); - } - } - } catch (IOException e) { - System.err.println("Error reading configuration file."); - System.exit(-1); - } - } else { - System.err.println("OpenSearch configuration file does not exist. Quit."); - System.exit(-1); - } - } - /** * Update opensearch.yml with security configuration information */ diff --git a/src/test/java/org/opensearch/security/tools/democonfig/InstallerTests.java b/src/test/java/org/opensearch/security/tools/democonfig/InstallerTests.java index 4e64fdea72..06c6edf734 100644 --- a/src/test/java/org/opensearch/security/tools/democonfig/InstallerTests.java +++ b/src/test/java/org/opensearch/security/tools/democonfig/InstallerTests.java @@ -394,9 +394,7 @@ public void testFinishScriptExecution() { + System.lineSeparator() + "### After that you can also use the Security Plugin ConfigurationGUI" + System.lineSeparator() - + "### To access your secured cluster open https://: and log in with admin/" - + SecuritySettingsConfigurer.ADMIN_PASSWORD - + "." + + "### To access your secured cluster open https://: and log in with admin/." + System.lineSeparator() + "### (Ignore the SSL certificate warning because we installed self-signed demo certificates)" + System.lineSeparator(); @@ -454,9 +452,7 @@ public void testFinishScriptExecution_withInitSecurityEnabled() { + System.lineSeparator() + "### To use the Security Plugin ConfigurationGUI" + System.lineSeparator() - + "### To access your secured cluster open https://: and log in with admin/" - + SecuritySettingsConfigurer.ADMIN_PASSWORD - + "." + + "### To access your secured cluster open https://: and log in with admin/." + System.lineSeparator() + "### (Ignore the SSL certificate warning because we installed self-signed demo certificates)" + System.lineSeparator(); diff --git a/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java b/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java index cb36ba0d6c..948a66996c 100644 --- a/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java +++ b/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java @@ -13,11 +13,9 @@ // CS-SUPPRESS-SINGLE: RegexpSingleline extension key-word is used in file ext variable import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileReader; -import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; @@ -45,7 +43,6 @@ import static org.opensearch.security.tools.democonfig.util.DemoConfigHelperUtil.createDirectory; import static org.opensearch.security.tools.democonfig.util.DemoConfigHelperUtil.createFile; import static org.opensearch.security.tools.democonfig.util.DemoConfigHelperUtil.deleteDirectoryRecursive; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) @@ -79,42 +76,25 @@ public void tearDown() throws NoSuchFieldException, IllegalAccessException { System.setErr(originalErr); System.setIn(originalIn); deleteDirectoryRecursive(installer.OPENSEARCH_CONF_DIR); - unsetEnv(adminPasswordKey); + unsetEnvVariables(); Installer.resetInstance(); } @Test public void testUpdateAdminPasswordWithCustomPassword() throws NoSuchFieldException, IllegalAccessException { - String customPassword = "myStrongPassword123"; // generateStrongPassword(); + String customPassword = "myStrongPassword123"; setEnv(adminPasswordKey, customPassword); securitySettingsConfigurer.updateAdminPassword(); assertThat(customPassword, is(equalTo(SecuritySettingsConfigurer.ADMIN_PASSWORD))); - verifyStdOutContainsString("ADMIN PASSWORD SET TO: " + customPassword); - } - - @Test - public void testUpdateAdminPasswordWithFilePassword() throws IOException { - String customPassword = "myStrongPassword123"; - String initialAdminPasswordTxt = installer.OPENSEARCH_CONF_DIR + adminPasswordKey + ".txt"; - createFile(initialAdminPasswordTxt); - - try (BufferedWriter writer = new BufferedWriter(new FileWriter(initialAdminPasswordTxt, StandardCharsets.UTF_8))) { - writer.write(customPassword); - } catch (IOException e) { - throw new IOException("Unable to update the internal users file with the hashed password."); - } - - securitySettingsConfigurer.updateAdminPassword(); - - assertEquals(customPassword, SecuritySettingsConfigurer.ADMIN_PASSWORD); - verifyStdOutContainsString("ADMIN PASSWORD SET TO: " + customPassword); + verifyStdOutContainsString("Admin password set successfully."); } @Test public void testUpdateAdminPassword_noPasswordSupplied() { + SecuritySettingsConfigurer.ADMIN_PASSWORD = ""; // to ensure 0 flaky-ness try { System.setSecurityManager(new NoExitSecurityManager()); securitySettingsConfigurer.updateAdminPassword(); @@ -150,7 +130,8 @@ public void testUpdateAdminPasswordWithWeakPassword_skipPasswordValidation() thr securitySettingsConfigurer.updateAdminPassword(); assertThat("weakpassword", is(equalTo(SecuritySettingsConfigurer.ADMIN_PASSWORD))); - verifyStdOutContainsString("ADMIN PASSWORD SET TO: weakpassword"); + + verifyStdOutContainsString("Admin password set successfully."); } @Test @@ -298,7 +279,7 @@ public static void setEnv(String key, String value) throws NoSuchFieldException, } @SuppressWarnings("unchecked") - public static void unsetEnv(String key) throws NoSuchFieldException, IllegalAccessException { + public static void unsetEnvVariables() throws NoSuchFieldException, IllegalAccessException { Class[] classes = Collections.class.getDeclaredClasses(); Map env = System.getenv(); for (Class cl : classes) { @@ -307,7 +288,7 @@ public static void unsetEnv(String key) throws NoSuchFieldException, IllegalAcce field.setAccessible(true); Object obj = field.get(env); Map map = (Map) obj; - map.remove(key); + map.clear(); } } } diff --git a/src/test/java/org/opensearch/security/tools/democonfig/util/NoExitSecurityManager.java b/src/test/java/org/opensearch/security/tools/democonfig/util/NoExitSecurityManager.java index 16bb61f169..0602812f5d 100644 --- a/src/test/java/org/opensearch/security/tools/democonfig/util/NoExitSecurityManager.java +++ b/src/test/java/org/opensearch/security/tools/democonfig/util/NoExitSecurityManager.java @@ -11,10 +11,13 @@ package org.opensearch.security.tools.democonfig.util; +/** + * Helper class to allow capturing and testing exit codes and block test execution from exiting mid-way + */ public class NoExitSecurityManager extends SecurityManager { @Override public void checkPermission(java.security.Permission perm) { - // Allow everything except System.exit code 0 &b -1 + // Allow everything except System.exit code 0 & -1 if (perm instanceof java.lang.RuntimePermission && ("exitVM.0".equals(perm.getName()) || "exitVM.-1".equals(perm.getName()))) { StringBuilder sb = new StringBuilder(); sb.append("System.exit("); From ceabe13b94dcd6dfd583d95cd43badeaad39be51 Mon Sep 17 00:00:00 2001 From: Darshit Chanpura <35282393+DarshitChanpura@users.noreply.github.com> Date: Fri, 15 Dec 2023 11:50:36 -0500 Subject: [PATCH 054/204] Updates developer guide and removes integtest.sh (#3844) Signed-off-by: Darshit Chanpura Signed-off-by: Darshit Chanpura <35282393+DarshitChanpura@users.noreply.github.com> --- DEVELOPER_GUIDE.md | 18 ++++++-- README.md | 4 -- scripts/integtest.sh | 105 ------------------------------------------- 3 files changed, 15 insertions(+), 112 deletions(-) delete mode 100755 scripts/integtest.sh diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index e4a3834046..472f3f06a9 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -162,6 +162,18 @@ extension_hw_greet: - "hw-user" ``` +### Setting up password for demo admin user + +This step is a pre-requisite to installing demo configuration. You can pass the demo `admin` user password by exporting `OPENSEARCH_INITIAL_ADMIN_PASSWORD` variable with a password. +```shell +export OPENSEARCH_INITIAL_ADMIN_PASSWORD= +``` + +**_Note:_** If no password is supplied, the installation will fail. The password supplied will also be tested for its strength and will be blocked if it is too simple. There is an option to skip this password validation by passing the `-t` option to the installation script. However, this should only be used for test environments. + + +### Executing the demo installation script + To install the demo certificates and default configuration, answer `y` to the first two questions and `n` to the last one. The log should look like below: ```bash @@ -192,17 +204,17 @@ Detected OpenSearch Security Version: * "/Users/XXXXX/Test/opensearch-*/plugins/opensearch-security/tools/securityadmin.sh" -cd "/Users/XXXXX/Test/opensearch-*/config/opensearch-security/" -icl -key "/Users/XXXXX/Test/opensearch-*/config/kirk-key.pem" -cert "/Users/XXXXX/Test/opensearch-*/config/kirk.pem" -cacert "/Users/XXXXX/Test/opensearch-*/config/root-ca.pem" -nhnv ### or run ./securityadmin_demo.sh ### To use the Security Plugin ConfigurationGUI -### To access your secured cluster open https://: and log in with admin/admin. +### To access your secured cluster open https://: and log in with admin/. ### (Ignore the SSL certificate warning because we installed self-signed demo certificates) ``` Now if we start our server again and try the original `curl localhost:9200`, it will fail. -Try this command instead: `curl -XGET https://localhost:9200 -u 'admin:admin' --insecure`. It should succeed. +Try this command instead: `curl -XGET https://localhost:9200 -u 'admin:' --insecure`. It should succeed. You can also make this call to return the authenticated user details: ```bash -curl -XGET https://localhost:9200/_plugins/_security/authinfo -u 'admin:admin' --insecure +curl -XGET https://localhost:9200/_plugins/_security/authinfo -u 'admin:' --insecure { "user": "User [name=admin, backend_roles=[admin], requestedTenant=null]", diff --git a/README.md b/README.md index 1aed7a5a0b..fe698a12d9 100644 --- a/README.md +++ b/README.md @@ -83,10 +83,6 @@ Run tests against local cluster: ```bash ./gradlew integTestRemote -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Dsecurity=true -Dhttps=true -Duser=admin -Dpassword=admin -Dcommon_utils.version="2.2.0.0" ``` -OR -```bash -./scripts/integtest.sh -``` Note: To run against a remote cluster replace cluster-name and `localhost:9200` with the IPAddress:Port of that cluster. Build artifacts (zip, deb, rpm): diff --git a/scripts/integtest.sh b/scripts/integtest.sh deleted file mode 100755 index 98ee40fbd6..0000000000 --- a/scripts/integtest.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/bash - -set -e - -function usage() { - echo "" - echo "This script is used to run integration tests for plugin installed on a remote OpenSearch/Dashboards cluster." - echo "--------------------------------------------------------------------------" - echo "Usage: $0 [args]" - echo "" - echo "Required arguments:" - echo "None" - echo "" - echo "Optional arguments:" - echo -e "-b BIND_ADDRESS\t, defaults to localhost | 127.0.0.1, can be changed to any IP or domain name for the cluster location." - echo -e "-p BIND_PORT\t, defaults to 9200, can be changed to any port for the cluster location." - echo -e "-s SECURITY_ENABLED\t(true | false), defaults to true. Specify the OpenSearch/Dashboards have security enabled or not." - echo -e "-c CREDENTIAL\t(usename:password), no defaults, effective when SECURITY_ENABLED=true." - echo -e "-h\tPrint this message." - echo -e "-v OPENSEARCH_VERSION\t, no defaults" - echo -e "-n SNAPSHOT\t, defaults to false" - echo -e "-m CLUSTER_NAME\t, defaults to docker-cluster" - echo "--------------------------------------------------------------------------" -} - -while getopts ":h:b:p:s:c:v:n:t:m:u:" arg; do - case $arg in - h) - usage - exit 1 - ;; - b) - BIND_ADDRESS=$OPTARG - ;; - p) - BIND_PORT=$OPTARG - ;; - t) - TRANSPORT_PORT=$OPTARG - ;; - s) - SECURITY_ENABLED=$OPTARG - ;; - c) - CREDENTIAL=$OPTARG - ;; - m) - CLUSTER_NAME=$OPTARG - ;; - v) - # Do nothing as we're not consuming this param. - ;; - n) - # Do nothing as we're not consuming this param. - ;; - u) - COMMON_UTILS_VERSION=$OPTARG - ;; - :) - echo "-${OPTARG} requires an argument" - usage - exit 1 - ;; - ?) - echo "Invalid option: -${OPTARG}" - exit 1 - ;; - esac -done - - -if [ -z "$BIND_ADDRESS" ] -then - BIND_ADDRESS="localhost" -fi - -if [ -z "$BIND_PORT" ] -then - BIND_PORT="9200" -fi - -if [ -z "$SECURITY_ENABLED" ] -then - SECURITY_ENABLED="true" -fi - -if [ -z "$CREDENTIAL" ] -then - CREDENTIAL="admin:admin" -fi - -if [ -z "$CREDENTIAL" ] -then - CREDENTIAL="admin:admin" -fi - -if [ -z "$CLUSTER_NAME" ] -then - CLUSTER_NAME="docker-cluster" -fi - -USERNAME=`echo $CREDENTIAL | awk -F ':' '{print $1}'` -PASSWORD=`echo $CREDENTIAL | awk -F ':' '{print $2}'` - -./gradlew integTestRemote -Dtests.rest.cluster="$BIND_ADDRESS:$BIND_PORT" -Dtests.cluster="$BIND_ADDRESS:$BIND_PORT" -Dsecurity_enabled=$SECURITY_ENABLED -Dtests.clustername=$CLUSTER_NAME -Dhttps=true -Duser=$USERNAME -Dpassword=$PASSWORD From b996eb15640c56d657758a58279578e38c047202 Mon Sep 17 00:00:00 2001 From: Amit Galitzky Date: Fri, 15 Dec 2023 12:45:12 -0800 Subject: [PATCH 055/204] Add flow framework system indices and roles (#3851) ### Description Adds 2 flow-framework related roles to the plugin: read only and fully access. Adds flow-framework system indices to demo scripts as well. ### Issues Resolved ### Testing manual testing of roles ### Check List - [x] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). Signed-off-by: Amit Galitzky --- config/roles.yml | 23 +++++++++++++++++++ .../SecuritySettingsConfigurer.java | 5 +++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/config/roles.yml b/config/roles.yml index 77906290a0..e307ee7de3 100644 --- a/config/roles.yml +++ b/config/roles.yml @@ -373,3 +373,26 @@ security_analytics_ack_alerts: reserved: true cluster_permissions: - 'cluster:admin/opensearch/securityanalytics/alerts/*' + +# Allows users to use all Flow Framework functionality +flow_framework_full_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opensearch/flow_framework/*' + - 'cluster_monitor' + index_permissions: + - index_patterns: + - '*' + allowed_actions: + - 'indices:admin/aliases/get' + - 'indices:admin/mappings/get' + - 'indices_monitor' + +# Allow users to read flow framework's workflows and their state +flow_framework_read_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opensearch/flow_framework/workflow/get' + - 'cluster:admin/opensearch/flow_framework/workflow/search' + - 'cluster:admin/opensearch/flow_framework/workflow_state/get' + - 'cluster:admin/opensearch/flow_framework/workflow_state/search' diff --git a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java index b3644e6c4d..116373a38f 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java @@ -69,7 +69,10 @@ public class SecuritySettingsConfigurer { ".opendistro-asynchronous-search-response*", ".replication-metadata-store", ".opensearch-knn-models", - ".geospatial-ip2geo-data*" + ".geospatial-ip2geo-data*", + ".plugins-flow-framework-config", + ".plugins-flow-framework-templates", + ".plugins-flow-framework-state" ); static String ADMIN_PASSWORD = ""; static String ADMIN_USERNAME = "admin"; From b3b202f2fbed8a2434ce4cbb3ba7437b3d109f9d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 08:43:10 -0500 Subject: [PATCH 056/204] Bump com.netflix.nebula.ospackage from 11.5.0 to 11.6.0 (#3855) Bumps com.netflix.nebula.ospackage from 11.5.0 to 11.6.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.netflix.nebula.ospackage&package-manager=gradle&previous-version=11.5.0&new-version=11.6.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 4f38953d58..526612cb7e 100644 --- a/build.gradle +++ b/build.gradle @@ -64,7 +64,7 @@ plugins { id 'maven-publish' id 'com.diffplug.spotless' version '6.23.3' id 'checkstyle' - id 'com.netflix.nebula.ospackage' version "11.5.0" + id 'com.netflix.nebula.ospackage' version "11.6.0" id "org.gradle.test-retry" version "1.5.8" id 'eclipse' id "com.github.spotbugs" version "5.2.5" From eaefc2d586843a9047ea1d0d856ae32a99d01a57 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 08:43:44 -0500 Subject: [PATCH 057/204] Bump org.apache.camel:camel-xmlsecurity from 3.21.2 to 3.21.3 (#3856) Bumps org.apache.camel:camel-xmlsecurity from 3.21.2 to 3.21.3. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.camel:camel-xmlsecurity&package-manager=gradle&previous-version=3.21.2&new-version=3.21.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 526612cb7e..b49cf98ef3 100644 --- a/build.gradle +++ b/build.gradle @@ -614,7 +614,7 @@ dependencies { runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.1' runtimeOnly 'org.ow2.asm:asm:9.6' - testImplementation 'org.apache.camel:camel-xmlsecurity:3.21.2' + testImplementation 'org.apache.camel:camel-xmlsecurity:3.21.3' //OpenSAML implementation 'net.shibboleth.utilities:java-support:8.4.0' From 604c25f1f91828608cdd1544f278ba4d17d30518 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 08:44:16 -0500 Subject: [PATCH 058/204] Bump org.checkerframework:checker-qual from 3.40.0 to 3.42.0 (#3857) Bumps [org.checkerframework:checker-qual](https://github.com/typetools/checker-framework) from 3.40.0 to 3.42.0.
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Checker Framework 3.42.0

Version 3.42.0 (December 15, 2023)

User-visible changes:

Method annotation @AssertMethod indicates that a method checks a value and possibly throws an assertion. Using it can make flow-sensitive type refinement more effective.

In org.checkerframework.common.util.debug, renamed EmptyProcessor to DoNothingProcessor. Removed org.checkerframework.common.util.report.DoNothingChecker. Moved ReportChecker from org.checkerframework.common.util.report to org.checkerframework.common.util.count.report.

Checker Framework 3.41.0

Version 3.41.0 (December 4, 2023)

User-visible changes:

New command-line options: -AassumePureGetters Unsoundly assume that every getter method is pure

Implementation details:

Added method isDeterministic() to the AnnotationProvider interface.

CFAbstractValue#leastUpperBound and CFAbstractValue#widenUpperBound are now final. Subclasses should override method CFAbstractValue#upperBound(V, TypeMirror, boolean) instead.

Closed issues:

#1497, #3345, #6037, #6204, #6276, #6282, #6290, #6296, #6319, #6327.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 3.42.0 (December 15, 2023)

User-visible changes:

Method annotation @AssertMethod indicates that a method checks a value and possibly throws an assertion. Using it can make flow-sensitive type refinement more effective.

In org.checkerframework.common.util.debug, renamed EmptyProcessor to DoNothingProcessor. Removed org.checkerframework.common.util.report.DoNothingChecker. Moved ReportChecker from org.checkerframework.common.util.report to org.checkerframework.common.util.count.report.

Version 3.41.0 (December 4, 2023)

User-visible changes:

New command-line options: -AassumePureGetters Unsoundly assume that every getter method is pure

Implementation details:

Added method isDeterministic() to the AnnotationProvider interface.

CFAbstractValue#leastUpperBound and CFAbstractValue#widenUpperBound are now final. Subclasses should override method CFAbstractValue#upperBound(V, TypeMirror, boolean) instead.

Closed issues:

#1497, #3345, #6037, #6204, #6276, #6282, #6290, #6296, #6319, #6327.

Commits
  • ed3a237 new release 3.42.0
  • 9053af5 Prep for release.
  • 886d0b3 Add support for opt.map(type::method) pattern. (#6370)
  • 4b5e2c9 ReportChecker: Fix array access crash
  • 3eb26a9 Fix guava-assertions.astub
  • 7906a83 Add -y, a second command-line option for exclusion
  • b38ed80 Fix Kotlin instructions
  • ae8a6e1 Add support for OptionalDouble, OptionalInt, OptionalLong
  • 17226ff Add Optional method annotations
  • 964d027 Permit Stream.filter(Optional::isPresent).map(Optional::get)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.checkerframework:checker-qual&package-manager=gradle&previous-version=3.40.0&new-version=3.42.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index b49cf98ef3..9d1ba4f107 100644 --- a/build.gradle +++ b/build.gradle @@ -500,7 +500,7 @@ configurations { force "org.apache.httpcomponents:httpclient:4.5.14" force "org.apache.httpcomponents:httpcore:4.4.16" force "com.google.errorprone:error_prone_annotations:2.23.0" - force "org.checkerframework:checker-qual:3.40.0" + force "org.checkerframework:checker-qual:3.42.0" force "ch.qos.logback:logback-classic:1.2.13" } } @@ -653,7 +653,7 @@ dependencies { runtimeOnly 'org.apache.ws.xmlschema:xmlschema-core:2.3.1' runtimeOnly 'org.apache.santuario:xmlsec:2.3.4' runtimeOnly "com.github.luben:zstd-jni:${versions.zstd}" - runtimeOnly 'org.checkerframework:checker-qual:3.40.0' + runtimeOnly 'org.checkerframework:checker-qual:3.42.0' runtimeOnly "org.bouncycastle:bcpkix-jdk15to18:${versions.bouncycastle}" runtimeOnly 'org.scala-lang.modules:scala-java8-compat_3:1.0.2' From 0552fe7ddaa6c5fd3429123405500184f59bf4df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 08:44:45 -0500 Subject: [PATCH 059/204] Bump com.flipkart.zjsonpatch:zjsonpatch from 0.4.14 to 0.4.16 (#3858) Bumps [com.flipkart.zjsonpatch:zjsonpatch](https://github.com/flipkart-incubator/zjsonpatch) from 0.4.14 to 0.4.16.
Release notes

Sourced from com.flipkart.zjsonpatch:zjsonpatch's releases.

Releasing 0.4.16

Changes include the following - #174 #172 #171 #165

Releasing 0.4.15

No release notes provided.

Commits
  • d3ea5b2 releasing 0.4.16
  • 08a8052 commenting out maven-surefire-plugin configuration
  • 3e4a920 Bump maven-surefire-plugin from 2.22.2 to 3.0.0 (#174)
  • 1e43697 Bump maven-compiler-plugin from 3.10.1 to 3.11.0 (#172)
  • 65d4a68 Bump maven-javadoc-plugin from 3.4.1 to 3.5.0 (#171)
  • 3d125e7 Bump junit.version from 5.9.1 to 5.9.2 (#165)
  • 2df8d86 updating readme.md with 0.4.15
  • bd4ad76 releasing 0.4.15
  • cc0f0bf make JsonPointer and Operation public for visibility in JsonPatchApplicationE...
  • 49ab75c Upgrade to CodeQL v2 (#169)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.flipkart.zjsonpatch:zjsonpatch&package-manager=gradle&previous-version=0.4.14&new-version=0.4.16)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 9d1ba4f107..84e6eb19f0 100644 --- a/build.gradle +++ b/build.gradle @@ -591,7 +591,7 @@ dependencies { implementation "io.jsonwebtoken:jjwt-impl:${jjwt_version}" implementation "io.jsonwebtoken:jjwt-jackson:${jjwt_version}" // JSON patch - implementation 'com.flipkart.zjsonpatch:zjsonpatch:0.4.14' + implementation 'com.flipkart.zjsonpatch:zjsonpatch:0.4.16' implementation 'org.apache.commons:commons-collections4:4.4' //Password generation From 1f9edf49bcf15e4d1ab3e2e0eb3c1abc8070eb56 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 08:45:21 -0500 Subject: [PATCH 060/204] Bump github/codeql-action from 2 to 3 (#3859) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2 to 3.
Release notes

Sourced from github/codeql-action's releases.

CodeQL Bundle v2.15.4

Bundles CodeQL CLI v2.15.4

Includes the following CodeQL language packs from github/codeql@codeql-cli/v2.15.4:

CodeQL Bundle

Bundles CodeQL CLI v2.15.3

Includes the following CodeQL language packs from github/codeql@codeql-cli/v2.15.3:

CodeQL Bundle

Bundles CodeQL CLI v2.15.2

Includes the following CodeQL language packs from github/codeql@codeql-cli/v2.15.2:

... (truncated)

Changelog

Sourced from github/codeql-action's changelog.

Commits
  • 3a9f6a8 update javascript files
  • cc4fead update version in various hardcoded locations
  • 183559c Merge branch 'main' into update-bundle/codeql-bundle-v2.15.4
  • 5b52b36 reintroduce PR check that confirm action can be still be compiled on node16
  • 5b19bef change to node20 for all actions
  • f2d0c2e upgrade node type definitions
  • d651fbc change to node20 for all actions
  • 382a50a Merge pull request #2021 from github/mergeback/v2.22.9-to-main-c0d1daa7
  • 458b422 Update checked-in dependencies
  • 5e0f9db Update changelog and version after v2.22.9
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github/codeql-action&package-manager=github_actions&previous-version=2&new-version=3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc5e322a6d..51268baed4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -218,11 +218,11 @@ jobs: with: distribution: temurin # Temurin is a distribution of adoptium java-version: 11 - - uses: github/codeql-action/init@v2 + - uses: github/codeql-action/init@v3 with: languages: java - run: ./gradlew clean assemble - - uses: github/codeql-action/analyze@v2 + - uses: github/codeql-action/analyze@v3 build-artifact-names: runs-on: ubuntu-latest From 918c82155145a689c33fc1a7f5d4304876505628 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 08:45:55 -0500 Subject: [PATCH 061/204] Bump actions/download-artifact from 3 to 4 (#3860) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 3 to 4.
Release notes

Sourced from actions/download-artifact's releases.

v4.0.0

What's Changed

The release of upload-artifact@v4 and download-artifact@v4 are major changes to the backend architecture of Artifacts. They have numerous performance and behavioral improvements.

For more information, see the @​actions/artifact documentation.

New Contributors

Full Changelog: https://github.com/actions/download-artifact/compare/v3...v4.0.0

v3.0.2

  • Bump @actions/artifact to v1.1.1 - actions/download-artifact#195
  • Fixed a bug in Node16 where if an HTTP download finished too quickly (<1ms, e.g. when it's mocked) we attempt to delete a temp file that has not been created yet actions/toolkit#1278

v3.0.1

Commits
  • 7a1cd32 Merge pull request #246 from actions/v4-beta
  • 8f32874 licensed cache
  • b5ff844 Merge pull request #245 from actions/robherley/v4-documentation
  • f07a0f7 Update README.md
  • 7226129 update test workflow to use different artifact names for matrix
  • ada9446 update docs and bump @​actions/artifact
  • 7eafc8b Merge pull request #244 from actions/robherley/bump-toolkit
  • 3132d12 consume latest toolkit
  • 5be1d38 Merge pull request #243 from actions/robherley/v4-beta-updates
  • 465b526 consume latest @​actions/toolkit
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/download-artifact&package-manager=github_actions&previous-version=3&new-version=4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51268baed4..49995947cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,7 +74,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: path: downloaded-artifacts From cc577101240a65b5932ebaf9193f3a8968ecba64 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 19 Dec 2023 12:59:08 -0500 Subject: [PATCH 062/204] Add render search template as a cluster permission (#3689) ### Description Companion PRs in core: - https://github.com/opensearch-project/OpenSearch/pull/11170 - https://github.com/opensearch-project/OpenSearch/pull/11591 This PR adds render search template as a cluster perm so that its separately permissioned from a SearchTemplateRequest which needs a set of indices to authorize the request. The companion PR in core separates the transport actions that handle search template request and render search template request so that they can be authorized separately. I am opening this in Draft until the core PR is merged because this PR depends on the core PR. * Category (Enhancement, New feature, Bug fix, Test fix, Refactoring, Maintenance, Documentation) Bug fix ### Issues Resolved - https://github.com/opensearch-project/security/issues/3672 ### Check List - [ ] New functionality includes testing - [ ] New functionality has been documented - [ ] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: Craig Perkins --- .../privileges/PrivilegesEvaluatorTest.java | 37 ++++++++++++++++++- .../privileges/PrivilegesEvaluator.java | 4 +- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java b/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java index 2315c979ea..561b4a0742 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java @@ -18,6 +18,7 @@ import org.junit.runner.RunWith; import org.opensearch.script.mustache.MustacheModulePlugin; +import org.opensearch.script.mustache.RenderSearchTemplateAction; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; import org.opensearch.test.framework.cluster.ClusterManager; @@ -49,15 +50,25 @@ public class PrivilegesEvaluatorTest { new Role("search_template_role").indexPermissions("read").on("services").clusterPermissions("cluster_composite_ops") ); + protected final static TestSecurityConfig.User RENDER_SEARCH_TEMPLATE = new TestSecurityConfig.User("render_search_template_user") + .roles( + new Role("render_search_template_role").indexPermissions("read") + .on("services") + .clusterPermissions(RenderSearchTemplateAction.NAME) + ); + private String TEST_QUERY = "{\"source\":{\"query\":{\"match\":{\"service\":\"{{service_name}}\"}}},\"params\":{\"service_name\":\"Oracle\"}}"; private String TEST_DOC = "{\"source\": {\"title\": \"Spirited Away\"}}"; + private String TEST_RENDER_SEARCH_TEMPLATE_QUERY = + "{\"params\":{\"status\":[\"pending\",\"published\"]},\"source\":\"{\\\"query\\\": {\\\"terms\\\": {\\\"status\\\": [\\\"{{#status}}\\\",\\\"{{.}}\\\",\\\"{{/status}}\\\"]}}}\"}"; + @ClassRule public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS) .authc(AUTHC_HTTPBASIC_INTERNAL) - .users(NEGATIVE_LOOKAHEAD, NEGATED_REGEX, SEARCH_TEMPLATE, TestSecurityConfig.User.USER_ADMIN) + .users(NEGATIVE_LOOKAHEAD, NEGATED_REGEX, SEARCH_TEMPLATE, RENDER_SEARCH_TEMPLATE, TestSecurityConfig.User.USER_ADMIN) .plugin(MustacheModulePlugin.class) .build(); @@ -118,4 +129,28 @@ public void testSearchTemplateRequestUnauthorizedAllIndices() { assertThat(searchOnAllIndicesResponse.getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN)); } } + + @Test + public void testRenderSearchTemplateRequestFailure() { + try (TestRestClient client = cluster.getRestClient(SEARCH_TEMPLATE)) { + final String renderSearchTemplate = "_render/template"; + final TestRestClient.HttpResponse renderSearchTemplateResponse = client.postJson( + renderSearchTemplate, + TEST_RENDER_SEARCH_TEMPLATE_QUERY + ); + assertThat(renderSearchTemplateResponse.getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN)); + } + } + + @Test + public void testRenderSearchTemplateRequestSuccess() { + try (TestRestClient client = cluster.getRestClient(RENDER_SEARCH_TEMPLATE)) { + final String renderSearchTemplate = "_render/template"; + final TestRestClient.HttpResponse renderSearchTemplateResponse = client.postJson( + renderSearchTemplate, + TEST_RENDER_SEARCH_TEMPLATE_QUERY + ); + assertThat(renderSearchTemplateResponse.getStatusCode(), equalTo(HttpStatus.SC_OK)); + } + } } diff --git a/src/main/java/org/opensearch/security/privileges/PrivilegesEvaluator.java b/src/main/java/org/opensearch/security/privileges/PrivilegesEvaluator.java index 1d09932131..0f8d132e3e 100644 --- a/src/main/java/org/opensearch/security/privileges/PrivilegesEvaluator.java +++ b/src/main/java/org/opensearch/security/privileges/PrivilegesEvaluator.java @@ -81,6 +81,7 @@ import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.index.reindex.ReindexAction; +import org.opensearch.script.mustache.RenderSearchTemplateAction; import org.opensearch.security.auditlog.AuditLog; import org.opensearch.security.configuration.ClusterInfoHolder; import org.opensearch.security.configuration.ConfigurationRepository; @@ -696,8 +697,7 @@ public static boolean isClusterPerm(String action0) { || (action0.startsWith(MultiSearchAction.NAME)) || (action0.equals(MultiTermVectorsAction.NAME)) || (action0.equals(ReindexAction.NAME)) - - ); + || (action0.equals(RenderSearchTemplateAction.NAME))); } @SuppressWarnings("unchecked") From 7f1febfbe5e0c979e069660997984f96fac347db Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Fri, 22 Dec 2023 08:36:39 -0600 Subject: [PATCH 063/204] Revert "Bump actions/download-artifact from 3 to 4" (#3892) Reverts opensearch-project/security#3860 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49995947cf..51268baed4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,7 +74,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v3 with: path: downloaded-artifacts From 83723432cca6e78d23e034b5149ff72d5907fde3 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Fri, 22 Dec 2023 13:14:14 -0500 Subject: [PATCH 064/204] Fix the CI / report-coverage check by switching to corresponding actions/upload-artifact@v4 (#3893) Signed-off-by: Craig Perkins --- .github/workflows/ci.yml | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51268baed4..0f64a0408b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,7 +60,7 @@ jobs: arguments: | ${{ matrix.gradle_task }} -Dbuild.snapshot=false - - uses: alehechka/upload-tartifact@v2 + - uses: actions/upload-artifact@v4 if: always() with: name: ${{ matrix.platform }}-JDK${{ matrix.jdk }}-${{ matrix.gradle_task }}-reports @@ -68,13 +68,11 @@ jobs: ./build/reports/ report-coverage: - needs: - - "test" - - "integration-tests" + needs: ["test", "integration-tests"] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: path: downloaded-artifacts @@ -82,17 +80,6 @@ jobs: run: ls -R working-directory: downloaded-artifacts - - name: Extract downloaded artifacts - run: | - for archive in ./*/artifact.tar; do - (cd "$(dirname "$archive")" && tar -xvf artifact.tar) - done - working-directory: downloaded-artifacts - - - name: Display structure of downloaded files - run: ls -R - working-directory: downloaded-artifacts - - name: Upload Coverage with retry uses: Wandalen/wretry.action@v1.3.0 with: From 3c566a42a1676acfe391886df7ecfe8de3e4349f Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Fri, 22 Dec 2023 13:46:45 -0600 Subject: [PATCH 065/204] Add deprecation check for `jwt_header` setting (#3887) ### Description Add deprecation check for `jwt_header` setting ### Issues Resolved - Related https://github.com/opensearch-project/security/issues/3886 ### Check List - [ ] ~New functionality includes testing~ - [ ] New functionality has been documented - [X] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: Peter Nied --- .../dlic/auth/http/jwt/AbstractHTTPJwtAuthenticator.java | 9 +++++++++ .../amazon/dlic/auth/http/jwt/HTTPJwtAuthenticator.java | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/src/main/java/com/amazon/dlic/auth/http/jwt/AbstractHTTPJwtAuthenticator.java b/src/main/java/com/amazon/dlic/auth/http/jwt/AbstractHTTPJwtAuthenticator.java index 8c6af4279b..ea0a6378d7 100644 --- a/src/main/java/com/amazon/dlic/auth/http/jwt/AbstractHTTPJwtAuthenticator.java +++ b/src/main/java/com/amazon/dlic/auth/http/jwt/AbstractHTTPJwtAuthenticator.java @@ -28,6 +28,7 @@ import org.opensearch.OpenSearchSecurityException; import org.opensearch.SpecialPermission; +import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.common.Strings; @@ -48,6 +49,7 @@ public abstract class AbstractHTTPJwtAuthenticator implements HTTPAuthenticator { private final static Logger log = LogManager.getLogger(AbstractHTTPJwtAuthenticator.class); + private final static DeprecationLogger deprecationLog = DeprecationLogger.getLogger(AbstractHTTPJwtAuthenticator.class); private static final String BEARER = "bearer "; private static final Pattern BASIC = Pattern.compile("^\\s*Basic\\s.*", Pattern.CASE_INSENSITIVE); @@ -75,6 +77,13 @@ public AbstractHTTPJwtAuthenticator(Settings settings, Path configPath) { requiredAudience = settings.get("required_audience"); requiredIssuer = settings.get("required_issuer"); + if (!jwtHeaderName.equals(AUTHORIZATION)) { + deprecationLog.deprecate( + "jwt_header", + "The 'jwt_header' setting will be removed in the next major version of OpenSearch. Consult https://github.com/opensearch-project/security/issues/3886 for more details." + ); + } + try { this.keyProvider = this.initKeyProvider(settings, configPath); jwtVerifier = new JwtVerifier(keyProvider, clockSkewToleranceSeconds, requiredIssuer, requiredAudience); diff --git a/src/main/java/com/amazon/dlic/auth/http/jwt/HTTPJwtAuthenticator.java b/src/main/java/com/amazon/dlic/auth/http/jwt/HTTPJwtAuthenticator.java index c5c3e0ddc5..9bf22bf7f3 100644 --- a/src/main/java/com/amazon/dlic/auth/http/jwt/HTTPJwtAuthenticator.java +++ b/src/main/java/com/amazon/dlic/auth/http/jwt/HTTPJwtAuthenticator.java @@ -26,6 +26,7 @@ import org.opensearch.OpenSearchSecurityException; import org.opensearch.SpecialPermission; +import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.security.auth.HTTPAuthenticator; @@ -44,6 +45,7 @@ public class HTTPJwtAuthenticator implements HTTPAuthenticator { protected final Logger log = LogManager.getLogger(this.getClass()); + protected final DeprecationLogger deprecationLog = DeprecationLogger.getLogger(this.getClass()); private static final Pattern BASIC = Pattern.compile("^\\s*Basic\\s.*", Pattern.CASE_INSENSITIVE); private static final String BEARER = "bearer "; @@ -69,6 +71,13 @@ public HTTPJwtAuthenticator(final Settings settings, final Path configPath) { requireAudience = settings.get("required_audience"); requireIssuer = settings.get("required_issuer"); + if (!jwtHeaderName.equals(AUTHORIZATION)) { + deprecationLog.deprecate( + "jwt_header", + "The 'jwt_header' setting will be removed in the next major version of OpenSearch. Consult https://github.com/opensearch-project/security/issues/3886 for more details." + ); + } + final JwtParserBuilder jwtParserBuilder = KeyUtils.createJwtParserBuilderFromSigningKey(signingKey, log); if (jwtParserBuilder == null) { jwtParser = null; From e8b215d10c62785cf65dd950385378e963332314 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Dec 2023 09:52:53 -0500 Subject: [PATCH 066/204] Bump io.dropwizard.metrics:metrics-core from 4.2.22 to 4.2.23 (#3899) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.dropwizard.metrics:metrics-core](https://github.com/dropwizard/metrics) from 4.2.22 to 4.2.23.
Release notes

Sourced from io.dropwizard.metrics:metrics-core's releases.

v4.2.23

What's Changed

Full Changelog: https://github.com/dropwizard/metrics/compare/v4.2.22...v4.2.23

Commits
  • 46cf51d [maven-release-plugin] prepare release v4.2.23
  • dde28a0 Update dependency org.mockito:mockito-core to v5.8.0 (#3784)
  • 03a1543 Update dependency org.checkerframework:checker-qual to v3.41.0 (#3782)
  • dae4a8c Update logback14.version to v1.4.14 (#3781)
  • 38c4339 Update jetty12.version to v12.0.4 (#3778)
  • 4c90ccd Update logback13.version to v1.3.14 (#3780)
  • 6217e67 Update logback.version to v1.2.13
  • da71539 Update dependency org.glassfish.jersey:jersey-bom to v3.1.4
  • eb8e31d Update dependency org.apache.maven.plugins:maven-javadoc-plugin to v3.6.3
  • 40f1fad Update dependency org.apache.httpcomponents.client5:httpclient5 to v5.2.3
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.dropwizard.metrics:metrics-core&package-manager=gradle&previous-version=4.2.22&new-version=4.2.23)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 84e6eb19f0..8717b5a508 100644 --- a/build.gradle +++ b/build.gradle @@ -643,7 +643,7 @@ dependencies { runtimeOnly 'com.google.j2objc:j2objc-annotations:2.8' compileOnly 'com.google.code.findbugs:jsr305:3.0.2' runtimeOnly 'org.lz4:lz4-java:1.8.0' - runtimeOnly 'io.dropwizard.metrics:metrics-core:4.2.22' + runtimeOnly 'io.dropwizard.metrics:metrics-core:4.2.23' runtimeOnly 'org.slf4j:slf4j-api:1.7.36' runtimeOnly "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}" runtimeOnly 'org.xerial.snappy:snappy-java:1.1.10.5' From c134ddf75d12659c9da95d442b874a27508caede Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Dec 2023 09:53:17 -0500 Subject: [PATCH 067/204] Bump com.google.googlejavaformat:google-java-format from 1.18.1 to 1.19.1 (#3898) Bumps [com.google.googlejavaformat:google-java-format](https://github.com/google/google-java-format) from 1.18.1 to 1.19.1.
Release notes

Sourced from com.google.googlejavaformat:google-java-format's releases.

v1.19.1

Changes

  • Fix support for Java 21 features by enabling JDK 21 for release builds (#1014).

Full Changelog: https://github.com/google/google-java-format/compare/v1.19.0...v1.19.1

v1.19.0

Changes

This release adds initial support for more Java 21 features, including:

  • String templates (#981)
  • Unnamed variables (#978)
  • Pattern matching and guard clauses in switch expressions (#937, #880, #983, #988)

Other changes:

  • Handle type annotations on method reference qualifiers (53390d99b56edae23cfab5adcafd7df28d9984c8)
  • Avoid reflowing text blocks (#976)

Full Changelog: https://github.com/google/google-java-format/compare/v1.18.1...v1.19.0

Commits
  • 8cafdb3 Release google-java-format 1.19.1
  • 8afdfca chore: bump checkout/setup-java to v4 (use nodejs20 runtime)
  • 627c97e chore: fix SyntaxWarning: invalid escape sequence + for py3.12
  • 0e7cc6f Upgrade jdk used by release action to 21
  • b5feefe Initial support for string templates
  • dc8b461 Make g-j-f native image more compatible
  • b92435a Support unnamed variables
  • 430ba3b Initial support for pattern matching in switches
  • b86c508 Add support for guard clauses in Java 21 switch expressions
  • ad77154 Bump Guava to 32.1.3
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.googlejavaformat:google-java-format&package-manager=gradle&previous-version=1.18.1&new-version=1.19.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8717b5a508..b52cd26d59 100644 --- a/build.gradle +++ b/build.gradle @@ -740,7 +740,7 @@ dependencies { integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5" //spotless - implementation('com.google.googlejavaformat:google-java-format:1.18.1') { + implementation('com.google.googlejavaformat:google-java-format:1.19.1') { exclude group: 'com.google.guava' } } From 7734fde172aa7e514866fba2532bee4acf5389d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Dec 2023 09:53:54 -0500 Subject: [PATCH 068/204] Bump com.google.errorprone:error_prone_annotations from 2.23.0 to 2.24.0 (#3897) Bumps [com.google.errorprone:error_prone_annotations](https://github.com/google/error-prone) from 2.23.0 to 2.24.0.
Release notes

Sourced from com.google.errorprone:error_prone_annotations's releases.

Error Prone 2.24.0

New checks:

Full Changelog: https://github.com/google/error-prone/compare/v2.23.0...v2.24.0

Commits
  • 2cc8504 Release Error Prone 2.24.0
  • 21c190a Document that javadoc shouldn't appear between annotations and the documented...
  • d272dfa Automated rollback of commit 654d1dbf1e6dd652cd6e8ca003643ddf02266ec2.
  • 654d1db Handle Joiner.on(...) in AbstractToString.
  • da7be27 Descend into VariableTrees when looking for variables to check.
  • affa37a Do not flag unused parameters on abstract methods.
  • d78dd6d Don't report NonFinalStaticField findings for fields modified in `@BeforeClas...
  • aadfdc3 WellKnownThreadSafety: Add common PKIX types to known thread-safe list.
  • ac52ca9 AutoValueFinalMethods: support method-level suppression.
  • 336323a Import eisop/checker-framework from GitHub.
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.errorprone:error_prone_annotations&package-manager=gradle&previous-version=2.23.0&new-version=2.24.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index b52cd26d59..7e2fb5ef26 100644 --- a/build.gradle +++ b/build.gradle @@ -499,7 +499,7 @@ configurations { // For integrationTest force "org.apache.httpcomponents:httpclient:4.5.14" force "org.apache.httpcomponents:httpcore:4.4.16" - force "com.google.errorprone:error_prone_annotations:2.23.0" + force "com.google.errorprone:error_prone_annotations:2.24.0" force "org.checkerframework:checker-qual:3.42.0" force "ch.qos.logback:logback-classic:1.2.13" } @@ -609,7 +609,7 @@ dependencies { runtimeOnly 'com.eclipsesource.minimal-json:minimal-json:0.9.5' runtimeOnly 'commons-codec:commons-codec:1.16.0' runtimeOnly 'org.cryptacular:cryptacular:1.2.6' - compileOnly 'com.google.errorprone:error_prone_annotations:2.23.0' + compileOnly 'com.google.errorprone:error_prone_annotations:2.24.0' runtimeOnly 'com.sun.istack:istack-commons-runtime:4.2.0' runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.1' runtimeOnly 'org.ow2.asm:asm:9.6' From 26e62d5b77300695e3bd94352a5e297e28828b15 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 11:50:52 +0100 Subject: [PATCH 069/204] Bump org.apache.camel:camel-xmlsecurity from 3.21.3 to 3.22.0 (#3904) Bumps org.apache.camel:camel-xmlsecurity from 3.21.3 to 3.22.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.camel:camel-xmlsecurity&package-manager=gradle&previous-version=3.21.3&new-version=3.22.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 7e2fb5ef26..4fb2d50ab5 100644 --- a/build.gradle +++ b/build.gradle @@ -614,7 +614,7 @@ dependencies { runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.1' runtimeOnly 'org.ow2.asm:asm:9.6' - testImplementation 'org.apache.camel:camel-xmlsecurity:3.21.3' + testImplementation 'org.apache.camel:camel-xmlsecurity:3.22.0' //OpenSAML implementation 'net.shibboleth.utilities:java-support:8.4.0' From 31f1625742792b7033fa8a24781b95695ef2e1a2 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Wed, 3 Jan 2024 14:10:22 -0600 Subject: [PATCH 070/204] Switched to more reliable OpenSearch Lucene snapshot location (#3912) ### Description Switched to more reliable OpenSearch Lucene snapshot location ### Issues Resolved - Resolves https://github.com/opensearch-project/security/issues/3597 ### Check List - [ ] ~New functionality includes testing~ - [ ] ~New functionality has been documented~ - [X] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). Signed-off-by: Peter Nied --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 4fb2d50ab5..3a368bc8a7 100644 --- a/build.gradle +++ b/build.gradle @@ -47,7 +47,7 @@ buildscript { mavenCentral() maven { url "https://plugins.gradle.org/m2/" } maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } - maven { url "https://d1nvenhzbhpy0q.cloudfront.net/snapshots/lucene/" } + maven { url "https://artifacts.opensearch.org/snapshots/lucene/" } maven { url "https://build.shibboleth.net/nexus/content/groups/public" } maven { url "https://build.shibboleth.net/nexus/content/repositories/releases" } } @@ -451,7 +451,7 @@ repositories { mavenCentral() maven { url "https://plugins.gradle.org/m2/" } maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } - maven { url "https://d1nvenhzbhpy0q.cloudfront.net/snapshots/lucene/" } + maven { url "https://artifacts.opensearch.org/snapshots/lucene/" } maven { url "https://build.shibboleth.net/nexus/content/repositories/releases" } } From f216743b16f0f7cbc7007bba5dde39a39072e3b4 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 4 Jan 2024 04:46:59 -0500 Subject: [PATCH 071/204] Re-enable disabled PIT integration tests (#3871) ### Description Re-enables 2 PIT integration tests that were previously disabled. This adds a @Before method to clean up all PITs created from previous test cases to ensure each test case runs pristinely. * Category (Enhancement, New feature, Bug fix, Test fix, Refactoring, Maintenance, Documentation) Test fix ### Issues Resolved - https://github.com/opensearch-project/security/issues/3424 Is this a backport? If so, please add backport PR # and/or commits # ### Testing [Please provide details of testing done: unit testing, integration testing and manual testing] ### Check List - [ ] New functionality includes testing - [ ] New functionality has been documented - [ ] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: Craig Perkins --- .../security/PointInTimeOperationTest.java | 31 +++++-------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/PointInTimeOperationTest.java b/src/integrationTest/java/org/opensearch/security/PointInTimeOperationTest.java index ce934a8e16..8d900639c2 100644 --- a/src/integrationTest/java/org/opensearch/security/PointInTimeOperationTest.java +++ b/src/integrationTest/java/org/opensearch/security/PointInTimeOperationTest.java @@ -14,13 +14,12 @@ import com.carrotsearch.randomizedtesting.RandomizedRunner; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import org.apache.commons.lang3.tuple.Pair; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; -import org.opensearch.OpenSearchStatusException; import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.search.CreatePitRequest; @@ -33,7 +32,6 @@ import org.opensearch.client.Client; import org.opensearch.client.RestHighLevelClient; import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.rest.RestStatus; import org.opensearch.search.builder.PointInTimeBuilder; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.test.framework.TestSecurityConfig; @@ -133,6 +131,13 @@ public static void createTestData() { } } + @Before + public void cleanUpPits() throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(ADMIN_USER)) { + restHighLevelClient.deleteAllPits(DEFAULT); + } + } + @ClassRule public static final LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS) .anonymousAuth(false) @@ -180,11 +185,9 @@ public void createPitWithIndexAlias_negative() throws IOException { } } - @Ignore("Pretty sure cleanUpPits is returning before all of the PITs have actually been deleted") @Test public void listAllPits_positive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(POINT_IN_TIME_USER)) { - cleanUpPits(); String firstIndexPit = createPitForIndices(FIRST_SONG_INDEX); String secondIndexPit = createPitForIndices(SECOND_SONG_INDEX); @@ -247,11 +250,9 @@ public void deletePitCreatedWithIndexAlias_negative() throws IOException { } } - @Ignore("Pretty sure cleanUpPits is returning before all of the PITs have actually been deleted") @Test public void deleteAllPits_positive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(POINT_IN_TIME_USER)) { - cleanUpPits(); String firstIndexPit = createPitForIndices(FIRST_SONG_INDEX); String secondIndexPit = createPitForIndices(SECOND_SONG_INDEX); @@ -410,20 +411,4 @@ private String createPitForIndices(String... indices) throws IOException { } } - /** - * Deletes all PITs. - */ - public void cleanUpPits() throws IOException { - try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(ADMIN_USER)) { - try { - restHighLevelClient.deleteAllPits(DEFAULT); - } catch (OpenSearchStatusException ex) { - if (ex.status() != RestStatus.NOT_FOUND) { - throw ex; - } - // tried to remove pits but no pit exists - } - } - } - } From 21c086c02c6aafb2f71d74595a3b5095c7054706 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 4 Jan 2024 15:32:02 -0500 Subject: [PATCH 072/204] Improve messaging on how to set initial admin password (#3879) ### Description When the demo configuration script fails when no initial admin password is set, it does not provide helpful output telling the user how to set the initial admin password. This PR provides some helpful output indicating what variable the user should modify in order for the setup to succeed. ### Issues Resolved [List any issues this PR will resolve] Is this a backport? If so, please add backport PR # and/or commits # ### Testing Changed test asserting output ### Check List - [ ] New functionality includes testing - [ ] New functionality has been documented - [x] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: Derek Ho --- .../tools/democonfig/SecuritySettingsConfigurer.java | 7 ++++++- .../tools/democonfig/SecuritySettingsConfigurerTests.java | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java index 116373a38f..9c51fbe1d4 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java @@ -149,7 +149,12 @@ void updateAdminPassword() { // if ADMIN_PASSWORD is still an empty string, it implies no custom password was provided. We exit the setup. if (Strings.isNullOrEmpty(ADMIN_PASSWORD)) { - System.out.println("No custom admin password found. Please provide a password."); + System.out.println( + String.format( + "No custom admin password found. Please provide a password via the environment variable %s.", + ConfigConstants.OPENSEARCH_INITIAL_ADMIN_PASSWORD + ) + ); System.exit(-1); } diff --git a/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java b/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java index 948a66996c..27ba150a78 100644 --- a/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java +++ b/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java @@ -104,7 +104,12 @@ public void testUpdateAdminPassword_noPasswordSupplied() { System.setSecurityManager(null); } - verifyStdOutContainsString("No custom admin password found. Please provide a password."); + verifyStdOutContainsString( + String.format( + "No custom admin password found. Please provide a password via the environment variable %s.", + ConfigConstants.OPENSEARCH_INITIAL_ADMIN_PASSWORD + ) + ); } @Test From 907913a140e5a6032736d66c3701969cbceff708 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 4 Jan 2024 21:21:57 -0500 Subject: [PATCH 073/204] Update to Gradle 8.5 (#3919) Signed-off-by: Andriy Redko --- gradle/wrapper/gradle-wrapper.jar | Bin 63721 -> 43462 bytes gradle/wrapper/gradle-wrapper.properties | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 7f93135c49b765f8051ef9d0a6055ff8e46073d8..d64cd4917707c1f8861d8cb53dd15194d4248596 100644 GIT binary patch literal 43462 zcma&NWl&^owk(X(xVyW%ySuwf;qI=D6|RlDJ2cR^yEKh!@I- zp9QeisK*rlxC>+~7Dk4IxIRsKBHqdR9b3+fyL=ynHmIDe&|>O*VlvO+%z5;9Z$|DJ zb4dO}-R=MKr^6EKJiOrJdLnCJn>np?~vU-1sSFgPu;pthGwf}bG z(1db%xwr#x)r+`4AGu$j7~u2MpVs3VpLp|mx&;>`0p0vH6kF+D2CY0fVdQOZ@h;A` z{infNyvmFUiu*XG}RNMNwXrbec_*a3N=2zJ|Wh5z* z5rAX$JJR{#zP>KY**>xHTuw?|-Rg|o24V)74HcfVT;WtQHXlE+_4iPE8QE#DUm%x0 zEKr75ur~W%w#-My3Tj`hH6EuEW+8K-^5P62$7Sc5OK+22qj&Pd1;)1#4tKihi=~8C zHiQSst0cpri6%OeaR`PY>HH_;CPaRNty%WTm4{wDK8V6gCZlG@U3$~JQZ;HPvDJcT1V{ z?>H@13MJcCNe#5z+MecYNi@VT5|&UiN1D4ATT+%M+h4c$t;C#UAs3O_q=GxK0}8%8 z8J(_M9bayxN}69ex4dzM_P3oh@ZGREjVvn%%r7=xjkqxJP4kj}5tlf;QosR=%4L5y zWhgejO=vao5oX%mOHbhJ8V+SG&K5dABn6!WiKl{|oPkq(9z8l&Mm%(=qGcFzI=eLu zWc_oCLyf;hVlB@dnwY98?75B20=n$>u3b|NB28H0u-6Rpl((%KWEBOfElVWJx+5yg z#SGqwza7f}$z;n~g%4HDU{;V{gXIhft*q2=4zSezGK~nBgu9-Q*rZ#2f=Q}i2|qOp z!!y4p)4o=LVUNhlkp#JL{tfkhXNbB=Ox>M=n6soptJw-IDI|_$is2w}(XY>a=H52d z3zE$tjPUhWWS+5h=KVH&uqQS=$v3nRs&p$%11b%5qtF}S2#Pc`IiyBIF4%A!;AVoI zXU8-Rpv!DQNcF~(qQnyyMy=-AN~U>#&X1j5BLDP{?K!%h!;hfJI>$mdLSvktEr*89 zdJHvby^$xEX0^l9g$xW-d?J;L0#(`UT~zpL&*cEh$L|HPAu=P8`OQZV!-}l`noSp_ zQ-1$q$R-gDL)?6YaM!=8H=QGW$NT2SeZlb8PKJdc=F-cT@j7Xags+Pr*jPtlHFnf- zh?q<6;)27IdPc^Wdy-mX%2s84C1xZq9Xms+==F4);O`VUASmu3(RlgE#0+#giLh-& zcxm3_e}n4{%|X zJp{G_j+%`j_q5}k{eW&TlP}J2wtZ2^<^E(O)4OQX8FDp6RJq!F{(6eHWSD3=f~(h} zJXCf7=r<16X{pHkm%yzYI_=VDP&9bmI1*)YXZeB}F? z(%QsB5fo*FUZxK$oX~X^69;x~j7ms8xlzpt-T15e9}$4T-pC z6PFg@;B-j|Ywajpe4~bk#S6(fO^|mm1hKOPfA%8-_iGCfICE|=P_~e;Wz6my&)h_~ zkv&_xSAw7AZ%ThYF(4jADW4vg=oEdJGVOs>FqamoL3Np8>?!W#!R-0%2Bg4h?kz5I zKV-rKN2n(vUL%D<4oj@|`eJ>0i#TmYBtYmfla;c!ATW%;xGQ0*TW@PTlGG><@dxUI zg>+3SiGdZ%?5N=8uoLA|$4isK$aJ%i{hECP$bK{J#0W2gQ3YEa zZQ50Stn6hqdfxJ*9#NuSLwKFCUGk@c=(igyVL;;2^wi4o30YXSIb2g_ud$ zgpCr@H0qWtk2hK8Q|&wx)}4+hTYlf;$a4#oUM=V@Cw#!$(nOFFpZ;0lc!qd=c$S}Z zGGI-0jg~S~cgVT=4Vo)b)|4phjStD49*EqC)IPwyeKBLcN;Wu@Aeph;emROAwJ-0< z_#>wVm$)ygH|qyxZaet&(Vf%pVdnvKWJn9`%DAxj3ot;v>S$I}jJ$FLBF*~iZ!ZXE zkvui&p}fI0Y=IDX)mm0@tAd|fEHl~J&K}ZX(Mm3cm1UAuwJ42+AO5@HwYfDH7ipIc zmI;1J;J@+aCNG1M`Btf>YT>~c&3j~Qi@Py5JT6;zjx$cvOQW@3oQ>|}GH?TW-E z1R;q^QFjm5W~7f}c3Ww|awg1BAJ^slEV~Pk`Kd`PS$7;SqJZNj->it4DW2l15}xP6 zoCl$kyEF%yJni0(L!Z&14m!1urXh6Btj_5JYt1{#+H8w?5QI%% zo-$KYWNMJVH?Hh@1n7OSu~QhSswL8x0=$<8QG_zepi_`y_79=nK=_ZP_`Em2UI*tyQoB+r{1QYZCpb?2OrgUw#oRH$?^Tj!Req>XiE#~B|~ z+%HB;=ic+R@px4Ld8mwpY;W^A%8%l8$@B@1m5n`TlKI6bz2mp*^^^1mK$COW$HOfp zUGTz-cN9?BGEp}5A!mDFjaiWa2_J2Iq8qj0mXzk; z66JBKRP{p%wN7XobR0YjhAuW9T1Gw3FDvR5dWJ8ElNYF94eF3ebu+QwKjtvVu4L zI9ip#mQ@4uqVdkl-TUQMb^XBJVLW(-$s;Nq;@5gr4`UfLgF$adIhd?rHOa%D);whv z=;krPp~@I+-Z|r#s3yCH+c1US?dnm+C*)r{m+86sTJusLdNu^sqLrfWed^ndHXH`m zd3#cOe3>w-ga(Dus_^ppG9AC>Iq{y%%CK+Cro_sqLCs{VLuK=dev>OL1dis4(PQ5R zcz)>DjEkfV+MO;~>VUlYF00SgfUo~@(&9$Iy2|G0T9BSP?&T22>K46D zL*~j#yJ?)^*%J3!16f)@Y2Z^kS*BzwfAQ7K96rFRIh>#$*$_Io;z>ux@}G98!fWR@ zGTFxv4r~v)Gsd|pF91*-eaZ3Qw1MH$K^7JhWIdX%o$2kCbvGDXy)a?@8T&1dY4`;L z4Kn+f%SSFWE_rpEpL9bnlmYq`D!6F%di<&Hh=+!VI~j)2mfil03T#jJ_s?}VV0_hp z7T9bWxc>Jm2Z0WMU?`Z$xE74Gu~%s{mW!d4uvKCx@WD+gPUQ zV0vQS(Ig++z=EHN)BR44*EDSWIyT~R4$FcF*VEY*8@l=218Q05D2$|fXKFhRgBIEE zdDFB}1dKkoO^7}{5crKX!p?dZWNz$m>1icsXG2N+((x0OIST9Zo^DW_tytvlwXGpn zs8?pJXjEG;T@qrZi%#h93?FP$!&P4JA(&H61tqQi=opRzNpm zkrG}$^t9&XduK*Qa1?355wd8G2CI6QEh@Ua>AsD;7oRUNLPb76m4HG3K?)wF~IyS3`fXuNM>${?wmB zpVz;?6_(Fiadfd{vUCBM*_kt$+F3J+IojI;9L(gc9n3{sEZyzR9o!_mOwFC#tQ{Q~ zP3-`#uK#tP3Q7~Q;4H|wjZHO8h7e4IuBxl&vz2w~D8)w=Wtg31zpZhz%+kzSzL*dV zwp@{WU4i;hJ7c2f1O;7Mz6qRKeASoIv0_bV=i@NMG*l<#+;INk-^`5w@}Dj~;k=|}qM1vq_P z|GpBGe_IKq|LNy9SJhKOQ$c=5L{Dv|Q_lZl=-ky*BFBJLW9&y_C|!vyM~rQx=!vun z?rZJQB5t}Dctmui5i31C_;_}CEn}_W%>oSXtt>@kE1=JW*4*v4tPp;O6 zmAk{)m!)}34pTWg8{i>($%NQ(Tl;QC@J@FfBoc%Gr&m560^kgSfodAFrIjF}aIw)X zoXZ`@IsMkc8_=w%-7`D6Y4e*CG8k%Ud=GXhsTR50jUnm+R*0A(O3UKFg0`K;qp1bl z7``HN=?39ic_kR|^R^~w-*pa?Vj#7|e9F1iRx{GN2?wK!xR1GW!qa=~pjJb-#u1K8 zeR?Y2i-pt}yJq;SCiVHODIvQJX|ZJaT8nO+(?HXbLefulKKgM^B(UIO1r+S=7;kLJ zcH}1J=Px2jsh3Tec&v8Jcbng8;V-`#*UHt?hB(pmOipKwf3Lz8rG$heEB30Sg*2rx zV<|KN86$soN(I!BwO`1n^^uF2*x&vJ$2d$>+`(romzHP|)K_KkO6Hc>_dwMW-M(#S zK(~SiXT1@fvc#U+?|?PniDRm01)f^#55;nhM|wi?oG>yBsa?~?^xTU|fX-R(sTA+5 zaq}-8Tx7zrOy#3*JLIIVsBmHYLdD}!0NP!+ITW+Thn0)8SS!$@)HXwB3tY!fMxc#1 zMp3H?q3eD?u&Njx4;KQ5G>32+GRp1Ee5qMO0lZjaRRu&{W<&~DoJNGkcYF<5(Ab+J zgO>VhBl{okDPn78<%&e2mR{jwVCz5Og;*Z;;3%VvoGo_;HaGLWYF7q#jDX=Z#Ml`H z858YVV$%J|e<1n`%6Vsvq7GmnAV0wW4$5qQ3uR@1i>tW{xrl|ExywIc?fNgYlA?C5 zh$ezAFb5{rQu6i7BSS5*J-|9DQ{6^BVQ{b*lq`xS@RyrsJN?-t=MTMPY;WYeKBCNg z^2|pN!Q^WPJuuO4!|P@jzt&tY1Y8d%FNK5xK(!@`jO2aEA*4 zkO6b|UVBipci?){-Ke=+1;mGlND8)6+P;8sq}UXw2hn;fc7nM>g}GSMWu&v&fqh

iViYT=fZ(|3Ox^$aWPp4a8h24tD<|8-!aK0lHgL$N7Efw}J zVIB!7=T$U`ao1?upi5V4Et*-lTG0XvExbf!ya{cua==$WJyVG(CmA6Of*8E@DSE%L z`V^$qz&RU$7G5mg;8;=#`@rRG`-uS18$0WPN@!v2d{H2sOqP|!(cQ@ zUHo!d>>yFArLPf1q`uBvY32miqShLT1B@gDL4XoVTK&@owOoD)OIHXrYK-a1d$B{v zF^}8D3Y^g%^cnvScOSJR5QNH+BI%d|;J;wWM3~l>${fb8DNPg)wrf|GBP8p%LNGN# z3EaIiItgwtGgT&iYCFy9-LG}bMI|4LdmmJt@V@% zb6B)1kc=T)(|L@0;wr<>=?r04N;E&ef+7C^`wPWtyQe(*pD1pI_&XHy|0gIGHMekd zF_*M4yi6J&Z4LQj65)S zXwdM{SwUo%3SbPwFsHgqF@V|6afT|R6?&S;lw=8% z3}@9B=#JI3@B*#4s!O))~z zc>2_4Q_#&+5V`GFd?88^;c1i7;Vv_I*qt!_Yx*n=;rj!82rrR2rQ8u5(Ejlo{15P% zs~!{%XJ>FmJ})H^I9bn^Re&38H{xA!0l3^89k(oU;bZWXM@kn$#aoS&Y4l^-WEn-fH39Jb9lA%s*WsKJQl?n9B7_~P z-XM&WL7Z!PcoF6_D>V@$CvUIEy=+Z&0kt{szMk=f1|M+r*a43^$$B^MidrT0J;RI` z(?f!O<8UZkm$_Ny$Hth1J#^4ni+im8M9mr&k|3cIgwvjAgjH z8`N&h25xV#v*d$qBX5jkI|xOhQn!>IYZK7l5#^P4M&twe9&Ey@@GxYMxBZq2e7?`q z$~Szs0!g{2fGcp9PZEt|rdQ6bhAgpcLHPz?f-vB?$dc*!9OL?Q8mn7->bFD2Si60* z!O%y)fCdMSV|lkF9w%x~J*A&srMyYY3{=&$}H zGQ4VG_?$2X(0|vT0{=;W$~icCI{b6W{B!Q8xdGhF|D{25G_5_+%s(46lhvNLkik~R z>nr(&C#5wwOzJZQo9m|U<;&Wk!_#q|V>fsmj1g<6%hB{jGoNUPjgJslld>xmODzGjYc?7JSuA?A_QzjDw5AsRgi@Y|Z0{F{!1=!NES-#*f^s4l0Hu zz468))2IY5dmD9pa*(yT5{EyP^G>@ZWumealS-*WeRcZ}B%gxq{MiJ|RyX-^C1V=0 z@iKdrGi1jTe8Ya^x7yyH$kBNvM4R~`fbPq$BzHum-3Zo8C6=KW@||>zsA8-Y9uV5V z#oq-f5L5}V<&wF4@X@<3^C%ptp6+Ce)~hGl`kwj)bsAjmo_GU^r940Z-|`<)oGnh7 zFF0Tde3>ui?8Yj{sF-Z@)yQd~CGZ*w-6p2U<8}JO-sRsVI5dBji`01W8A&3$?}lxBaC&vn0E$c5tW* zX>5(zzZ=qn&!J~KdsPl;P@bmA-Pr8T*)eh_+Dv5=Ma|XSle6t(k8qcgNyar{*ReQ8 zTXwi=8vr>!3Ywr+BhggHDw8ke==NTQVMCK`$69fhzEFB*4+H9LIvdt-#IbhZvpS}} zO3lz;P?zr0*0$%-Rq_y^k(?I{Mk}h@w}cZpMUp|ucs55bcloL2)($u%mXQw({Wzc~ z;6nu5MkjP)0C(@%6Q_I_vsWrfhl7Zpoxw#WoE~r&GOSCz;_ro6i(^hM>I$8y>`!wW z*U^@?B!MMmb89I}2(hcE4zN2G^kwyWCZp5JG>$Ez7zP~D=J^LMjSM)27_0B_X^C(M z`fFT+%DcKlu?^)FCK>QzSnV%IsXVcUFhFdBP!6~se&xxrIxsvySAWu++IrH;FbcY$ z2DWTvSBRfLwdhr0nMx+URA$j3i7_*6BWv#DXfym?ZRDcX9C?cY9sD3q)uBDR3uWg= z(lUIzB)G$Hr!){>E{s4Dew+tb9kvToZp-1&c?y2wn@Z~(VBhqz`cB;{E4(P3N2*nJ z_>~g@;UF2iG{Kt(<1PyePTKahF8<)pozZ*xH~U-kfoAayCwJViIrnqwqO}7{0pHw$ zs2Kx?s#vQr7XZ264>5RNKSL8|Ty^=PsIx^}QqOOcfpGUU4tRkUc|kc7-!Ae6!+B{o~7nFpm3|G5^=0#Bnm6`V}oSQlrX(u%OWnC zoLPy&Q;1Jui&7ST0~#+}I^&?vcE*t47~Xq#YwvA^6^} z`WkC)$AkNub|t@S!$8CBlwbV~?yp&@9h{D|3z-vJXgzRC5^nYm+PyPcgRzAnEi6Q^gslXYRv4nycsy-SJu?lMps-? zV`U*#WnFsdPLL)Q$AmD|0`UaC4ND07+&UmOu!eHruzV|OUox<+Jl|Mr@6~C`T@P%s zW7sgXLF2SSe9Fl^O(I*{9wsFSYb2l%-;&Pi^dpv!{)C3d0AlNY6!4fgmSgj_wQ*7Am7&$z;Jg&wgR-Ih;lUvWS|KTSg!&s_E9_bXBkZvGiC6bFKDWZxsD$*NZ#_8bl zG1P-#@?OQzED7@jlMJTH@V!6k;W>auvft)}g zhoV{7$q=*;=l{O>Q4a@ ziMjf_u*o^PsO)#BjC%0^h>Xp@;5$p{JSYDt)zbb}s{Kbt!T*I@Pk@X0zds6wsefuU zW$XY%yyRGC94=6mf?x+bbA5CDQ2AgW1T-jVAJbm7K(gp+;v6E0WI#kuACgV$r}6L? zd|Tj?^%^*N&b>Dd{Wr$FS2qI#Ucs1yd4N+RBUQiSZGujH`#I)mG&VKoDh=KKFl4=G z&MagXl6*<)$6P}*Tiebpz5L=oMaPrN+caUXRJ`D?=K9!e0f{@D&cZLKN?iNP@X0aF zE(^pl+;*T5qt?1jRC=5PMgV!XNITRLS_=9{CJExaQj;lt!&pdzpK?8p>%Mb+D z?yO*uSung=-`QQ@yX@Hyd4@CI^r{2oiu`%^bNkz+Nkk!IunjwNC|WcqvX~k=><-I3 zDQdbdb|!v+Iz01$w@aMl!R)koD77Xp;eZwzSl-AT zr@Vu{=xvgfq9akRrrM)}=!=xcs+U1JO}{t(avgz`6RqiiX<|hGG1pmop8k6Q+G_mv zJv|RfDheUp2L3=^C=4aCBMBn0aRCU(DQwX-W(RkRwmLeuJYF<0urcaf(=7)JPg<3P zQs!~G)9CT18o!J4{zX{_e}4eS)U-E)0FAt}wEI(c0%HkxgggW;(1E=>J17_hsH^sP z%lT0LGgbUXHx-K*CI-MCrP66UP0PvGqM$MkeLyqHdbgP|_Cm!7te~b8p+e6sQ_3k| zVcwTh6d83ltdnR>D^)BYQpDKlLk3g0Hdcgz2}%qUs9~~Rie)A-BV1mS&naYai#xcZ z(d{8=-LVpTp}2*y)|gR~;qc7fp26}lPcLZ#=JpYcn3AT9(UIdOyg+d(P5T7D&*P}# zQCYplZO5|7+r19%9e`v^vfSS1sbX1c%=w1;oyruXB%Kl$ACgKQ6=qNWLsc=28xJjg zwvsI5-%SGU|3p>&zXVl^vVtQT3o-#$UT9LI@Npz~6=4!>mc431VRNN8od&Ul^+G_kHC`G=6WVWM z%9eWNyy(FTO|A+@x}Ou3CH)oi;t#7rAxdIXfNFwOj_@Y&TGz6P_sqiB`Q6Lxy|Q{`|fgmRG(k+!#b*M+Z9zFce)f-7;?Km5O=LHV9f9_87; zF7%R2B+$?@sH&&-$@tzaPYkw0;=i|;vWdI|Wl3q_Zu>l;XdIw2FjV=;Mq5t1Q0|f< zs08j54Bp`3RzqE=2enlkZxmX6OF+@|2<)A^RNQpBd6o@OXl+i)zO%D4iGiQNuXd+zIR{_lb96{lc~bxsBveIw6umhShTX+3@ZJ=YHh@ zWY3(d0azg;7oHn>H<>?4@*RQbi>SmM=JrHvIG(~BrvI)#W(EAeO6fS+}mxxcc+X~W6&YVl86W9WFSS}Vz-f9vS?XUDBk)3TcF z8V?$4Q)`uKFq>xT=)Y9mMFVTUk*NIA!0$?RP6Ig0TBmUFrq*Q-Agq~DzxjStQyJ({ zBeZ;o5qUUKg=4Hypm|}>>L=XKsZ!F$yNTDO)jt4H0gdQ5$f|d&bnVCMMXhNh)~mN z@_UV6D7MVlsWz+zM+inZZp&P4fj=tm6fX)SG5H>OsQf_I8c~uGCig$GzuwViK54bcgL;VN|FnyQl>Ed7(@>=8$a_UKIz|V6CeVSd2(P z0Uu>A8A+muM%HLFJQ9UZ5c)BSAv_zH#1f02x?h9C}@pN@6{>UiAp>({Fn(T9Q8B z^`zB;kJ5b`>%dLm+Ol}ty!3;8f1XDSVX0AUe5P#@I+FQ-`$(a;zNgz)4x5hz$Hfbg z!Q(z26wHLXko(1`;(BAOg_wShpX0ixfWq3ponndY+u%1gyX)_h=v1zR#V}#q{au6; z!3K=7fQwnRfg6FXtNQmP>`<;!N137paFS%y?;lb1@BEdbvQHYC{976l`cLqn;b8lp zIDY>~m{gDj(wfnK!lpW6pli)HyLEiUrNc%eXTil|F2s(AY+LW5hkKb>TQ3|Q4S9rr zpDs4uK_co6XPsn_z$LeS{K4jFF`2>U`tbgKdyDne`xmR<@6AA+_hPNKCOR-Zqv;xk zu5!HsBUb^!4uJ7v0RuH-7?l?}b=w5lzzXJ~gZcxRKOovSk@|#V+MuX%Y+=;14i*%{)_gSW9(#4%)AV#3__kac1|qUy!uyP{>?U#5wYNq}y$S9pCc zFc~4mgSC*G~j0u#qqp9 z${>3HV~@->GqEhr_Xwoxq?Hjn#=s2;i~g^&Hn|aDKpA>Oc%HlW(KA1?BXqpxB;Ydx)w;2z^MpjJ(Qi(X!$5RC z*P{~%JGDQqojV>2JbEeCE*OEu!$XJ>bWA9Oa_Hd;y)F%MhBRi*LPcdqR8X`NQ&1L# z5#9L*@qxrx8n}LfeB^J{%-?SU{FCwiWyHp682F+|pa+CQa3ZLzBqN1{)h4d6+vBbV zC#NEbQLC;}me3eeYnOG*nXOJZEU$xLZ1<1Y=7r0(-U0P6-AqwMAM`a(Ed#7vJkn6plb4eI4?2y3yOTGmmDQ!z9`wzbf z_OY#0@5=bnep;MV0X_;;SJJWEf^E6Bd^tVJ9znWx&Ks8t*B>AM@?;D4oWUGc z!H*`6d7Cxo6VuyS4Eye&L1ZRhrRmN6Lr`{NL(wDbif|y&z)JN>Fl5#Wi&mMIr5i;x zBx}3YfF>>8EC(fYnmpu~)CYHuHCyr5*`ECap%t@y=jD>!_%3iiE|LN$mK9>- zHdtpy8fGZtkZF?%TW~29JIAfi2jZT8>OA7=h;8T{{k?c2`nCEx9$r zS+*&vt~2o^^J+}RDG@+9&M^K*z4p{5#IEVbz`1%`m5c2};aGt=V?~vIM}ZdPECDI)47|CWBCfDWUbxBCnmYivQ*0Nu_xb*C>~C9(VjHM zxe<*D<#dQ8TlpMX2c@M<9$w!RP$hpG4cs%AI){jp*Sj|*`m)5(Bw*A0$*i-(CA5#%>a)$+jI2C9r6|(>J8InryENI z$NohnxDUB;wAYDwrb*!N3noBTKPpPN}~09SEL18tkG zxgz(RYU_;DPT{l?Q$+eaZaxnsWCA^ds^0PVRkIM%bOd|G2IEBBiz{&^JtNsODs;5z zICt_Zj8wo^KT$7Bg4H+y!Df#3mbl%%?|EXe!&(Vmac1DJ*y~3+kRKAD=Ovde4^^%~ zw<9av18HLyrf*_>Slp;^i`Uy~`mvBjZ|?Ad63yQa#YK`4+c6;pW4?XIY9G1(Xh9WO8{F-Aju+nS9Vmv=$Ac0ienZ+p9*O%NG zMZKy5?%Z6TAJTE?o5vEr0r>f>hb#2w2U3DL64*au_@P!J!TL`oH2r*{>ffu6|A7tv zL4juf$DZ1MW5ZPsG!5)`k8d8c$J$o;%EIL0va9&GzWvkS%ZsGb#S(?{!UFOZ9<$a| zY|a+5kmD5N&{vRqkgY>aHsBT&`rg|&kezoD)gP0fsNYHsO#TRc_$n6Lf1Z{?+DLziXlHrq4sf(!>O{?Tj;Eh@%)+nRE_2VxbN&&%%caU#JDU%vL3}Cb zsb4AazPI{>8H&d=jUaZDS$-0^AxE@utGs;-Ez_F(qC9T=UZX=>ok2k2 ziTn{K?y~a5reD2A)P${NoI^>JXn>`IeArow(41c-Wm~)wiryEP(OS{YXWi7;%dG9v zI?mwu1MxD{yp_rrk!j^cKM)dc4@p4Ezyo%lRN|XyD}}>v=Xoib0gOcdXrQ^*61HNj z=NP|pd>@yfvr-=m{8$3A8TQGMTE7g=z!%yt`8`Bk-0MMwW~h^++;qyUP!J~ykh1GO z(FZ59xuFR$(WE;F@UUyE@Sp>`aVNjyj=Ty>_Vo}xf`e7`F;j-IgL5`1~-#70$9_=uBMq!2&1l zomRgpD58@)YYfvLtPW}{C5B35R;ZVvB<<#)x%srmc_S=A7F@DW8>QOEGwD6suhwCg z>Pa+YyULhmw%BA*4yjDp|2{!T98~<6Yfd(wo1mQ!KWwq0eg+6)o1>W~f~kL<-S+P@$wx*zeI|1t7z#Sxr5 zt6w+;YblPQNplq4Z#T$GLX#j6yldXAqj>4gAnnWtBICUnA&-dtnlh=t0Ho_vEKwV` z)DlJi#!@nkYV#$!)@>udAU*hF?V`2$Hf=V&6PP_|r#Iv*J$9)pF@X3`k;5})9^o4y z&)~?EjX5yX12O(BsFy-l6}nYeuKkiq`u9145&3Ssg^y{5G3Pse z9w(YVa0)N-fLaBq1`P!_#>SS(8fh_5!f{UrgZ~uEdeMJIz7DzI5!NHHqQtm~#CPij z?=N|J>nPR6_sL7!f4hD_|KH`vf8(Wpnj-(gPWH+ZvID}%?~68SwhPTC3u1_cB`otq z)U?6qo!ZLi5b>*KnYHWW=3F!p%h1;h{L&(Q&{qY6)_qxNfbP6E3yYpW!EO+IW3?@J z);4>g4gnl^8klu7uA>eGF6rIGSynacogr)KUwE_R4E5Xzi*Qir@b-jy55-JPC8c~( zo!W8y9OGZ&`xmc8;=4-U9=h{vCqfCNzYirONmGbRQlR`WWlgnY+1wCXbMz&NT~9*| z6@FrzP!LX&{no2!Ln_3|I==_4`@}V?4a;YZKTdw;vT<+K+z=uWbW(&bXEaWJ^W8Td z-3&1bY^Z*oM<=M}LVt>_j+p=2Iu7pZmbXrhQ_k)ysE9yXKygFNw$5hwDn(M>H+e1&9BM5!|81vd%r%vEm zqxY3?F@fb6O#5UunwgAHR9jp_W2zZ}NGp2%mTW@(hz7$^+a`A?mb8|_G*GNMJ) zjqegXQio=i@AINre&%ofexAr95aop5C+0MZ0m-l=MeO8m3epm7U%vZB8+I+C*iNFM z#T3l`gknX;D$-`2XT^Cg*vrv=RH+P;_dfF++cP?B_msQI4j+lt&rX2)3GaJx%W*Nn zkML%D{z5tpHH=dksQ*gzc|}gzW;lwAbxoR07VNgS*-c3d&8J|;@3t^ zVUz*J*&r7DFRuFVDCJDK8V9NN5hvpgGjwx+5n)qa;YCKe8TKtdnh{I7NU9BCN!0dq zczrBk8pE{{@vJa9ywR@mq*J=v+PG;?fwqlJVhijG!3VmIKs>9T6r7MJpC)m!Tc#>g zMtVsU>wbwFJEfwZ{vB|ZlttNe83)$iz`~#8UJ^r)lJ@HA&G#}W&ZH*;k{=TavpjWE z7hdyLZPf*X%Gm}i`Y{OGeeu^~nB8=`{r#TUrM-`;1cBvEd#d!kPqIgYySYhN-*1;L z^byj%Yi}Gx)Wnkosi337BKs}+5H5dth1JA{Ir-JKN$7zC)*}hqeoD(WfaUDPT>0`- z(6sa0AoIqASwF`>hP}^|)a_j2s^PQn*qVC{Q}htR z5-)duBFXT_V56-+UohKXlq~^6uf!6sA#ttk1o~*QEy_Y-S$gAvq47J9Vtk$5oA$Ct zYhYJ@8{hsC^98${!#Ho?4y5MCa7iGnfz}b9jE~h%EAAv~Qxu)_rAV;^cygV~5r_~?l=B`zObj7S=H=~$W zPtI_m%g$`kL_fVUk9J@>EiBH zOO&jtn~&`hIFMS5S`g8w94R4H40mdNUH4W@@XQk1sr17b{@y|JB*G9z1|CrQjd+GX z6+KyURG3;!*BQrentw{B2R&@2&`2}n(z-2&X7#r!{yg@Soy}cRD~j zj9@UBW+N|4HW4AWapy4wfUI- zZ`gSL6DUlgj*f1hSOGXG0IVH8HxK?o2|3HZ;KW{K+yPAlxtb)NV_2AwJm|E)FRs&& z=c^e7bvUsztY|+f^k7NXs$o1EUq>cR7C0$UKi6IooHWlK_#?IWDkvywnzg&ThWo^? z2O_N{5X39#?eV9l)xI(>@!vSB{DLt*oY!K1R8}_?%+0^C{d9a%N4 zoxHVT1&Lm|uDX%$QrBun5e-F`HJ^T$ zmzv)p@4ZHd_w9!%Hf9UYNvGCw2TTTbrj9pl+T9%-_-}L(tES>Or-}Z4F*{##n3~L~TuxjirGuIY#H7{%$E${?p{Q01 zi6T`n;rbK1yIB9jmQNycD~yZq&mbIsFWHo|ZAChSFPQa<(%d8mGw*V3fh|yFoxOOiWJd(qvVb!Z$b88cg->N=qO*4k~6;R==|9ihg&riu#P~s4Oap9O7f%crSr^rljeIfXDEg>wi)&v*a%7zpz<9w z*r!3q9J|390x`Zk;g$&OeN&ctp)VKRpDSV@kU2Q>jtok($Y-*x8_$2piTxun81@vt z!Vj?COa0fg2RPXMSIo26T=~0d`{oGP*eV+$!0I<(4azk&Vj3SiG=Q!6mX0p$z7I}; z9BJUFgT-K9MQQ-0@Z=^7R<{bn2Fm48endsSs`V7_@%8?Bxkqv>BDoVcj?K#dV#uUP zL1ND~?D-|VGKe3Rw_7-Idpht>H6XRLh*U7epS6byiGvJpr%d}XwfusjH9g;Z98H`x zyde%%5mhGOiL4wljCaWCk-&uE4_OOccb9c!ZaWt4B(wYl!?vyzl%7n~QepN&eFUrw zFIOl9c({``6~QD+43*_tzP{f2x41h(?b43^y6=iwyB)2os5hBE!@YUS5?N_tXd=h( z)WE286Fbd>R4M^P{!G)f;h<3Q>Fipuy+d2q-)!RyTgt;wr$(?9ox3;q+{E*ZQHhOn;lM`cjnu9 zXa48ks-v(~b*;MAI<>YZH(^NV8vjb34beE<_cwKlJoR;k6lJNSP6v}uiyRD?|0w+X@o1ONrH8a$fCxXpf? z?$DL0)7|X}Oc%h^zrMKWc-NS9I0Utu@>*j}b@tJ=ixQSJ={4@854wzW@E>VSL+Y{i z#0b=WpbCZS>kUCO_iQz)LoE>P5LIG-hv9E+oG}DtlIDF>$tJ1aw9^LuhLEHt?BCj& z(O4I8v1s#HUi5A>nIS-JK{v!7dJx)^Yg%XjNmlkWAq2*cv#tHgz`Y(bETc6CuO1VkN^L-L3j_x<4NqYb5rzrLC-7uOv z!5e`GZt%B782C5-fGnn*GhDF$%(qP<74Z}3xx+{$4cYKy2ikxI7B2N+2r07DN;|-T->nU&!=Cm#rZt%O_5c&1Z%nlWq3TKAW0w zQqemZw_ue--2uKQsx+niCUou?HjD`xhEjjQd3%rrBi82crq*~#uA4+>vR<_S{~5ce z-2EIl?~s z1=GVL{NxP1N3%=AOaC}j_Fv=ur&THz zyO!d9kHq|c73kpq`$+t+8Bw7MgeR5~`d7ChYyGCBWSteTB>8WAU(NPYt2Dk`@#+}= zI4SvLlyk#pBgVigEe`?NG*vl7V6m+<}%FwPV=~PvvA)=#ths==DRTDEYh4V5}Cf$z@#;< zyWfLY_5sP$gc3LLl2x+Ii)#b2nhNXJ{R~vk`s5U7Nyu^3yFg&D%Txwj6QezMX`V(x z=C`{76*mNb!qHHs)#GgGZ_7|vkt9izl_&PBrsu@}L`X{95-2jf99K)0=*N)VxBX2q z((vkpP2RneSIiIUEnGb?VqbMb=Zia+rF~+iqslydE34cSLJ&BJW^3knX@M;t*b=EA zNvGzv41Ld_T+WT#XjDB840vovUU^FtN_)G}7v)1lPetgpEK9YS^OWFkPoE{ovj^=@ zO9N$S=G$1ecndT_=5ehth2Lmd1II-PuT~C9`XVePw$y8J#dpZ?Tss<6wtVglm(Ok7 z3?^oi@pPio6l&!z8JY(pJvG=*pI?GIOu}e^EB6QYk$#FJQ%^AIK$I4epJ+9t?KjqA+bkj&PQ*|vLttme+`9G=L% ziadyMw_7-M)hS(3E$QGNCu|o23|%O+VN7;Qggp?PB3K-iSeBa2b}V4_wY`G1Jsfz4 z9|SdB^;|I8E8gWqHKx!vj_@SMY^hLEIbSMCuE?WKq=c2mJK z8LoG-pnY!uhqFv&L?yEuxo{dpMTsmCn)95xanqBrNPTgXP((H$9N${Ow~Is-FBg%h z53;|Y5$MUN)9W2HBe2TD`ct^LHI<(xWrw}$qSoei?}s)&w$;&!14w6B6>Yr6Y8b)S z0r71`WmAvJJ`1h&poLftLUS6Ir zC$bG9!Im_4Zjse)#K=oJM9mHW1{%l8sz$1o?ltdKlLTxWWPB>Vk22czVt|1%^wnN@*!l)}?EgtvhC>vlHm^t+ogpgHI1_$1ox9e;>0!+b(tBrmXRB`PY1vp-R**8N7 zGP|QqI$m(Rdu#=(?!(N}G9QhQ%o!aXE=aN{&wtGP8|_qh+7a_j_sU5|J^)vxq;# zjvzLn%_QPHZZIWu1&mRAj;Sa_97p_lLq_{~j!M9N^1yp3U_SxRqK&JnR%6VI#^E12 z>CdOVI^_9aPK2eZ4h&^{pQs}xsijXgFYRIxJ~N7&BB9jUR1fm!(xl)mvy|3e6-B3j zJn#ajL;bFTYJ2+Q)tDjx=3IklO@Q+FFM}6UJr6km7hj7th9n_&JR7fnqC!hTZoM~T zBeaVFp%)0cbPhejX<8pf5HyRUj2>aXnXBqDJe73~J%P(2C?-RT{c3NjE`)om! zl$uewSgWkE66$Kb34+QZZvRn`fob~Cl9=cRk@Es}KQm=?E~CE%spXaMO6YmrMl%9Q zlA3Q$3|L1QJ4?->UjT&CBd!~ru{Ih^in&JXO=|<6J!&qp zRe*OZ*cj5bHYlz!!~iEKcuE|;U4vN1rk$xq6>bUWD*u(V@8sG^7>kVuo(QL@Ki;yL zWC!FT(q{E8#on>%1iAS0HMZDJg{Z{^!De(vSIq&;1$+b)oRMwA3nc3mdTSG#3uYO_ z>+x;7p4I;uHz?ZB>dA-BKl+t-3IB!jBRgdvAbW!aJ(Q{aT>+iz?91`C-xbe)IBoND z9_Xth{6?(y3rddwY$GD65IT#f3<(0o#`di{sh2gm{dw*#-Vnc3r=4==&PU^hCv$qd zjw;>i&?L*Wq#TxG$mFIUf>eK+170KG;~+o&1;Tom9}}mKo23KwdEM6UonXgc z!6N(@k8q@HPw{O8O!lAyi{rZv|DpgfU{py+j(X_cwpKqcalcqKIr0kM^%Br3SdeD> zHSKV94Yxw;pjzDHo!Q?8^0bb%L|wC;4U^9I#pd5O&eexX+Im{ z?jKnCcsE|H?{uGMqVie_C~w7GX)kYGWAg%-?8|N_1#W-|4F)3YTDC+QSq1s!DnOML3@d`mG%o2YbYd#jww|jD$gotpa)kntakp#K;+yo-_ZF9qrNZw<%#C zuPE@#3RocLgPyiBZ+R_-FJ_$xP!RzWm|aN)S+{$LY9vvN+IW~Kf3TsEIvP+B9Mtm! zpfNNxObWQpLoaO&cJh5>%slZnHl_Q~(-Tfh!DMz(dTWld@LG1VRF`9`DYKhyNv z2pU|UZ$#_yUx_B_|MxUq^glT}O5Xt(Vm4Mr02><%C)@v;vPb@pT$*yzJ4aPc_FZ3z z3}PLoMBIM>q_9U2rl^sGhk1VUJ89=*?7|v`{!Z{6bqFMq(mYiA?%KbsI~JwuqVA9$H5vDE+VocjX+G^%bieqx->s;XWlKcuv(s%y%D5Xbc9+ zc(_2nYS1&^yL*ey664&4`IoOeDIig}y-E~_GS?m;D!xv5-xwz+G`5l6V+}CpeJDi^ z%4ed$qowm88=iYG+(`ld5Uh&>Dgs4uPHSJ^TngXP_V6fPyl~>2bhi20QB%lSd#yYn zO05?KT1z@?^-bqO8Cg`;ft>ilejsw@2%RR7;`$Vs;FmO(Yr3Fp`pHGr@P2hC%QcA|X&N2Dn zYf`MqXdHi%cGR@%y7Rg7?d3?an){s$zA{!H;Ie5exE#c~@NhQUFG8V=SQh%UxUeiV zd7#UcYqD=lk-}sEwlpu&H^T_V0{#G?lZMxL7ih_&{(g)MWBnCZxtXg znr#}>U^6!jA%e}@Gj49LWG@*&t0V>Cxc3?oO7LSG%~)Y5}f7vqUUnQ;STjdDU}P9IF9d9<$;=QaXc zL1^X7>fa^jHBu_}9}J~#-oz3Oq^JmGR#?GO7b9a(=R@fw@}Q{{@`Wy1vIQ#Bw?>@X z-_RGG@wt|%u`XUc%W{J z>iSeiz8C3H7@St3mOr_mU+&bL#Uif;+Xw-aZdNYUpdf>Rvu0i0t6k*}vwU`XNO2he z%miH|1tQ8~ZK!zmL&wa3E;l?!!XzgV#%PMVU!0xrDsNNZUWKlbiOjzH-1Uoxm8E#r`#2Sz;-o&qcqB zC-O_R{QGuynW14@)7&@yw1U}uP(1cov)twxeLus0s|7ayrtT8c#`&2~Fiu2=R;1_4bCaD=*E@cYI>7YSnt)nQc zohw5CsK%m?8Ack)qNx`W0_v$5S}nO|(V|RZKBD+btO?JXe|~^Qqur%@eO~<8-L^9d z=GA3-V14ng9L29~XJ>a5k~xT2152zLhM*@zlp2P5Eu}bywkcqR;ISbas&#T#;HZSf z2m69qTV(V@EkY(1Dk3`}j)JMo%ZVJ*5eB zYOjIisi+igK0#yW*gBGj?@I{~mUOvRFQR^pJbEbzFxTubnrw(Muk%}jI+vXmJ;{Q6 zrSobKD>T%}jV4Ub?L1+MGOD~0Ir%-`iTnWZN^~YPrcP5y3VMAzQ+&en^VzKEb$K!Q z<7Dbg&DNXuow*eD5yMr+#08nF!;%4vGrJI++5HdCFcGLfMW!KS*Oi@=7hFwDG!h2< zPunUEAF+HncQkbfFj&pbzp|MU*~60Z(|Ik%Tn{BXMN!hZOosNIseT?R;A`W?=d?5X zK(FB=9mZusYahp|K-wyb={rOpdn=@;4YI2W0EcbMKyo~-#^?h`BA9~o285%oY zfifCh5Lk$SY@|2A@a!T2V+{^!psQkx4?x0HSV`(w9{l75QxMk!)U52Lbhn{8ol?S) zCKo*7R(z!uk<6*qO=wh!Pul{(qq6g6xW;X68GI_CXp`XwO zxuSgPRAtM8K7}5E#-GM!*ydOOG_{A{)hkCII<|2=ma*71ci_-}VPARm3crFQjLYV! z9zbz82$|l01mv`$WahE2$=fAGWkd^X2kY(J7iz}WGS z@%MyBEO=A?HB9=^?nX`@nh;7;laAjs+fbo!|K^mE!tOB>$2a_O0y-*uaIn8k^6Y zSbuv;5~##*4Y~+y7Z5O*3w4qgI5V^17u*ZeupVGH^nM&$qmAk|anf*>r zWc5CV;-JY-Z@Uq1Irpb^O`L_7AGiqd*YpGUShb==os$uN3yYvb`wm6d=?T*it&pDk zo`vhw)RZX|91^^Wa_ti2zBFyWy4cJu#g)_S6~jT}CC{DJ_kKpT`$oAL%b^!2M;JgT zM3ZNbUB?}kP(*YYvXDIH8^7LUxz5oE%kMhF!rnPqv!GiY0o}NR$OD=ITDo9r%4E>E0Y^R(rS^~XjWyVI6 zMOR5rPXhTp*G*M&X#NTL`Hu*R+u*QNoiOKg4CtNPrjgH>c?Hi4MUG#I917fx**+pJfOo!zFM&*da&G_x)L(`k&TPI*t3e^{crd zX<4I$5nBQ8Ax_lmNRa~E*zS-R0sxkz`|>7q_?*e%7bxqNm3_eRG#1ae3gtV9!fQpY z+!^a38o4ZGy9!J5sylDxZTx$JmG!wg7;>&5H1)>f4dXj;B+@6tMlL=)cLl={jLMxY zbbf1ax3S4>bwB9-$;SN2?+GULu;UA-35;VY*^9Blx)Jwyb$=U!D>HhB&=jSsd^6yw zL)?a|>GxU!W}ocTC(?-%z3!IUhw^uzc`Vz_g>-tv)(XA#JK^)ZnC|l1`@CdX1@|!| z_9gQ)7uOf?cR@KDp97*>6X|;t@Y`k_N@)aH7gY27)COv^P3ya9I{4z~vUjLR9~z1Z z5=G{mVtKH*&$*t0@}-i_v|3B$AHHYale7>E+jP`ClqG%L{u;*ff_h@)al?RuL7tOO z->;I}>%WI{;vbLP3VIQ^iA$4wl6@0sDj|~112Y4OFjMs`13!$JGkp%b&E8QzJw_L5 zOnw9joc0^;O%OpF$Qp)W1HI!$4BaXX84`%@#^dk^hFp^pQ@rx4g(8Xjy#!X%+X5Jd@fs3amGT`}mhq#L97R>OwT5-m|h#yT_-v@(k$q7P*9X~T*3)LTdzP!*B} z+SldbVWrrwQo9wX*%FyK+sRXTa@O?WM^FGWOE?S`R(0P{<6p#f?0NJvnBia?k^fX2 zNQs7K-?EijgHJY}&zsr;qJ<*PCZUd*x|dD=IQPUK_nn)@X4KWtqoJNHkT?ZWL_hF? zS8lp2(q>;RXR|F;1O}EE#}gCrY~#n^O`_I&?&z5~7N;zL0)3Tup`%)oHMK-^r$NT% zbFg|o?b9w(q@)6w5V%si<$!U<#}s#x@0aX-hP>zwS#9*75VXA4K*%gUc>+yzupTDBOKH8WR4V0pM(HrfbQ&eJ79>HdCvE=F z|J>s;;iDLB^3(9}?biKbxf1$lI!*Z%*0&8UUq}wMyPs_hclyQQi4;NUY+x2qy|0J; zhn8;5)4ED1oHwg+VZF|80<4MrL97tGGXc5Sw$wAI#|2*cvQ=jB5+{AjMiDHmhUC*a zlmiZ`LAuAn_}hftXh;`Kq0zblDk8?O-`tnilIh|;3lZp@F_osJUV9`*R29M?7H{Fy z`nfVEIDIWXmU&YW;NjU8)EJpXhxe5t+scf|VXM!^bBlwNh)~7|3?fWwo_~ZFk(22% zTMesYw+LNx3J-_|DM~`v93yXe=jPD{q;li;5PD?Dyk+b? zo21|XpT@)$BM$%F=P9J19Vi&1#{jM3!^Y&fr&_`toi`XB1!n>sbL%U9I5<7!@?t)~ z;&H%z>bAaQ4f$wIzkjH70;<8tpUoxzKrPhn#IQfS%9l5=Iu))^XC<58D!-O z{B+o5R^Z21H0T9JQ5gNJnqh#qH^na|z92=hONIM~@_iuOi|F>jBh-?aA20}Qx~EpDGElELNn~|7WRXRFnw+Wdo`|# zBpU=Cz3z%cUJ0mx_1($X<40XEIYz(`noWeO+x#yb_pwj6)R(__%@_Cf>txOQ74wSJ z0#F3(zWWaR-jMEY$7C*3HJrohc79>MCUu26mfYN)f4M~4gD`}EX4e}A!U}QV8!S47 z6y-U-%+h`1n`*pQuKE%Av0@)+wBZr9mH}@vH@i{v(m-6QK7Ncf17x_D=)32`FOjjo zg|^VPf5c6-!FxN{25dvVh#fog=NNpXz zfB$o+0jbRkHH{!TKhE709f+jI^$3#v1Nmf80w`@7-5$1Iv_`)W^px8P-({xwb;D0y z7LKDAHgX<84?l!I*Dvi2#D@oAE^J|g$3!)x1Ua;_;<@#l1fD}lqU2_tS^6Ht$1Wl} zBESo7o^)9-Tjuz$8YQSGhfs{BQV6zW7dA?0b(Dbt=UnQs&4zHfe_sj{RJ4uS-vQpC zX;Bbsuju4%!o8?&m4UZU@~ZZjeFF6ex2ss5_60_JS_|iNc+R0GIjH1@Z z=rLT9%B|WWgOrR7IiIwr2=T;Ne?30M!@{%Qf8o`!>=s<2CBpCK_TWc(DX51>e^xh8 z&@$^b6CgOd7KXQV&Y4%}_#uN*mbanXq(2=Nj`L7H7*k(6F8s6{FOw@(DzU`4-*77{ zF+dxpv}%mFpYK?>N_2*#Y?oB*qEKB}VoQ@bzm>ptmVS_EC(#}Lxxx730trt0G)#$b zE=wVvtqOct1%*9}U{q<)2?{+0TzZzP0jgf9*)arV)*e!f`|jgT{7_9iS@e)recI#z zbzolURQ+TOzE!ymqvBY7+5NnAbWxvMLsLTwEbFqW=CPyCsmJ}P1^V30|D5E|p3BC5 z)3|qgw@ra7aXb-wsa|l^in~1_fm{7bS9jhVRkYVO#U{qMp z)Wce+|DJ}4<2gp8r0_xfZpMo#{Hl2MfjLcZdRB9(B(A(f;+4s*FxV{1F|4d`*sRNd zp4#@sEY|?^FIJ;tmH{@keZ$P(sLh5IdOk@k^0uB^BWr@pk6mHy$qf&~rI>P*a;h0C{%oA*i!VjWn&D~O#MxN&f@1Po# zKN+ zrGrkSjcr?^R#nGl<#Q722^wbYcgW@{+6CBS<1@%dPA8HC!~a`jTz<`g_l5N1M@9wn9GOAZ>nqNgq!yOCbZ@1z`U_N`Z>}+1HIZxk*5RDc&rd5{3qjRh8QmT$VyS;jK z;AF+r6XnnCp=wQYoG|rT2@8&IvKq*IB_WvS%nt%e{MCFm`&W*#LXc|HrD?nVBo=(8*=Aq?u$sDA_sC_RPDUiQ+wnIJET8vx$&fxkW~kP9qXKt zozR)@xGC!P)CTkjeWvXW5&@2?)qt)jiYWWBU?AUtzAN}{JE1I)dfz~7$;}~BmQF`k zpn11qmObXwRB8&rnEG*#4Xax3XBkKlw(;tb?Np^i+H8m(Wyz9k{~ogba@laiEk;2! zV*QV^6g6(QG%vX5Um#^sT&_e`B1pBW5yVth~xUs#0}nv?~C#l?W+9Lsb_5)!71rirGvY zTIJ$OPOY516Y|_014sNv+Z8cc5t_V=i>lWV=vNu#!58y9Zl&GsMEW#pPYPYGHQ|;vFvd*9eM==$_=vc7xnyz0~ zY}r??$<`wAO?JQk@?RGvkWVJlq2dk9vB(yV^vm{=NVI8dhsX<)O(#nr9YD?I?(VmQ z^r7VfUBn<~p3()8yOBjm$#KWx!5hRW)5Jl7wY@ky9lNM^jaT##8QGVsYeaVywmpv>X|Xj7gWE1Ezai&wVLt3p)k4w~yrskT-!PR!kiyQlaxl(( zXhF%Q9x}1TMt3~u@|#wWm-Vq?ZerK={8@~&@9r5JW}r#45#rWii};t`{5#&3$W)|@ zbAf2yDNe0q}NEUvq_Quq3cTjcw z@H_;$hu&xllCI9CFDLuScEMg|x{S7GdV8<&Mq=ezDnRZAyX-8gv97YTm0bg=d)(>N z+B2FcqvI9>jGtnK%eO%y zoBPkJTk%y`8TLf4)IXPBn`U|9>O~WL2C~C$z~9|0m*YH<-vg2CD^SX#&)B4ngOSG$ zV^wmy_iQk>dfN@Pv(ckfy&#ak@MLC7&Q6Ro#!ezM*VEh`+b3Jt%m(^T&p&WJ2Oqvj zs-4nq0TW6cv~(YI$n0UkfwN}kg3_fp?(ijSV#tR9L0}l2qjc7W?i*q01=St0eZ=4h zyGQbEw`9OEH>NMuIe)hVwYHsGERWOD;JxEiO7cQv%pFCeR+IyhwQ|y@&^24k+|8fD zLiOWFNJ2&vu2&`Jv96_z-Cd5RLgmeY3*4rDOQo?Jm`;I_(+ejsPM03!ly!*Cu}Cco zrQSrEDHNyzT(D5s1rZq!8#?f6@v6dB7a-aWs(Qk>N?UGAo{gytlh$%_IhyL7h?DLXDGx zgxGEBQoCAWo-$LRvM=F5MTle`M})t3vVv;2j0HZY&G z22^iGhV@uaJh(XyyY%} zd4iH_UfdV#T=3n}(Lj^|n;O4|$;xhu*8T3hR1mc_A}fK}jfZ7LX~*n5+`8N2q#rI$ z@<_2VANlYF$vIH$ zl<)+*tIWW78IIINA7Rr7i{<;#^yzxoLNkXL)eSs=%|P>$YQIh+ea_3k z_s7r4%j7%&*NHSl?R4k%1>Z=M9o#zxY!n8sL5>BO-ZP;T3Gut>iLS@U%IBrX6BA3k z)&@q}V8a{X<5B}K5s(c(LQ=%v1ocr`t$EqqY0EqVjr65usa=0bkf|O#ky{j3)WBR(((L^wmyHRzoWuL2~WTC=`yZ zn%VX`L=|Ok0v7?s>IHg?yArBcync5rG#^+u)>a%qjES%dRZoIyA8gQ;StH z1Ao7{<&}6U=5}4v<)1T7t!J_CL%U}CKNs-0xWoTTeqj{5{?Be$L0_tk>M9o8 zo371}S#30rKZFM{`H_(L`EM9DGp+Mifk&IP|C2Zu_)Ghr4Qtpmkm1osCf@%Z$%t+7 zYH$Cr)Ro@3-QDeQJ8m+x6%;?YYT;k6Z0E-?kr>x33`H%*ueBD7Zx~3&HtWn0?2Wt} zTG}*|v?{$ajzt}xPzV%lL1t-URi8*Zn)YljXNGDb>;!905Td|mpa@mHjIH%VIiGx- zd@MqhpYFu4_?y5N4xiHn3vX&|e6r~Xt> zZG`aGq|yTNjv;9E+Txuoa@A(9V7g?1_T5FzRI;!=NP1Kqou1z5?%X~Wwb{trRfd>i z8&y^H)8YnKyA_Fyx>}RNmQIczT?w2J4SNvI{5J&}Wto|8FR(W;Qw#b1G<1%#tmYzQ zQ2mZA-PAdi%RQOhkHy9Ea#TPSw?WxwL@H@cbkZwIq0B!@ns}niALidmn&W?!Vd4Gj zO7FiuV4*6Mr^2xlFSvM;Cp_#r8UaqIzHJQg_z^rEJw&OMm_8NGAY2)rKvki|o1bH~ z$2IbfVeY2L(^*rMRU1lM5Y_sgrDS`Z??nR2lX;zyR=c%UyGb*%TC-Dil?SihkjrQy~TMv6;BMs7P8il`H7DmpVm@rJ;b)hW)BL)GjS154b*xq-NXq2cwE z^;VP7ua2pxvCmxrnqUYQMH%a%nHmwmI33nJM(>4LznvY*k&C0{8f*%?zggpDgkuz&JBx{9mfb@wegEl2v!=}Sq2Gaty0<)UrOT0{MZtZ~j5y&w zXlYa_jY)I_+VA-^#mEox#+G>UgvM!Ac8zI<%JRXM_73Q!#i3O|)lOP*qBeJG#BST0 zqohi)O!|$|2SeJQo(w6w7%*92S})XfnhrH_Z8qe!G5>CglP=nI7JAOW?(Z29;pXJ9 zR9`KzQ=WEhy*)WH>$;7Cdz|>*i>=##0bB)oU0OR>>N<21e4rMCHDemNi2LD>Nc$;& zQRFthpWniC1J6@Zh~iJCoLOxN`oCKD5Q4r%ynwgUKPlIEd#?QViIqovY|czyK8>6B zSP%{2-<;%;1`#0mG^B(8KbtXF;Nf>K#Di72UWE4gQ%(_26Koiad)q$xRL~?pN71ZZ zujaaCx~jXjygw;rI!WB=xrOJO6HJ!!w}7eiivtCg5K|F6$EXa)=xUC za^JXSX98W`7g-tm@uo|BKj39Dl;sg5ta;4qjo^pCh~{-HdLl6qI9Ix6f$+qiZ$}s= zNguKrU;u+T@ko(Vr1>)Q%h$?UKXCY>3se%&;h2osl2D zE4A9bd7_|^njDd)6cI*FupHpE3){4NQ*$k*cOWZ_?CZ>Z4_fl@n(mMnYK62Q1d@+I zr&O))G4hMihgBqRIAJkLdk(p(D~X{-oBUA+If@B}j& zsHbeJ3RzTq96lB7d($h$xTeZ^gP0c{t!Y0c)aQE;$FY2!mACg!GDEMKXFOPI^)nHZ z`aSPJpvV0|bbrzhWWkuPURlDeN%VT8tndV8?d)eN*i4I@u zVKl^6{?}A?P)Fsy?3oi#clf}L18t;TjNI2>eI&(ezDK7RyqFxcv%>?oxUlonv(px) z$vnPzRH`y5A(x!yOIfL0bmgeMQB$H5wenx~!ujQK*nUBW;@Em&6Xv2%s(~H5WcU2R z;%Nw<$tI)a`Ve!>x+qegJnQsN2N7HaKzrFqM>`6R*gvh%O*-%THt zrB$Nk;lE;z{s{r^PPm5qz(&lM{sO*g+W{sK+m3M_z=4=&CC>T`{X}1Vg2PEfSj2x_ zmT*(x;ov%3F?qoEeeM>dUn$a*?SIGyO8m806J1W1o+4HRhc2`9$s6hM#qAm zChQ87b~GEw{ADfs+5}FJ8+|bIlIv(jT$Ap#hSHoXdd9#w<#cA<1Rkq^*EEkknUd4& zoIWIY)sAswy6fSERVm&!SO~#iN$OgOX*{9@_BWFyJTvC%S++ilSfCrO(?u=Dc?CXZ zzCG&0yVR{Z`|ZF0eEApWEo#s9osV>F{uK{QA@BES#&;#KsScf>y zvs?vIbI>VrT<*!;XmQS=bhq%46-aambZ(8KU-wOO2=en~D}MCToB_u;Yz{)1ySrPZ z@=$}EvjTdzTWU7c0ZI6L8=yP+YRD_eMMos}b5vY^S*~VZysrkq<`cK3>>v%uy7jgq z0ilW9KjVDHLv0b<1K_`1IkbTOINs0=m-22c%M~l=^S}%hbli-3?BnNq?b`hx^HX2J zIe6ECljRL0uBWb`%{EA=%!i^4sMcj+U_TaTZRb+~GOk z^ZW!nky0n*Wb*r+Q|9H@ml@Z5gU&W`(z4-j!OzC1wOke`TRAYGZVl$PmQ16{3196( zO*?`--I}Qf(2HIwb2&1FB^!faPA2=sLg(@6P4mN)>Dc3i(B0;@O-y2;lM4akD>@^v z=u>*|!s&9zem70g7zfw9FXl1bpJW(C#5w#uy5!V?Q(U35A~$dR%LDVnq@}kQm13{} zd53q3N(s$Eu{R}k2esbftfjfOITCL;jWa$}(mmm}d(&7JZ6d3%IABCapFFYjdEjdK z&4Edqf$G^MNAtL=uCDRs&Fu@FXRgX{*0<(@c3|PNHa>L%zvxWS={L8%qw`STm+=Rd zA}FLspESSIpE_^41~#5yI2bJ=9`oc;GIL!JuW&7YetZ?0H}$$%8rW@*J37L-~Rsx!)8($nI4 zZhcZ2^=Y+p4YPl%j!nFJA|*M^gc(0o$i3nlphe+~-_m}jVkRN{spFs(o0ajW@f3K{ zDV!#BwL322CET$}Y}^0ixYj2w>&Xh12|R8&yEw|wLDvF!lZ#dOTHM9pK6@Nm-@9Lnng4ZHBgBSrr7KI8YCC9DX5Kg|`HsiwJHg2(7#nS;A{b3tVO?Z% za{m5b3rFV6EpX;=;n#wltDv1LE*|g5pQ+OY&*6qCJZc5oDS6Z6JD#6F)bWxZSF@q% z+1WV;m!lRB!n^PC>RgQCI#D1br_o^#iPk>;K2hB~0^<~)?p}LG%kigm@moD#q3PE+ zA^Qca)(xnqw6x>XFhV6ku9r$E>bWNrVH9fum0?4s?Rn2LG{Vm_+QJHse6xa%nzQ?k zKug4PW~#Gtb;#5+9!QBgyB@q=sk9=$S{4T>wjFICStOM?__fr+Kei1 z3j~xPqW;W@YkiUM;HngG!;>@AITg}vAE`M2Pj9Irl4w1fo4w<|Bu!%rh%a(Ai^Zhi zs92>v5;@Y(Zi#RI*ua*h`d_7;byQSa*v9E{2x$<-_=5Z<7{%)}4XExANcz@rK69T0x3%H<@frW>RA8^swA+^a(FxK| zFl3LD*ImHN=XDUkrRhp6RY5$rQ{bRgSO*(vEHYV)3Mo6Jy3puiLmU&g82p{qr0F?ohmbz)f2r{X2|T2 z$4fdQ=>0BeKbiVM!e-lIIs8wVTuC_m7}y4A_%ikI;Wm5$9j(^Y z(cD%U%k)X>_>9~t8;pGzL6L-fmQO@K; zo&vQzMlgY95;1BSkngY)e{`n0!NfVgf}2mB3t}D9@*N;FQ{HZ3Pb%BK6;5#-O|WI( zb6h@qTLU~AbVW#_6?c!?Dj65Now7*pU{h!1+eCV^KCuPAGs28~3k@ueL5+u|Z-7}t z9|lskE`4B7W8wMs@xJa{#bsCGDFoRSNSnmNYB&U7 zVGKWe%+kFB6kb)e;TyHfqtU6~fRg)f|>=5(N36)0+C z`hv65J<$B}WUc!wFAb^QtY31yNleq4dzmG`1wHTj=c*=hay9iD071Hc?oYoUk|M*_ zU1GihAMBsM@5rUJ(qS?9ZYJ6@{bNqJ`2Mr+5#hKf?doa?F|+^IR!8lq9)wS3tF_9n zW_?hm)G(M+MYb?V9YoX^_mu5h-LP^TL^!Q9Z7|@sO(rg_4+@=PdI)WL(B7`!K^ND- z-uIuVDCVEdH_C@c71YGYT^_Scf_dhB8Z2Xy6vGtBSlYud9vggOqv^L~F{BraSE_t} zIkP+Hp2&nH^-MNEs}^`oMLy11`PQW$T|K(`Bu*(f@)mv1-qY(_YG&J2M2<7k;;RK~ zL{Fqj9yCz8(S{}@c)S!65aF<=&eLI{hAMErCx&>i7OeDN>okvegO87OaG{Jmi<|}D zaT@b|0X{d@OIJ7zvT>r+eTzgLq~|Dpu)Z&db-P4z*`M$UL51lf>FLlq6rfG)%doyp z)3kk_YIM!03eQ8Vu_2fg{+osaEJPtJ-s36R+5_AEG12`NG)IQ#TF9c@$99%0iye+ zUzZ57=m2)$D(5Nx!n)=5Au&O0BBgwxIBaeI(mro$#&UGCr<;C{UjJVAbVi%|+WP(a zL$U@TYCxJ=1{Z~}rnW;7UVb7+ZnzgmrogDxhjLGo>c~MiJAWs&&;AGg@%U?Y^0JhL ze(x6Z74JG6FlOFK(T}SXQfhr}RIFl@QXKnIcXYF)5|V~e-}suHILKT-k|<*~Ij|VF zC;t@=uj=hot~*!C68G8hTA%8SzOfETOXQ|3FSaIEjvBJp(A)7SWUi5!Eu#yWgY+;n zlm<$+UDou*V+246_o#V4kMdto8hF%%Lki#zPh}KYXmMf?hrN0;>Mv%`@{0Qn`Ujp) z=lZe+13>^Q!9zT);H<(#bIeRWz%#*}sgUX9P|9($kexOyKIOc`dLux}c$7It4u|Rl z6SSkY*V~g_B-hMPo_ak>>z@AVQ(_N)VY2kB3IZ0G(iDUYw+2d7W^~(Jq}KY=JnWS( z#rzEa&0uNhJ>QE8iiyz;n2H|SV#Og+wEZv=f2%1ELX!SX-(d3tEj$5$1}70Mp<&eI zCkfbByL7af=qQE@5vDVxx1}FSGt_a1DoE3SDI+G)mBAna)KBG4p8Epxl9QZ4BfdAN zFnF|Y(umr;gRgG6NLQ$?ZWgllEeeq~z^ZS7L?<(~O&$5|y)Al^iMKy}&W+eMm1W z7EMU)u^ke(A1#XCV>CZ71}P}0x)4wtHO8#JRG3MA-6g=`ZM!FcICCZ{IEw8Dm2&LQ z1|r)BUG^0GzI6f946RrBlfB1Vs)~8toZf~7)+G;pv&XiUO(%5bm)pl=p>nV^o*;&T z;}@oZSibzto$arQgfkp|z4Z($P>dTXE{4O=vY0!)kDO* zGF8a4wq#VaFpLfK!iELy@?-SeRrdz%F*}hjKcA*y@mj~VD3!it9lhRhX}5YOaR9$} z3mS%$2Be7{l(+MVx3 z(4?h;P!jnRmX9J9sYN#7i=iyj_5q7n#X(!cdqI2lnr8T$IfOW<_v`eB!d9xY1P=2q&WtOXY=D9QYteP)De?S4}FK6#6Ma z=E*V+#s8>L;8aVroK^6iKo=MH{4yEZ_>N-N z`(|;aOATba1^asjxlILk<4}f~`39dBFlxj>Dw(hMYKPO3EEt1@S`1lxFNM+J@uB7T zZ8WKjz7HF1-5&2=l=fqF-*@>n5J}jIxdDwpT?oKM3s8Nr`x8JnN-kCE?~aM1H!hAE z%%w(3kHfGwMnMmNj(SU(w42OrC-euI>Dsjk&jz3ts}WHqmMpzQ3vZrsXrZ|}+MHA7 z068obeXZTsO*6RS@o3x80E4ok``rV^Y3hr&C1;|ZZ0|*EKO`$lECUYG2gVFtUTw)R z4Um<0ZzlON`zTdvVdL#KFoMFQX*a5wM0Czp%wTtfK4Sjs)P**RW&?lP$(<}q%r68Z zS53Y!d@&~ne9O)A^tNrXHhXBkj~$8j%pT1%%mypa9AW5E&s9)rjF4@O3ytH{0z6riz|@< zB~UPh*wRFg2^7EbQrHf0y?E~dHlkOxof_a?M{LqQ^C!i2dawHTPYUE=X@2(3<=OOxs8qn_(y>pU>u^}3y&df{JarR0@VJn0f+U%UiF=$Wyq zQvnVHESil@d|8&R<%}uidGh7@u^(%?$#|&J$pvFC-n8&A>utA=n3#)yMkz+qnG3wd zP7xCnF|$9Dif@N~L)Vde3hW8W!UY0BgT2v(wzp;tlLmyk2%N|0jfG$%<;A&IVrOI< z!L)o>j>;dFaqA3pL}b-Je(bB@VJ4%!JeX@3x!i{yIeIso^=n?fDX`3bU=eG7sTc%g%ye8$v8P@yKE^XD=NYxTb zbf!Mk=h|otpqjFaA-vs5YOF-*GwWPc7VbaOW&stlANnCN8iftFMMrUdYNJ_Bnn5Vt zxfz@Ah|+4&P;reZxp;MmEI7C|FOv8NKUm8njF7Wb6Gi7DeODLl&G~}G4be&*Hi0Qw z5}77vL0P+7-B%UL@3n1&JPxW^d@vVwp?u#gVcJqY9#@-3X{ok#UfW3<1fb%FT`|)V~ggq z(3AUoUS-;7)^hCjdT0Kf{i}h)mBg4qhtHHBti=~h^n^OTH5U*XMgDLIR@sre`AaB$ zg)IGBET_4??m@cx&c~bA80O7B8CHR7(LX7%HThkeC*@vi{-pL%e)yXp!B2InafbDF zjPXf1mko3h59{lT6EEbxKO1Z5GF71)WwowO6kY|6tjSVSWdQ}NsK2x{>i|MKZK8%Q zfu&_0D;CO-Jg0#YmyfctyJ!mRJp)e#@O0mYdp|8x;G1%OZQ3Q847YWTyy|%^cpA;m zze0(5p{tMu^lDkpe?HynyO?a1$_LJl2L&mpeKu%8YvgRNr=%2z${%WThHG=vrWY@4 zsA`OP#O&)TetZ>s%h!=+CE15lOOls&nvC~$Qz0Ph7tHiP;O$i|eDwpT{cp>+)0-|; zY$|bB+Gbel>5aRN3>c0x)4U=|X+z+{ zn*_p*EQoquRL+=+p;=lm`d71&1NqBz&_ph)MXu(Nv6&XE7(RsS)^MGj5Q?Fwude-(sq zjJ>aOq!7!EN>@(fK7EE#;i_BGvli`5U;r!YA{JRodLBc6-`n8K+Fjgwb%sX;j=qHQ z7&Tr!)!{HXoO<2BQrV9Sw?JRaLXV8HrsNevvnf>Y-6|{T!pYLl7jp$-nEE z#X!4G4L#K0qG_4Z;Cj6=;b|Be$hi4JvMH!-voxqx^@8cXp`B??eFBz2lLD8RRaRGh zn7kUfy!YV~p(R|p7iC1Rdgt$_24i0cd-S8HpG|`@my70g^y`gu%#Tf_L21-k?sRRZHK&at(*ED0P8iw{7?R$9~OF$Ko;Iu5)ur5<->x!m93Eb zFYpIx60s=Wxxw=`$aS-O&dCO_9?b1yKiPCQmSQb>T)963`*U+Ydj5kI(B(B?HNP8r z*bfSBpSu)w(Z3j7HQoRjUG(+d=IaE~tv}y14zHHs|0UcN52fT8V_<@2ep_ee{QgZG zmgp8iv4V{k;~8@I%M3<#B;2R>Ef(Gg_cQM7%}0s*^)SK6!Ym+~P^58*wnwV1BW@eG z4sZLqsUvBbFsr#8u7S1r4teQ;t)Y@jnn_m5jS$CsW1um!p&PqAcc8!zyiXHVta9QC zY~wCwCF0U%xiQPD_INKtTb;A|Zf29(mu9NI;E zc-e>*1%(LSXB`g}kd`#}O;veb<(sk~RWL|f3ljxCnEZDdNSTDV6#Td({6l&y4IjKF z^}lIUq*ZUqgTPumD)RrCN{M^jhY>E~1pn|KOZ5((%F)G|*ZQ|r4zIbrEiV%42hJV8 z3xS)=!X1+=olbdGJ=yZil?oXLct8FM{(6ikLL3E%=q#O6(H$p~gQu6T8N!plf!96| z&Q3=`L~>U0zZh;z(pGR2^S^{#PrPxTRHD1RQOON&f)Siaf`GLj#UOk&(|@0?zm;Sx ztsGt8=29-MZs5CSf1l1jNFtNt5rFNZxJPvkNu~2}7*9468TWm>nN9TP&^!;J{-h)_ z7WsHH9|F%I`Pb!>KAS3jQWKfGivTVkMJLO-HUGM_a4UQ_%RgL6WZvrW+Z4ujZn;y@ zz9$=oO!7qVTaQAA^BhX&ZxS*|5dj803M=k&2%QrXda`-Q#IoZL6E(g+tN!6CA!CP* zCpWtCujIea)ENl0liwVfj)Nc<9mV%+e@=d`haoZ*`B7+PNjEbXBkv=B+Pi^~L#EO$D$ZqTiD8f<5$eyb54-(=3 zh)6i8i|jp(@OnRrY5B8t|LFXFQVQ895n*P16cEKTrT*~yLH6Z4e*bZ5otpRDri&+A zfNbK1D5@O=sm`fN=WzWyse!za5n%^+6dHPGX#8DyIK>?9qyX}2XvBWVqbP%%D)7$= z=#$WulZlZR<{m#gU7lwqK4WS1Ne$#_P{b17qe$~UOXCl>5b|6WVh;5vVnR<%d+Lnp z$uEmML38}U4vaW8>shm6CzB(Wei3s#NAWE3)a2)z@i{4jTn;;aQS)O@l{rUM`J@K& l00vQ5JBs~;vo!vr%%-k{2_Fq1Mn4QF81S)AQ99zk{{c4yR+0b! literal 63721 zcmb5Wb9gP!wgnp7wrv|bwr$&XvSZt}Z6`anZSUAlc9NHKf9JdJ;NJVr`=eI(_pMp0 zy1VAAG3FfAOI`{X1O)&90s;U4K;XLp008~hCjbEC_fbYfS%6kTR+JtXK>nW$ZR+`W ze|#J8f4A@M|F5BpfUJb5h>|j$jOe}0oE!`Zf6fM>CR?!y@zU(cL8NsKk`a z6tx5mAkdjD;J=LcJ;;Aw8p!v#ouk>mUDZF@ zK>yvw%+bKu+T{Nk@LZ;zkYy0HBKw06_IWcMHo*0HKpTsEFZhn5qCHH9j z)|XpN&{`!0a>Vl+PmdQc)Yg4A(AG-z!+@Q#eHr&g<9D?7E)_aEB?s_rx>UE9TUq|? z;(ggJt>9l?C|zoO@5)tu?EV0x_7T17q4fF-q3{yZ^ipUbKcRZ4Qftd!xO(#UGhb2y>?*@{xq%`(-`2T^vc=#< zx!+@4pRdk&*1ht2OWk^Z5IAQ0YTAXLkL{(D*$gENaD)7A%^XXrCchN&z2x+*>o2FwPFjWpeaL=!tzv#JOW#( z$B)Nel<+$bkH1KZv3&-}=SiG~w2sbDbAWarg%5>YbC|}*d9hBjBkR(@tyM0T)FO$# zPtRXukGPnOd)~z=?avu+4Co@wF}1T)-uh5jI<1$HLtyDrVak{gw`mcH@Q-@wg{v^c zRzu}hMKFHV<8w}o*yg6p@Sq%=gkd~;`_VGTS?L@yVu`xuGy+dH6YOwcP6ZE`_0rK% zAx5!FjDuss`FQ3eF|mhrWkjux(Pny^k$u_)dyCSEbAsecHsq#8B3n3kDU(zW5yE|( zgc>sFQywFj5}U*qtF9Y(bi*;>B7WJykcAXF86@)z|0-Vm@jt!EPoLA6>r)?@DIobIZ5Sx zsc@OC{b|3%vaMbyeM|O^UxEYlEMHK4r)V-{r)_yz`w1*xV0|lh-LQOP`OP`Pk1aW( z8DSlGN>Ts|n*xj+%If~+E_BxK)~5T#w6Q1WEKt{!Xtbd`J;`2a>8boRo;7u2M&iOop4qcy<)z023=oghSFV zST;?S;ye+dRQe>ygiJ6HCv4;~3DHtJ({fWeE~$H@mKn@Oh6Z(_sO>01JwH5oA4nvK zr5Sr^g+LC zLt(i&ecdmqsIJGNOSUyUpglvhhrY8lGkzO=0USEKNL%8zHshS>Qziu|`eyWP^5xL4 zRP122_dCJl>hZc~?58w~>`P_s18VoU|7(|Eit0-lZRgLTZKNq5{k zE?V=`7=R&ro(X%LTS*f+#H-mGo_j3dm@F_krAYegDLk6UV{`UKE;{YSsn$ z(yz{v1@p|p!0>g04!eRSrSVb>MQYPr8_MA|MpoGzqyd*$@4j|)cD_%^Hrd>SorF>@ zBX+V<@vEB5PRLGR(uP9&U&5=(HVc?6B58NJT_igiAH*q~Wb`dDZpJSKfy5#Aag4IX zj~uv74EQ_Q_1qaXWI!7Vf@ZrdUhZFE;L&P_Xr8l@GMkhc#=plV0+g(ki>+7fO%?Jb zl+bTy7q{w^pTb{>(Xf2q1BVdq?#f=!geqssXp z4pMu*q;iiHmA*IjOj4`4S&|8@gSw*^{|PT}Aw~}ZXU`6=vZB=GGeMm}V6W46|pU&58~P+?LUs%n@J}CSrICkeng6YJ^M? zS(W?K4nOtoBe4tvBXs@@`i?4G$S2W&;$z8VBSM;Mn9 zxcaEiQ9=vS|bIJ>*tf9AH~m&U%2+Dim<)E=}KORp+cZ^!@wI`h1NVBXu{@%hB2Cq(dXx_aQ9x3mr*fwL5!ZryQqi|KFJuzvP zK1)nrKZ7U+B{1ZmJub?4)Ln^J6k!i0t~VO#=q1{?T)%OV?MN}k5M{}vjyZu#M0_*u z8jwZKJ#Df~1jcLXZL7bnCEhB6IzQZ-GcoQJ!16I*39iazoVGugcKA{lhiHg4Ta2fD zk1Utyc5%QzZ$s3;p0N+N8VX{sd!~l*Ta3|t>lhI&G`sr6L~G5Lul`>m z{!^INm?J|&7X=;{XveF!(b*=?9NAp4y&r&N3(GKcW4rS(Ejk|Lzs1PrxPI_owB-`H zg3(Rruh^&)`TKA6+_!n>RdI6pw>Vt1_j&+bKIaMTYLiqhZ#y_=J8`TK{Jd<7l9&sY z^^`hmi7^14s16B6)1O;vJWOF$=$B5ONW;;2&|pUvJlmeUS&F;DbSHCrEb0QBDR|my zIs+pE0Y^`qJTyH-_mP=)Y+u^LHcuZhsM3+P||?+W#V!_6E-8boP#R-*na4!o-Q1 zVthtYhK{mDhF(&7Okzo9dTi03X(AE{8cH$JIg%MEQca`S zy@8{Fjft~~BdzWC(di#X{ny;!yYGK9b@=b|zcKZ{vv4D8i+`ilOPl;PJl{!&5-0!w z^fOl#|}vVg%=n)@_e1BrP)`A zKPgs`O0EO}Y2KWLuo`iGaKu1k#YR6BMySxQf2V++Wo{6EHmK>A~Q5o73yM z-RbxC7Qdh0Cz!nG+7BRZE>~FLI-?&W_rJUl-8FDIaXoNBL)@1hwKa^wOr1($*5h~T zF;%f^%<$p8Y_yu(JEg=c_O!aZ#)Gjh$n(hfJAp$C2he555W5zdrBqjFmo|VY+el;o z=*D_w|GXG|p0**hQ7~9-n|y5k%B}TAF0iarDM!q-jYbR^us(>&y;n^2l0C%@2B}KM zyeRT9)oMt97Agvc4sEKUEy%MpXr2vz*lb zh*L}}iG>-pqDRw7ud{=FvTD?}xjD)w{`KzjNom-$jS^;iw0+7nXSnt1R@G|VqoRhE%12nm+PH?9`(4rM0kfrZzIK9JU=^$YNyLvAIoxl#Q)xxDz!^0@zZ zSCs$nfcxK_vRYM34O<1}QHZ|hp4`ioX3x8(UV(FU$J@o%tw3t4k1QPmlEpZa2IujG&(roX_q*%e`Hq|);0;@k z0z=fZiFckp#JzW0p+2A+D$PC~IsakhJJkG(c;CqAgFfU0Z`u$PzG~-9I1oPHrCw&)@s^Dc~^)#HPW0Ra}J^=|h7Fs*<8|b13ZzG6MP*Q1dkoZ6&A^!}|hbjM{2HpqlSXv_UUg1U4gn z3Q)2VjU^ti1myodv+tjhSZp%D978m~p& z43uZUrraHs80Mq&vcetqfQpQP?m!CFj)44t8Z}k`E798wxg&~aCm+DBoI+nKq}&j^ zlPY3W$)K;KtEajks1`G?-@me7C>{PiiBu+41#yU_c(dITaqE?IQ(DBu+c^Ux!>pCj zLC|HJGU*v+!it1(;3e`6igkH(VA)-S+k(*yqxMgUah3$@C zz`7hEM47xr>j8^g`%*f=6S5n>z%Bt_Fg{Tvmr+MIsCx=0gsu_sF`q2hlkEmisz#Fy zj_0;zUWr;Gz}$BS%Y`meb(=$d%@Crs(OoJ|}m#<7=-A~PQbyN$x%2iXP2@e*nO0b7AwfH8cCUa*Wfu@b)D_>I*%uE4O3 z(lfnB`-Xf*LfC)E}e?%X2kK7DItK6Tf<+M^mX0Ijf_!IP>7c8IZX%8_#0060P{QMuV^B9i<^E`_Qf0pv9(P%_s8D`qvDE9LK9u-jB}J2S`(mCO&XHTS04Z5Ez*vl^T%!^$~EH8M-UdwhegL>3IQ*)(MtuH2Xt1p!fS4o~*rR?WLxlA!sjc2(O znjJn~wQ!Fp9s2e^IWP1C<4%sFF}T4omr}7+4asciyo3DntTgWIzhQpQirM$9{EbQd z3jz9vS@{aOqTQHI|l#aUV@2Q^Wko4T0T04Me4!2nsdrA8QY1%fnAYb~d2GDz@lAtfcHq(P7 zaMBAGo}+NcE-K*@9y;Vt3*(aCaMKXBB*BJcD_Qnxpt75r?GeAQ}*|>pYJE=uZb73 zC>sv)18)q#EGrTG6io*}JLuB_jP3AU1Uiu$D7r|2_zlIGb9 zjhst#ni)Y`$)!fc#reM*$~iaYoz~_Cy7J3ZTiPm)E?%`fbk`3Tu-F#`{i!l5pNEn5 zO-Tw-=TojYhzT{J=?SZj=Z8#|eoF>434b-DXiUsignxXNaR3 zm_}4iWU$gt2Mw5NvZ5(VpF`?X*f2UZDs1TEa1oZCif?Jdgr{>O~7}-$|BZ7I(IKW`{f;@|IZFX*R8&iT= zoWstN8&R;}@2Ka%d3vrLtR|O??ben;k8QbS-WB0VgiCz;<$pBmIZdN!aalyCSEm)crpS9dcD^Y@XT1a3+zpi-`D}e#HV<} z$Y(G&o~PvL-xSVD5D?JqF3?B9rxGWeb=oEGJ3vRp5xfBPlngh1O$yI95EL+T8{GC@ z98i1H9KhZGFl|;`)_=QpM6H?eDPpw~^(aFQWwyXZ8_EEE4#@QeT_URray*mEOGsGc z6|sdXtq!hVZo=d#+9^@lm&L5|q&-GDCyUx#YQiccq;spOBe3V+VKdjJA=IL=Zn%P} zNk=_8u}VhzFf{UYZV0`lUwcD&)9AFx0@Fc6LD9A6Rd1=ga>Mi0)_QxM2ddCVRmZ0d z+J=uXc(?5JLX3=)e)Jm$HS2yF`44IKhwRnm2*669_J=2LlwuF5$1tAo@ROSU@-y+;Foy2IEl2^V1N;fk~YR z?&EP8#t&m0B=?aJeuz~lHjAzRBX>&x=A;gIvb>MD{XEV zV%l-+9N-)i;YH%nKP?>f`=?#`>B(`*t`aiPLoQM(a6(qs4p5KFjDBN?8JGrf3z8>= zi7sD)c)Nm~x{e<^jy4nTx${P~cwz_*a>%0_;ULou3kHCAD7EYkw@l$8TN#LO9jC( z1BeFW`k+bu5e8Ns^a8dPcjEVHM;r6UX+cN=Uy7HU)j-myRU0wHd$A1fNI~`4;I~`zC)3ul#8#^rXVSO*m}Ag>c%_;nj=Nv$rCZ z*~L@C@OZg%Q^m)lc-kcX&a*a5`y&DaRxh6O*dfhLfF+fU5wKs(1v*!TkZidw*)YBP za@r`3+^IHRFeO%!ai%rxy;R;;V^Fr=OJlpBX;(b*3+SIw}7= zIq$*Thr(Zft-RlY)D3e8V;BmD&HOfX+E$H#Y@B3?UL5L~_fA-@*IB-!gItK7PIgG9 zgWuGZK_nuZjHVT_Fv(XxtU%)58;W39vzTI2n&)&4Dmq7&JX6G>XFaAR{7_3QB6zsT z?$L8c*WdN~nZGiscY%5KljQARN;`w$gho=p006z;n(qIQ*Zu<``TMO3n0{ARL@gYh zoRwS*|Niw~cR!?hE{m*y@F`1)vx-JRfqET=dJ5_(076st(=lFfjtKHoYg`k3oNmo_ zNbQEw8&sO5jAYmkD|Zaz_yUb0rC})U!rCHOl}JhbYIDLzLvrZVw0~JO`d*6f;X&?V=#T@ND*cv^I;`sFeq4 z##H5;gpZTb^0Hz@3C*~u0AqqNZ-r%rN3KD~%Gw`0XsIq$(^MEb<~H(2*5G^<2(*aI z%7}WB+TRlMIrEK#s0 z93xn*Ohb=kWFc)BNHG4I(~RPn-R8#0lqyBBz5OM6o5|>x9LK@%HaM}}Y5goCQRt2C z{j*2TtT4ne!Z}vh89mjwiSXG=%DURar~=kGNNaO_+Nkb+tRi~Rkf!7a$*QlavziD( z83s4GmQ^Wf*0Bd04f#0HX@ua_d8 z23~z*53ePD6@xwZ(vdl0DLc=>cPIOPOdca&MyR^jhhKrdQO?_jJh`xV3GKz&2lvP8 zEOwW6L*ufvK;TN{=S&R@pzV^U=QNk^Ec}5H z+2~JvEVA{`uMAr)?Kf|aW>33`)UL@bnfIUQc~L;TsTQ6>r-<^rB8uoNOJ>HWgqMI8 zSW}pZmp_;z_2O5_RD|fGyTxaxk53Hg_3Khc<8AUzV|ZeK{fp|Ne933=1&_^Dbv5^u zB9n=*)k*tjHDRJ@$bp9mrh}qFn*s}npMl5BMDC%Hs0M0g-hW~P*3CNG06G!MOPEQ_ zi}Qs-6M8aMt;sL$vlmVBR^+Ry<64jrm1EI1%#j?c?4b*7>)a{aDw#TfTYKq+SjEFA z(aJ&z_0?0JB83D-i3Vh+o|XV4UP+YJ$9Boid2^M2en@APw&wx7vU~t$r2V`F|7Qfo z>WKgI@eNBZ-+Og<{u2ZiG%>YvH2L3fNpV9J;WLJoBZda)01Rn;o@){01{7E#ke(7U zHK>S#qZ(N=aoae*4X!0A{)nu0R_sKpi1{)u>GVjC+b5Jyl6#AoQ-1_3UDovNSo`T> z?c-@7XX*2GMy?k?{g)7?Sv;SJkmxYPJPs!&QqB12ejq`Lee^-cDveVWL^CTUldb(G zjDGe(O4P=S{4fF=#~oAu>LG>wrU^z_?3yt24FOx>}{^lCGh8?vtvY$^hbZ)9I0E3r3NOlb9I?F-Yc=r$*~l`4N^xzlV~N zl~#oc>U)Yjl0BxV>O*Kr@lKT{Z09OXt2GlvE38nfs+DD7exl|&vT;)>VFXJVZp9Np zDK}aO;R3~ag$X*|hRVY3OPax|PG`@_ESc8E!mHRByJbZQRS38V2F__7MW~sgh!a>98Q2%lUNFO=^xU52|?D=IK#QjwBky-C>zOWlsiiM&1n z;!&1((Xn1$9K}xabq~222gYvx3hnZPg}VMF_GV~5ocE=-v>V=T&RsLBo&`)DOyIj* zLV{h)JU_y*7SdRtDajP_Y+rBkNN*1_TXiKwHH2&p51d(#zv~s#HwbNy?<+(=9WBvo zw2hkk2Dj%kTFhY+$T+W-b7@qD!bkfN#Z2ng@Pd=i3-i?xYfs5Z*1hO?kd7Sp^9`;Y zM2jeGg<-nJD1er@Pc_cSY7wo5dzQX44=%6rn}P_SRbpzsA{6B+!$3B0#;}qwO37G^ zL(V_5JK`XT?OHVk|{_$vQ|oNEpab*BO4F zUTNQ7RUhnRsU`TK#~`)$icsvKh~(pl=3p6m98@k3P#~upd=k*u20SNcb{l^1rUa)>qO997)pYRWMncC8A&&MHlbW?7i^7M`+B$hH~Y|J zd>FYOGQ;j>Zc2e7R{KK7)0>>nn_jYJy&o@sK!4G>-rLKM8Hv)f;hi1D2fAc$+six2 zyVZ@wZ6x|fJ!4KrpCJY=!Mq0;)X)OoS~{Lkh6u8J`eK%u0WtKh6B>GW_)PVc zl}-k`p09qwGtZ@VbYJC!>29V?Dr>>vk?)o(x?!z*9DJ||9qG-&G~#kXxbw{KKYy}J zQKa-dPt~M~E}V?PhW0R26xdA%1T*%ra6SguGu50YHngOTIv)@N|YttEXo#OZfgtP7;H?EeZZxo<}3YlYxtBq znJ!WFR^tmGf0Py}N?kZ(#=VtpC@%xJkDmfcCoBTxq zr_|5gP?u1@vJZbxPZ|G0AW4=tpb84gM2DpJU||(b8kMOV1S3|(yuwZJ&rIiFW(U;5 zUtAW`O6F6Zy+eZ1EDuP~AAHlSY-+A_eI5Gx)%*uro5tljy}kCZU*_d7)oJ>oQSZ3* zneTn`{gnNC&uJd)0aMBzAg021?YJ~b(fmkwZAd696a=0NzBAqBN54KuNDwa*no(^O z6p05bioXUR^uXjpTol*ppHp%1v9e)vkoUAUJyBx3lw0UO39b0?^{}yb!$yca(@DUn zCquRF?t=Zb9`Ed3AI6|L{eX~ijVH`VzSMheKoP7LSSf4g>md>`yi!TkoG5P>Ofp+n z(v~rW+(5L96L{vBb^g51B=(o)?%%xhvT*A5btOpw(TKh^g^4c zw>0%X!_0`{iN%RbVk+A^f{w-4-SSf*fu@FhruNL##F~sF24O~u zyYF<3el2b$$wZ_|uW#@Ak+VAGk#e|kS8nL1g>2B-SNMjMp^8;-FfeofY2fphFHO!{ z*!o4oTb{4e;S<|JEs<1_hPsmAlVNk?_5-Fp5KKU&d#FiNW~Y+pVFk@Cua1I{T+1|+ zHx6rFMor)7L)krbilqsWwy@T+g3DiH5MyVf8Wy}XbEaoFIDr~y;@r&I>FMW{ z?Q+(IgyebZ)-i4jNoXQhq4Muy9Fv+OxU;9_Jmn+<`mEC#%2Q_2bpcgzcinygNI!&^ z=V$)o2&Yz04~+&pPWWn`rrWxJ&}8khR)6B(--!9Q zubo}h+1T)>a@c)H^i``@<^j?|r4*{;tQf78(xn0g39IoZw0(CwY1f<%F>kEaJ zp9u|IeMY5mRdAlw*+gSN^5$Q)ShM<~E=(c8QM+T-Qk)FyKz#Sw0EJ*edYcuOtO#~Cx^(M7w5 z3)rl#L)rF|(Vun2LkFr!rg8Q@=r>9p>(t3Gf_auiJ2Xx9HmxYTa|=MH_SUlYL`mz9 zTTS$`%;D-|Jt}AP1&k7PcnfFNTH0A-*FmxstjBDiZX?}%u%Yq94$fUT&z6od+(Uk> zuqsld#G(b$G8tus=M!N#oPd|PVFX)?M?tCD0tS%2IGTfh}3YA3f&UM)W$_GNV8 zQo+a(ml2Km4o6O%gKTCSDNq+#zCTIQ1*`TIJh~k6Gp;htHBFnne))rlFdGqwC6dx2+La1&Mnko*352k0y z+tQcwndQlX`nc6nb$A9?<-o|r*%aWXV#=6PQic0Ok_D;q>wbv&j7cKc!w4~KF#-{6 z(S%6Za)WpGIWf7jZ3svNG5OLs0>vCL9{V7cgO%zevIVMH{WgP*^D9ws&OqA{yr|m| zKD4*07dGXshJHd#e%x%J+qmS^lS|0Bp?{drv;{@{l9ArPO&?Q5=?OO9=}h$oVe#3b z3Yofj&Cb}WC$PxmRRS)H%&$1-)z7jELS}!u!zQ?A^Y{Tv4QVt*vd@uj-^t2fYRzQj zfxGR>-q|o$3sGn^#VzZ!QQx?h9`njeJry}@x?|k0-GTTA4y3t2E`3DZ!A~D?GiJup z)8%PK2^9OVRlP(24P^4_<|D=H^7}WlWu#LgsdHzB%cPy|f8dD3|A^mh4WXxhLTVu_ z@abE{6Saz|Y{rXYPd4$tfPYo}ef(oQWZ=4Bct-=_9`#Qgp4ma$n$`tOwq#&E18$B; z@Bp)bn3&rEi0>fWWZ@7k5WazfoX`SCO4jQWwVuo+$PmSZn^Hz?O(-tW@*DGxuf)V1 zO_xm&;NVCaHD4dqt(-MlszI3F-p?0!-e$fbiCeuaw66h^TTDLWuaV<@C-`=Xe5WL) zwooG7h>4&*)p3pKMS3O!4>-4jQUN}iAMQ)2*70?hP~)TzzR?-f@?Aqy$$1Iy8VGG$ zMM?8;j!pUX7QQD$gRc_#+=raAS577ga-w?jd`vCiN5lu)dEUkkUPl9!?{$IJNxQys z*E4e$eF&n&+AMRQR2gcaFEjAy*r)G!s(P6D&TfoApMFC_*Ftx0|D0@E-=B7tezU@d zZ{hGiN;YLIoSeRS;9o%dEua4b%4R3;$SugDjP$x;Z!M!@QibuSBb)HY!3zJ7M;^jw zlx6AD50FD&p3JyP*>o+t9YWW8(7P2t!VQQ21pHJOcG_SXQD;(5aX#M6x##5H_Re>6lPyDCjxr*R(+HE%c&QN+b^tbT zXBJk?p)zhJj#I?&Y2n&~XiytG9!1ox;bw5Rbj~)7c(MFBb4>IiRATdhg zmiEFlj@S_hwYYI(ki{}&<;_7(Z0Qkfq>am z&LtL=2qc7rWguk3BtE4zL41@#S;NN*-jWw|7Kx7H7~_%7fPt;TIX}Ubo>;Rmj94V> zNB1=;-9AR7s`Pxn}t_6^3ahlq53e&!Lh85uG zec0vJY_6e`tg7LgfrJ3k!DjR)Bi#L@DHIrZ`sK=<5O0Ip!fxGf*OgGSpP@Hbbe&$9 z;ZI}8lEoC2_7;%L2=w?tb%1oL0V+=Z`7b=P&lNGY;yVBazXRYu;+cQDKvm*7NCxu&i;zub zAJh#11%?w>E2rf2e~C4+rAb-&$^vsdACs7 z@|Ra!OfVM(ke{vyiqh7puf&Yp6cd6{DptUteYfIRWG3pI+5< zBVBI_xkBAc<(pcb$!Y%dTW(b;B;2pOI-(QCsLv@U-D1XJ z(Gk8Q3l7Ws46Aktuj>|s{$6zA&xCPuXL-kB`CgYMs}4IeyG*P51IDwW?8UNQd+$i~ zlxOPtSi5L|gJcF@DwmJA5Ju8HEJ>o{{upwIpb!f{2(vLNBw`7xMbvcw<^{Fj@E~1( z?w`iIMieunS#>nXlmUcSMU+D3rX28f?s7z;X=se6bo8;5vM|O^(D6{A9*ChnGH!RG zP##3>LDC3jZPE4PH32AxrqPk|yIIrq~`aL-=}`okhNu9aT%q z1b)7iJ)CN=V#Ly84N_r7U^SH2FGdE5FpTO2 z630TF$P>GNMu8`rOytb(lB2};`;P4YNwW1<5d3Q~AX#P0aX}R2b2)`rgkp#zTxcGj zAV^cvFbhP|JgWrq_e`~exr~sIR$6p5V?o4Wym3kQ3HA+;Pr$bQ0(PmADVO%MKL!^q z?zAM8j1l4jrq|5X+V!8S*2Wl@=7*pPgciTVK6kS1Ge zMsd_u6DFK$jTnvVtE;qa+8(1sGBu~n&F%dh(&c(Zs4Fc#A=gG^^%^AyH}1^?|8quj zl@Z47h$){PlELJgYZCIHHL= z{U8O>Tw4x3<1{?$8>k-P<}1y9DmAZP_;(3Y*{Sk^H^A=_iSJ@+s5ktgwTXz_2$~W9>VVZsfwCm@s0sQ zeB50_yu@uS+e7QoPvdCwDz{prjo(AFwR%C?z`EL{1`|coJHQTk^nX=tvs1<0arUOJ z!^`*x&&BvTYmemyZ)2p~{%eYX=JVR?DYr(rNgqRMA5E1PR1Iw=prk=L2ldy3r3Vg@27IZx43+ywyzr-X*p*d@tZV+!U#~$-q=8c zgdSuh#r?b4GhEGNai)ayHQpk>5(%j5c@C1K3(W1pb~HeHpaqijJZa-e6vq_8t-^M^ zBJxq|MqZc?pjXPIH}70a5vt!IUh;l}<>VX<-Qcv^u@5(@@M2CHSe_hD$VG-eiV^V( zj7*9T0?di?P$FaD6oo?)<)QT>Npf6Og!GO^GmPV(Km0!=+dE&bk#SNI+C9RGQ|{~O*VC+tXK3!n`5 zHfl6>lwf_aEVV3`0T!aHNZLsj$paS$=LL(?b!Czaa5bbSuZ6#$_@LK<(7yrrl+80| z{tOFd=|ta2Z`^ssozD9BINn45NxUeCQis?-BKmU*Kt=FY-NJ+)8S1ecuFtN-M?&42 zl2$G>u!iNhAk*HoJ^4v^9#ORYp5t^wDj6|lx~5w45#E5wVqI1JQ~9l?nPp1YINf++ zMAdSif~_ETv@Er(EFBI^@L4BULFW>)NI+ejHFP*T}UhWNN`I)RRS8za? z*@`1>9ZB}An%aT5K=_2iQmfE;GcBVHLF!$`I99o5GO`O%O_zLr9AG18>&^HkG(;=V z%}c!OBQ~?MX(9h~tajX{=x)+!cbM7$YzTlmsPOdp2L-?GoW`@{lY9U3f;OUo*BwRB z8A+nv(br0-SH#VxGy#ZrgnGD(=@;HME;yd46EgWJ`EL%oXc&lFpc@Y}^>G(W>h_v_ zlN!`idhX+OjL+~T?19sroAFVGfa5tX-D49w$1g2g_-T|EpHL6}K_aX4$K=LTvwtlF zL*z}j{f+Uoe7{-px3_5iKPA<_7W=>Izkk)!l9ez2w%vi(?Y;i8AxRNLSOGDzNoqoI zP!1uAl}r=_871(G?y`i&)-7{u=%nxk7CZ_Qh#!|ITec zwQn`33GTUM`;D2POWnkqngqJhJRlM>CTONzTG}>^Q0wUunQyn|TAiHzyX2_%ATx%P z%7gW)%4rA9^)M<_%k@`Y?RbC<29sWU&5;@|9thf2#zf8z12$hRcZ!CSb>kUp=4N#y zl3hE#y6>kkA8VY2`W`g5Ip?2qC_BY$>R`iGQLhz2-S>x(RuWv)SPaGdl^)gGw7tjR zH@;jwk!jIaCgSg_*9iF|a);sRUTq30(8I(obh^|}S~}P4U^BIGYqcz;MPpC~Y@k_m zaw4WG1_vz2GdCAX!$_a%GHK**@IrHSkGoN>)e}>yzUTm52on`hYot7cB=oA-h1u|R ztH$11t?54Qg2L+i33FPFKKRm1aOjKST{l1*(nps`>sv%VqeVMWjl5+Gh+9);hIP8? zA@$?}Sc z3qIRpba+y5yf{R6G(u8Z^vkg0Fu&D-7?1s=QZU`Ub{-!Y`I?AGf1VNuc^L3v>)>i# z{DV9W$)>34wnzAXUiV^ZpYKw>UElrN_5Xj6{r_3| z$X5PK`e5$7>~9Dj7gK5ash(dvs`vwfk}&RD`>04;j62zoXESkFBklYaKm5seyiX(P zqQ-;XxlV*yg?Dhlx%xt!b0N3GHp@(p$A;8|%# zZ5m2KL|{on4nr>2_s9Yh=r5ScQ0;aMF)G$-9-Ca6%wA`Pa)i?NGFA|#Yi?{X-4ZO_ z^}%7%vkzvUHa$-^Y#aA+aiR5sa%S|Ebyn`EV<3Pc?ax_f>@sBZF1S;7y$CXd5t5=WGsTKBk8$OfH4v|0?0I=Yp}7c=WBSCg!{0n)XmiU;lfx)**zZaYqmDJelxk$)nZyx5`x$6R|fz(;u zEje5Dtm|a%zK!!tk3{i9$I2b{vXNFy%Bf{50X!x{98+BsDr_u9i>G5%*sqEX|06J0 z^IY{UcEbj6LDwuMh7cH`H@9sVt1l1#8kEQ(LyT@&+K}(ReE`ux8gb0r6L_#bDUo^P z3Ka2lRo52Hdtl_%+pwVs14=q`{d^L58PsU@AMf(hENumaxM{7iAT5sYmWh@hQCO^ zK&}ijo=`VqZ#a3vE?`7QW0ZREL17ZvDfdqKGD?0D4fg{7v%|Yj&_jcKJAB)>=*RS* zto8p6@k%;&^ZF>hvXm&$PCuEp{uqw3VPG$9VMdW5$w-fy2CNNT>E;>ejBgy-m_6`& z97L1p{%srn@O_JQgFpa_#f(_)eb#YS>o>q3(*uB;uZb605(iqM$=NK{nHY=+X2*G) zO3-_Xh%aG}fHWe*==58zBwp%&`mge<8uq8;xIxOd=P%9EK!34^E9sk|(Zq1QSz-JVeP12Fp)-`F|KY$LPwUE?rku zY@OJ)Z9A!ojfzfeyJ9;zv2EM7ZQB)AR5xGa-tMn^bl)FmoIiVyJ@!~@%{}qXXD&Ns zPnfe5U+&ohKefILu_1mPfLGuapX@btta5C#gPB2cjk5m4T}Nfi+Vfka!Yd(L?-c~5 z#ZK4VeQEXNPc4r$K00Fg>g#_W!YZ)cJ?JTS<&68_$#cZT-ME`}tcwqg3#``3M3UPvn+pi}(VNNx6y zFIMVb6OwYU(2`at$gHba*qrMVUl8xk5z-z~fb@Q3Y_+aXuEKH}L+>eW__!IAd@V}L zkw#s%H0v2k5-=vh$^vPCuAi22Luu3uKTf6fPo?*nvj$9(u)4$6tvF-%IM+3pt*cgs z_?wW}J7VAA{_~!?))?s6{M=KPpVhg4fNuU*|3THp@_(q!b*hdl{fjRVFWtu^1dV(f z6iOux9hi&+UK=|%M*~|aqFK{Urfl!TA}UWY#`w(0P!KMe1Si{8|o))Gy6d7;!JQYhgMYmXl?3FfOM2nQGN@~Ap6(G z3+d_5y@=nkpKAhRqf{qQ~k7Z$v&l&@m7Ppt#FSNzKPZM z8LhihcE6i=<(#87E|Wr~HKvVWhkll4iSK$^mUHaxgy8*K$_Zj;zJ`L$naPj+^3zTi z-3NTaaKnD5FPY-~?Tq6QHnmDDRxu0mh0D|zD~Y=vv_qig5r-cIbCpxlju&8Sya)@{ zsmv6XUSi)@(?PvItkiZEeN*)AE~I_?#+Ja-r8$(XiXei2d@Hi7Rx8+rZZb?ZLa{;@*EHeRQ-YDadz~M*YCM4&F-r;E#M+@CSJMJ0oU|PQ^ z=E!HBJDMQ2TN*Y(Ag(ynAL8%^v;=~q?s4plA_hig&5Z0x_^Oab!T)@6kRN$)qEJ6E zNuQjg|G7iwU(N8pI@_6==0CL;lRh1dQF#wePhmu@hADFd3B5KIH#dx(2A zp~K&;Xw}F_N6CU~0)QpQk7s$a+LcTOj1%=WXI(U=Dv!6 z{#<#-)2+gCyyv=Jw?Ab#PVkxPDeH|sAxyG`|Ys}A$PW4TdBv%zDz z^?lwrxWR<%Vzc8Sgt|?FL6ej_*e&rhqJZ3Y>k=X(^dytycR;XDU16}Pc9Vn0>_@H+ zQ;a`GSMEG64=JRAOg%~L)x*w{2re6DVprNp+FcNra4VdNjiaF0M^*>CdPkt(m150rCue?FVdL0nFL$V%5y6N z%eLr5%YN7D06k5ji5*p4v$UMM)G??Q%RB27IvH7vYr_^3>1D-M66#MN8tWGw>WED} z5AhlsanO=STFYFs)Il_0i)l)f<8qn|$DW7ZXhf5xI;m+7M5-%P63XFQrG9>DMqHc} zsgNU9nR`b}E^mL5=@7<1_R~j@q_2U^3h|+`7YH-?C=vme1C3m`Fe0HC>pjt6f_XMh zy~-i-8R46QNYneL4t@)<0VU7({aUO?aH`z4V2+kxgH5pYD5)wCh75JqQY)jIPN=U6 z+qi8cGiOtXG2tXm;_CfpH9ESCz#i5B(42}rBJJF$jh<1sbpj^8&L;gzGHb8M{of+} zzF^8VgML2O9nxBW7AvdEt90vp+#kZxWf@A)o9f9}vKJy9NDBjBW zSt=Hcs=YWCwnfY1UYx*+msp{g!w0HC<_SM!VL1(I2PE?CS}r(eh?{I)mQixmo5^p# zV?2R!R@3GV6hwTCrfHiK#3Orj>I!GS2kYhk1S;aFBD_}u2v;0HYFq}Iz1Z(I4oca4 zxquja8$+8JW_EagDHf$a1OTk5S97umGSDaj)gH=fLs9>_=XvVj^Xj9a#gLdk=&3tl zfmK9MNnIX9v{?%xdw7568 zNrZ|roYs(vC4pHB5RJ8>)^*OuyNC>x7ad)tB_}3SgQ96+-JT^Qi<`xi=)_=$Skwv~ zdqeT9Pa`LYvCAn&rMa2aCDV(TMI#PA5g#RtV|CWpgDYRA^|55LLN^uNh*gOU>Z=a06qJ;$C9z8;n-Pq=qZnc1zUwJ@t)L;&NN+E5m zRkQ(SeM8=l-aoAKGKD>!@?mWTW&~)uF2PYUJ;tB^my`r9n|Ly~0c%diYzqs9W#FTjy?h&X3TnH zXqA{QI82sdjPO->f=^K^f>N`+B`q9&rN0bOXO79S&a9XX8zund(kW7O76f4dcWhIu zER`XSMSFbSL>b;Rp#`CuGJ&p$s~G|76){d?xSA5wVg##_O0DrmyEYppyBr%fyWbbv zp`K84JwRNP$d-pJ!Qk|(RMr?*!wi1if-9G#0p>>1QXKXWFy)eB3ai)l3601q8!9JC zvU#ZWWDNKq9g6fYs?JQ)Q4C_cgTy3FhgKb8s&m)DdmL5zhNK#8wWg!J*7G7Qhe9VU zha?^AQTDpYcuN!B+#1dE*X{<#!M%zfUQbj=zLE{dW0XeQ7-oIsGY6RbkP2re@Q{}r_$iiH0xU%iN*ST`A)-EH6eaZB$GA#v)cLi z*MpA(3bYk$oBDKAzu^kJoSUsDd|856DApz={3u8sbQV@JnRkp2nC|)m;#T=DvIL-O zI4vh;g7824l}*`_p@MT4+d`JZ2%6NQh=N9bmgJ#q!hK@_<`HQq3}Z8Ij>3%~<*= zcv=!oT#5xmeGI92lqm9sGVE%#X$ls;St|F#u!?5Y7syhx6q#MVRa&lBmmn%$C0QzU z);*ldgwwCmzM3uglr}!Z2G+?& zf%Dpo&mD%2ZcNFiN-Z0f;c_Q;A%f@>26f?{d1kxIJD}LxsQkB47SAdwinfMILZdN3 zfj^HmTzS3Ku5BxY>ANutS8WPQ-G>v4^_Qndy==P3pDm+Xc?>rUHl-4+^%Sp5atOja z2oP}ftw-rqnb}+khR3CrRg^ibi6?QYk1*i^;kQGirQ=uB9Sd1NTfT-Rbv;hqnY4neE5H1YUrjS2m+2&@uXiAo- zrKUX|Ohg7(6F(AoP~tj;NZlV#xsfo-5reuQHB$&EIAhyZk;bL;k9ouDmJNBAun;H& zn;Of1z_Qj`x&M;5X;{s~iGzBQTY^kv-k{ksbE*Dl%Qf%N@hQCfY~iUw!=F-*$cpf2 z3wix|aLBV0b;W@z^%7S{>9Z^T^fLOI68_;l@+Qzaxo`nAI8emTV@rRhEKZ z?*z_{oGdI~R*#<2{bkz$G~^Qef}$*4OYTgtL$e9q!FY7EqxJ2`zk6SQc}M(k(_MaV zSLJnTXw&@djco1~a(vhBl^&w=$fa9{Sru>7g8SHahv$&Bl(D@(Zwxo_3r=;VH|uc5 zi1Ny)J!<(KN-EcQ(xlw%PNwK8U>4$9nVOhj(y0l9X^vP1TA>r_7WtSExIOsz`nDOP zs}d>Vxb2Vo2e5x8p(n~Y5ggAyvib>d)6?)|E@{FIz?G3PVGLf7-;BxaP;c?7ddH$z zA+{~k^V=bZuXafOv!RPsE1GrR3J2TH9uB=Z67gok+u`V#}BR86hB1xl}H4v`F+mRfr zYhortD%@IGfh!JB(NUNSDh+qDz?4ztEgCz&bIG-Wg7w-ua4ChgQR_c+z8dT3<1?uX z*G(DKy_LTl*Ea!%v!RhpCXW1WJO6F`bgS-SB;Xw9#! z<*K}=#wVu9$`Yo|e!z-CPYH!nj7s9dEPr-E`DXUBu0n!xX~&|%#G=BeM?X@shQQMf zMvr2!y7p_gD5-!Lnm|a@z8Of^EKboZsTMk%5VsJEm>VsJ4W7Kv{<|#4f-qDE$D-W>gWT%z-!qXnDHhOvLk=?^a1*|0j z{pW{M0{#1VcR5;F!!fIlLVNh_Gj zbnW(_j?0c2q$EHIi@fSMR{OUKBcLr{Y&$hrM8XhPByyZaXy|dd&{hYQRJ9@Fn%h3p7*VQolBIV@Eq`=y%5BU~3RPa^$a?ixp^cCg z+}Q*X+CW9~TL29@OOng(#OAOd!)e$d%sr}^KBJ-?-X&|4HTmtemxmp?cT3uA?md4% zT8yZ0U;6Rg6JHy3fJae{6TMGS?ZUX6+gGTT{Q{)SI85$5FD{g-eR%O0KMpWPY`4@O zx!hen1*8^E(*}{m^V_?}(b5k3hYo=T+$&M32+B`}81~KKZhY;2H{7O-M@vbCzuX0n zW-&HXeyr1%I3$@ns-V1~Lb@wIpkmx|8I~ob1Of7i6BTNysEwI}=!nU%q7(V_^+d*G z7G;07m(CRTJup!`cdYi93r^+LY+`M*>aMuHJm(A8_O8C#A*$!Xvddgpjx5)?_EB*q zgE8o5O>e~9IiSC@WtZpF{4Bj2J5eZ>uUzY%TgWF7wdDE!fSQIAWCP)V{;HsU3ap?4 znRsiiDbtN7i9hapO;(|Ew>Ip2TZSvK9Z^N21%J?OiA_&eP1{(Pu_=%JjKy|HOardq ze?zK^K zA%sjF64*Wufad%H<) z^|t>e*h+Z1#l=5wHexzt9HNDNXgM=-OPWKd^5p!~%SIl>Fo&7BvNpbf8{NXmH)o{r zO=aBJ;meX1^{O%q;kqdw*5k!Y7%t_30 zy{nGRVc&5qt?dBwLs+^Sfp;f`YVMSB#C>z^a9@fpZ!xb|b-JEz1LBX7ci)V@W+kvQ89KWA0T~Lj$aCcfW#nD5bt&Y_< z-q{4ZXDqVg?|0o)j1%l0^_it0WF*LCn-+)c!2y5yS7aZIN$>0LqNnkujV*YVes(v$ zY@_-!Q;!ZyJ}Bg|G-~w@or&u0RO?vlt5*9~yeoPV_UWrO2J54b4#{D(D>jF(R88u2 zo#B^@iF_%S>{iXSol8jpmsZuJ?+;epg>k=$d`?GSegAVp3n$`GVDvK${N*#L_1`44 z{w0fL{2%)0|E+qgZtjX}itZz^KJt4Y;*8uSK}Ft38+3>j|K(PxIXXR-t4VopXo#9# zt|F{LWr-?34y`$nLBVV_*UEgA6AUI65dYIbqpNq9cl&uLJ0~L}<=ESlOm?Y-S@L*d z<7vt}`)TW#f%Rp$Q}6@3=j$7Tze@_uZO@aMn<|si{?S}~maII`VTjs&?}jQ4_cut9$)PEqMukwoXobzaKx^MV z2fQwl+;LSZ$qy%Tys0oo^K=jOw$!YwCv^ei4NBVauL)tN%=wz9M{uf{IB(BxK|lT*pFkmNK_1tV`nb%jH=a0~VNq2RCKY(rG7jz!-D^k)Ec)yS%17pE#o6&eY+ z^qN(hQT$}5F(=4lgNQhlxj?nB4N6ntUY6(?+R#B?W3hY_a*)hnr4PA|vJ<6p`K3Z5Hy z{{8(|ux~NLUW=!?9Qe&WXMTAkQnLXg(g=I@(VG3{HE13OaUT|DljyWXPs2FE@?`iU z4GQlM&Q=T<4&v@Fe<+TuXiZQT3G~vZ&^POfmI1K2h6t4eD}Gk5XFGpbj1n_g*{qmD6Xy z`6Vv|lLZtLmrnv*{Q%xxtcWVj3K4M%$bdBk_a&ar{{GWyu#ljM;dII;*jP;QH z#+^o-A4np{@|Mz+LphTD0`FTyxYq#wY)*&Ls5o{0z9yg2K+K7ZN>j1>N&;r+Z`vI| zDzG1LJZ+sE?m?>x{5LJx^)g&pGEpY=fQ-4}{x=ru;}FL$inHemOg%|R*ZXPodU}Kh zFEd5#+8rGq$Y<_?k-}r5zgQ3jRV=ooHiF|@z_#D4pKVEmn5CGV(9VKCyG|sT9nc=U zEoT67R`C->KY8Wp-fEcjjFm^;Cg(ls|*ABVHq8clBE(;~K^b+S>6uj70g? z&{XQ5U&!Z$SO7zfP+y^8XBbiu*Cv-yJG|l-oe*!s5$@Lh_KpxYL2sx`B|V=dETN>5K+C+CU~a_3cI8{vbu$TNVdGf15*>D zz@f{zIlorkY>TRh7mKuAlN9A0>N>SV`X)+bEHms=mfYTMWt_AJtz_h+JMmrgH?mZt zm=lfdF`t^J*XLg7v+iS)XZROygK=CS@CvUaJo&w2W!Wb@aa?~Drtf`JV^cCMjngVZ zv&xaIBEo8EYWuML+vxCpjjY^s1-ahXJzAV6hTw%ZIy!FjI}aJ+{rE&u#>rs)vzuxz z+$5z=7W?zH2>Eb32dvgHYZtCAf!=OLY-pb4>Ae79rd68E2LkVPj-|jFeyqtBCCwiW zkB@kO_(3wFq)7qwV}bA=zD!*@UhT`geq}ITo%@O(Z5Y80nEX~;0-8kO{oB6|(4fQh z);73T!>3@{ZobPwRv*W?7m0Ml9GmJBCJd&6E?hdj9lV= z4flNfsc(J*DyPv?RCOx!MSvk(M952PJ-G|JeVxWVjN~SNS6n-_Ge3Q;TGE;EQvZg86%wZ`MB zSMQua(i*R8a75!6$QRO^(o7sGoomb+Y{OMy;m~Oa`;P9Yqo>?bJAhqXxLr7_3g_n>f#UVtxG!^F#1+y@os6x(sg z^28bsQ@8rw%Gxk-stAEPRbv^}5sLe=VMbkc@Jjimqjvmd!3E7+QnL>|(^3!R} zD-l1l7*Amu@j+PWLGHXXaFG0Ct2Q=}5YNUxEQHCAU7gA$sSC<5OGylNnQUa>>l%sM zyu}z6i&({U@x^hln**o6r2s-(C-L50tQvz|zHTqW!ir?w&V23tuYEDJVV#5pE|OJu z7^R!A$iM$YCe?8n67l*J-okwfZ+ZTkGvZ)tVPfR;|3gyFjF)8V zyXXN=!*bpyRg9#~Bg1+UDYCt0 ztp4&?t1X0q>uz;ann$OrZs{5*r`(oNvw=$7O#rD|Wuv*wIi)4b zGtq4%BX+kkagv3F9Id6~-c+1&?zny%w5j&nk9SQfo0k4LhdSU_kWGW7axkfpgR`8* z!?UTG*Zi_baA1^0eda8S|@&F z{)Rad0kiLjB|=}XFJhD(S3ssKlveFFmkN{Vl^_nb!o5M!RC=m)V&v2%e?ZoRC@h3> zJ(?pvToFd`*Zc@HFPL#=otWKwtuuQ_dT-Hr{S%pQX<6dqVJ8;f(o)4~VM_kEQkMR+ zs1SCVi~k>M`u1u2xc}>#D!V&6nOOh-E$O&SzYrjJdZpaDv1!R-QGA141WjQe2s0J~ zQ;AXG)F+K#K8_5HVqRoRM%^EduqOnS(j2)|ctA6Q^=|s_WJYU;Z%5bHp08HPL`YF2 zR)Ad1z{zh`=sDs^&V}J z%$Z$!jd7BY5AkT?j`eqMs%!Gm@T8)4w3GYEX~IwgE~`d|@T{WYHkudy(47brgHXx& zBL1yFG6!!!VOSmDxBpefy2{L_u5yTwja&HA!mYA#wg#bc-m%~8aRR|~AvMnind@zs zy>wkShe5&*un^zvSOdlVu%kHsEo>@puMQ`b1}(|)l~E{5)f7gC=E$fP(FC2=F<^|A zxeIm?{EE!3sO!Gr7e{w)Dx(uU#3WrFZ>ibmKSQ1tY?*-Nh1TDHLe+k*;{Rp!Bmd_m zb#^kh`Y*8l|9Cz2e{;RL%_lg{#^Ar+NH|3z*Zye>!alpt{z;4dFAw^^H!6ING*EFc z_yqhr8d!;%nHX9AKhFQZBGrSzfzYCi%C!(Q5*~hX>)0N`vbhZ@N|i;_972WSx*>LH z87?en(;2_`{_JHF`Sv6Wlps;dCcj+8IJ8ca6`DsOQCMb3n# z3)_w%FuJ3>fjeOOtWyq)ag|PmgQbC-s}KRHG~enBcIwqIiGW8R8jFeBNY9|YswRY5 zjGUxdGgUD26wOpwM#8a!Nuqg68*dG@VM~SbOroL_On0N6QdT9?)NeB3@0FCC?Z|E0 z6TPZj(AsPtwCw>*{eDEE}Gby>0q{*lI+g2e&(YQrsY&uGM{O~}(oM@YWmb*F zA0^rr5~UD^qmNljq$F#ARXRZ1igP`MQx4aS6*MS;Ot(1L5jF2NJ;de!NujUYg$dr# z=TEL_zTj2@>ZZN(NYCeVX2==~=aT)R30gETO{G&GM4XN<+!&W&(WcDP%oL8PyIVUC zs5AvMgh6qr-2?^unB@mXK*Dbil^y-GTC+>&N5HkzXtozVf93m~xOUHn8`HpX=$_v2 z61H;Z1qK9o;>->tb8y%#4H)765W4E>TQ1o0PFj)uTOPEvv&}%(_mG0ISmyhnQV33Z$#&yd{ zc{>8V8XK$3u8}04CmAQ#I@XvtmB*s4t8va?-IY4@CN>;)mLb_4!&P3XSw4pA_NzDb zORn!blT-aHk1%Jpi>T~oGLuh{DB)JIGZ9KOsciWs2N7mM1JWM+lna4vkDL?Q)z_Ct z`!mi0jtr+4*L&N7jk&LodVO#6?_qRGVaucqVB8*us6i3BTa^^EI0x%EREQSXV@f!lak6Wf1cNZ8>*artIJ(ADO*=<-an`3zB4d*oO*8D1K!f z*A@P1bZCNtU=p!742MrAj%&5v%Xp_dSX@4YCw%F|%Dk=u|1BOmo)HsVz)nD5USa zR~??e61sO(;PR)iaxK{M%QM_rIua9C^4ppVS$qCT9j2%?*em?`4Z;4@>I(c%M&#cH z>4}*;ej<4cKkbCAjjDsyKS8rIm90O)Jjgyxj5^venBx&7B!xLmzxW3jhj7sR(^3Fz z84EY|p1NauwXUr;FfZjdaAfh%ivyp+^!jBjJuAaKa!yCq=?T_)R!>16?{~p)FQ3LDoMyG%hL#pR!f@P%*;#90rs_y z@9}@r1BmM-SJ#DeuqCQk=J?ixDSwL*wh|G#us;dd{H}3*-Y7Tv5m=bQJMcH+_S`zVtf;!0kt*(zwJ zs+kedTm!A}cMiM!qv(c$o5K%}Yd0|nOd0iLjus&;s0Acvoi-PFrWm?+q9f^FslxGi z6ywB`QpL$rJzWDg(4)C4+!2cLE}UPCTBLa*_=c#*$b2PWrRN46$y~yST3a2$7hEH= zNjux+wna^AzQ=KEa_5#9Ph=G1{S0#hh1L3hQ`@HrVnCx{!fw_a0N5xV(iPdKZ-HOM za)LdgK}1ww*C_>V7hbQnTzjURJL`S%`6nTHcgS+dB6b_;PY1FsrdE8(2K6FN>37!62j_cBlui{jO^$dPkGHV>pXvW0EiOA zqW`YaSUBWg_v^Y5tPJfWLcLpsA8T zG)!x>pKMpt!lv3&KV!-um= zKCir6`bEL_LCFx4Z5bAFXW$g3Cq`?Q%)3q0r852XI*Der*JNuKUZ`C{cCuu8R8nkt z%pnF>R$uY8L+D!V{s^9>IC+bmt<05h**>49R*#vpM*4i0qRB2uPbg8{{s#9yC;Z18 zD7|4m<9qneQ84uX|J&f-g8a|nFKFt34@Bt{CU`v(SYbbn95Q67*)_Esl_;v291s=9 z+#2F2apZU4Tq=x+?V}CjwD(P=U~d<=mfEFuyPB`Ey82V9G#Sk8H_Ob_RnP3s?)S_3 zr%}Pb?;lt_)Nf>@zX~D~TBr;-LS<1I##8z`;0ZCvI_QbXNh8Iv)$LS=*gHr;}dgb=w5$3k2la1keIm|=7<-JD>)U%=Avl0Vj@+&vxn zt-)`vJxJr88D&!}2^{GPXc^nmRf#}nb$4MMkBA21GzB`-Or`-3lq^O^svO7Vs~FdM zv`NvzyG+0T!P8l_&8gH|pzE{N(gv_tgDU7SWeiI-iHC#0Ai%Ixn4&nt{5y3(GQs)i z&uA;~_0shP$0Wh0VooIeyC|lak__#KVJfxa7*mYmZ22@(<^W}FdKjd*U1CqSjNKW% z*z$5$=t^+;Ui=MoDW~A7;)Mj%ibX1_p4gu>RC}Z_pl`U*{_z@+HN?AF{_W z?M_X@o%w8fgFIJ$fIzBeK=v#*`mtY$HC3tqw7q^GCT!P$I%=2N4FY7j9nG8aIm$c9 zeKTxVKN!UJ{#W)zxW|Q^K!3s;(*7Gbn;e@pQBCDS(I|Y0euK#dSQ_W^)sv5pa%<^o zyu}3d?Lx`)3-n5Sy9r#`I{+t6x%I%G(iewGbvor&I^{lhu-!#}*Q3^itvY(^UWXgvthH52zLy&T+B)Pw;5>4D6>74 zO_EBS)>l!zLTVkX@NDqyN2cXTwsUVao7$HcqV2%t$YzdAC&T)dwzExa3*kt9d(}al zA~M}=%2NVNUjZiO7c>04YH)sRelXJYpWSn^aC$|Ji|E13a^-v2MB!Nc*b+=KY7MCm zqIteKfNkONq}uM;PB?vvgQvfKLPMB8u5+Am=d#>g+o&Ysb>dX9EC8q?D$pJH!MTAqa=DS5$cb+;hEvjwVfF{4;M{5U&^_+r zvZdu_rildI!*|*A$TzJ&apQWV@p{!W`=?t(o0{?9y&vM)V)ycGSlI3`;ps(vf2PUq zX745#`cmT*ra7XECC0gKkpu2eyhFEUb?;4@X7weEnLjXj_F~?OzL1U1L0|s6M+kIhmi%`n5vvDALMagi4`wMc=JV{XiO+^ z?s9i7;GgrRW{Mx)d7rj)?(;|b-`iBNPqdwtt%32se@?w4<^KU&585_kZ=`Wy^oLu9 z?DQAh5z%q;UkP48jgMFHTf#mj?#z|=w= z(q6~17Vn}P)J3M?O)x))%a5+>TFW3No~TgP;f}K$#icBh;rSS+R|}l鯊%1Et zwk~hMkhq;MOw^Q5`7oC{CUUyTw9x>^%*FHx^qJw(LB+E0WBX@{Ghw;)6aA-KyYg8p z7XDveQOpEr;B4je@2~usI5BlFadedX^ma{b{ypd|RNYqo#~d*mj&y`^iojR}s%~vF z(H!u`yx68D1Tj(3(m;Q+Ma}s2n#;O~bcB1`lYk%Irx60&-nWIUBr2x&@}@76+*zJ5 ze&4?q8?m%L9c6h=J$WBzbiTf1Z-0Eb5$IZs>lvm$>1n_Mezp*qw_pr8<8$6f)5f<@ zyV#tzMCs51nTv_5ca`x`yfE5YA^*%O_H?;tWYdM_kHPubA%vy47i=9>Bq) zRQ&0UwLQHeswmB1yP)+BiR;S+Vc-5TX84KUA;8VY9}yEj0eESSO`7HQ4lO z4(CyA8y1G7_C;6kd4U3K-aNOK!sHE}KL_-^EDl(vB42P$2Km7$WGqNy=%fqB+ zSLdrlcbEH=T@W8V4(TgoXZ*G1_aq$K^@ek=TVhoKRjw;HyI&coln|uRr5mMOy2GXP zwr*F^Y|!Sjr2YQXX(Fp^*`Wk905K%$bd03R4(igl0&7IIm*#f`A!DCarW9$h$z`kYk9MjjqN&5-DsH@8xh63!fTNPxWsFQhNv z#|3RjnP$Thdb#Ys7M+v|>AHm0BVTw)EH}>x@_f4zca&3tXJhTZ8pO}aN?(dHo)44Z z_5j+YP=jMlFqwvf3lq!57-SAuRV2_gJ*wsR_!Y4Z(trO}0wmB9%f#jNDHPdQGHFR; zZXzS-$`;7DQ5vF~oSgP3bNV$6Z(rwo6W(U07b1n3UHqml>{=6&-4PALATsH@Bh^W? z)ob%oAPaiw{?9HfMzpGb)@Kys^J$CN{uf*HX?)z=g`J(uK1YO^8~s1(ZIbG%Et(|q z$D@_QqltVZu9Py4R0Ld8!U|#`5~^M=b>fnHthzKBRr=i+w@0Vr^l|W;=zFT#PJ?*a zbC}G#It}rQP^Ait^W&aa6B;+0gNvz4cWUMzpv(1gvfw-X4xJ2Sv;mt;zb2Tsn|kSS zo*U9N?I{=-;a-OybL4r;PolCfiaL=y@o9{%`>+&FI#D^uy#>)R@b^1ue&AKKwuI*` zx%+6r48EIX6nF4o;>)zhV_8(IEX})NGU6Vs(yslrx{5fII}o3SMHW7wGtK9oIO4OM&@@ECtXSICLcPXoS|{;=_yj>hh*%hP27yZwOmj4&Lh z*Nd@OMkd!aKReoqNOkp5cW*lC)&C$P?+H3*%8)6HcpBg&IhGP^77XPZpc%WKYLX$T zsSQ$|ntaVVOoRat$6lvZO(G-QM5s#N4j*|N_;8cc2v_k4n6zx9c1L4JL*83F-C1Cn zaJhd;>rHXB%%ZN=3_o3&Qd2YOxrK~&?1=UuN9QhL$~OY-Qyg&})#ez*8NpQW_*a&kD&ANjedxT0Ar z<6r{eaVz3`d~+N~vkMaV8{F?RBVemN(jD@S8qO~L{rUw#=2a$V(7rLE+kGUZ<%pdr z?$DP|Vg#gZ9S}w((O2NbxzQ^zTot=89!0^~hE{|c9q1hVzv0?YC5s42Yx($;hAp*E zyoGuRyphQY{Q2ee0Xx`1&lv(l-SeC$NEyS~8iil3_aNlnqF_G|;zt#F%1;J)jnPT& z@iU0S;wHJ2$f!juqEzPZeZkjcQ+Pa@eERSLKsWf=`{R@yv7AuRh&ALRTAy z8=g&nxsSJCe!QLchJ=}6|LshnXIK)SNd zRkJNiqHwKK{SO;N5m5wdL&qK`v|d?5<4!(FAsDxR>Ky#0#t$8XCMptvNo?|SY?d8b z`*8dVBlXTUanlh6n)!EHf2&PDG8sXNAt6~u-_1EjPI1|<=33T8 zEnA00E!`4Ave0d&VVh0e>)Dc}=FfAFxpsC1u9ATfQ`-Cu;mhc8Z>2;uyXtqpLb7(P zd2F9<3cXS} znMg?{&8_YFTGRQZEPU-XPq55%51}RJpw@LO_|)CFAt62-_!u_Uq$csc+7|3+TV_!h z+2a7Yh^5AA{q^m|=KSJL+w-EWDBc&I_I1vOr^}P8i?cKMhGy$CP0XKrQzCheG$}G# zuglf8*PAFO8%xop7KSwI8||liTaQ9NCAFarr~psQt)g*pC@9bORZ>m`_GA`_K@~&% zijH0z;T$fd;-Liw8%EKZas>BH8nYTqsK7F;>>@YsE=Rqo?_8}UO-S#|6~CAW0Oz1} z3F(1=+#wrBJh4H)9jTQ_$~@#9|Bc1Pd3rAIA_&vOpvvbgDJOM(yNPhJJq2%PCcMaI zrbe~toYzvkZYQ{ea(Wiyu#4WB#RRN%bMe=SOk!CbJZv^m?Flo5p{W8|0i3`hI3Np# zvCZqY%o258CI=SGb+A3yJe~JH^i{uU`#U#fvSC~rWTq+K`E%J@ zasU07&pB6A4w3b?d?q}2=0rA#SA7D`X+zg@&zm^iA*HVi z009#PUH<%lk4z~p^l0S{lCJk1Uxi=F4e_DwlfHA`X`rv(|JqWKAA5nH+u4Da+E_p+ zVmH@lg^n4ixs~*@gm_dgQ&eDmE1mnw5wBz9Yg?QdZwF|an67Xd*x!He)Gc8&2!urh z4_uXzbYz-aX)X1>&iUjGp;P1u8&7TID0bTH-jCL&Xk8b&;;6p2op_=y^m@Nq*0{#o!!A;wNAFG@0%Z9rHo zcJs?Th>Ny6+hI`+1XoU*ED$Yf@9f91m9Y=#N(HJP^Y@ZEYR6I?oM{>&Wq4|v0IB(p zqX#Z<_3X(&{H+{3Tr|sFy}~=bv+l=P;|sBz$wk-n^R`G3p0(p>p=5ahpaD7>r|>pm zv;V`_IR@tvZreIuv2EM7ZQHhO+qUgw#kOs%*ekY^n|=1#x9&c;Ro&I~{rG-#_3ZB1 z?|9}IFdbP}^DneP*T-JaoYHt~r@EfvnPE5EKUwIxjPbsr$% zfWW83pgWST7*B(o=kmo)74$8UU)v0{@4DI+ci&%=#90}!CZz|rnH+Mz=HN~97G3~@ z;v5(9_2%eca(9iu@J@aqaMS6*$TMw!S>H(b z4(*B!|H|8&EuB%mITr~O?vVEf%(Gr)6E=>H~1VR z&1YOXluJSG1!?TnT)_*YmJ*o_Q@om~(GdrhI{$Fsx_zrkupc#y{DK1WOUR>tk>ZE) ziOLoBkhZZ?0Uf}cm>GsA>Rd6V8@JF)J*EQlQ<=JD@m<)hyElXR0`pTku*3MU`HJn| zIf7$)RlK^pW-$87U;431;Ye4Ie+l~_B3*bH1>*yKzn23cH0u(i5pXV! z4K?{3oF7ZavmmtTq((wtml)m6i)8X6ot_mrE-QJCW}Yn!(3~aUHYG=^fA<^~`e3yc z-NWTb{gR;DOUcK#zPbN^D*e=2eR^_!(!RKkiwMW@@yYtEoOp4XjOGgzi`;=8 zi3`Ccw1%L*y(FDj=C7Ro-V?q)-%p?Ob2ZElu`eZ99n14-ZkEV#y5C+{Pq87Gu3&>g zFy~Wk7^6v*)4pF3@F@rE__k3ikx(hzN3@e*^0=KNA6|jC^B5nf(XaoQaZN?Xi}Rn3 z$8&m*KmWvPaUQ(V<#J+S&zO|8P-#!f%7G+n_%sXp9=J%Z4&9OkWXeuZN}ssgQ#Tcj z8p6ErJQJWZ+fXLCco=RN8D{W%+*kko*2-LEb))xcHwNl~Xmir>kmAxW?eW50Osw3# zki8Fl$#fvw*7rqd?%E?}ZX4`c5-R&w!Y0#EBbelVXSng+kUfeUiqofPehl}$ormli zg%r)}?%=?_pHb9`Cq9Z|B`L8b>(!+8HSX?`5+5mm81AFXfnAt1*R3F z%b2RPIacKAddx%JfQ8l{3U|vK@W7KB$CdLqn@wP^?azRks@x8z59#$Q*7q!KilY-P zHUbs(IFYRGG1{~@RF;Lqyho$~7^hNC`NL3kn^Td%A7dRgr_&`2k=t+}D-o9&C!y^? z6MsQ=tc3g0xkK(O%DzR9nbNB(r@L;1zQrs8mzx&4dz}?3KNYozOW5;=w18U6$G4U2 z#2^qRLT*Mo4bV1Oeo1PKQ2WQS2Y-hv&S|C7`xh6=Pj7MNLC5K-zokZ67S)C;(F0Dd zloDK2_o1$Fmza>EMj3X9je7e%Q`$39Dk~GoOj89-6q9|_WJlSl!!+*{R=tGp z8u|MuSwm^t7K^nUe+^0G3dkGZr3@(X+TL5eah)K^Tn zXEtHmR9UIaEYgD5Nhh(s*fcG_lh-mfy5iUF3xxpRZ0q3nZ=1qAtUa?(LnT9I&~uxX z`pV?+=|-Gl(kz?w!zIieXT}o}7@`QO>;u$Z!QB${a08_bW0_o@&9cjJUXzVyNGCm8 zm=W+$H!;_Kzp6WQqxUI;JlPY&`V}9C$8HZ^m?NvI*JT@~BM=()T()Ii#+*$y@lTZBkmMMda>7s#O(1YZR+zTG@&}!EXFG{ zEWPSDI5bFi;NT>Yj*FjH((=oe%t%xYmE~AGaOc4#9K_XsVpl<4SP@E!TgC0qpe1oi zNpxU2b0(lEMcoibQ-G^cxO?ySVW26HoBNa;n0}CWL*{k)oBu1>F18X061$SP{Gu67 z-v-Fa=Fl^u3lnGY^o5v)Bux}bNZ~ z5pL+7F_Esoun8^5>z8NFoIdb$sNS&xT8_|`GTe8zSXQzs4r^g0kZjg(b0bJvz`g<70u9Z3fQILX1Lj@;@+##bP|FAOl)U^9U>0rx zGi)M1(Hce)LAvQO-pW!MN$;#ZMX?VE(22lTlJrk#pB0FJNqVwC+*%${Gt#r_tH9I_ z;+#)#8cWAl?d@R+O+}@1A^hAR1s3UcW{G+>;X4utD2d9X(jF555}!TVN-hByV6t+A zdFR^aE@GNNgSxxixS2p=on4(+*+f<8xrwAObC)D5)4!z7)}mTpb7&ofF3u&9&wPS< zB62WHLGMhmrmOAgmJ+|c>qEWTD#jd~lHNgT0?t-p{T=~#EMcB| z=AoDKOL+qXCfk~F)-Rv**V}}gWFl>liXOl7Uec_8v)(S#av99PX1sQIVZ9eNLkhq$ zt|qu0b?GW_uo}TbU8!jYn8iJeIP)r@;!Ze_7mj{AUV$GEz6bDSDO=D!&C9!M@*S2! zfGyA|EPlXGMjkH6x7OMF?gKL7{GvGfED=Jte^p=91FpCu)#{whAMw`vSLa`K#atdN zThnL+7!ZNmP{rc=Z>%$meH;Qi1=m1E3Lq2D_O1-X5C;!I0L>zur@tPAC9*7Jeh)`;eec}1`nkRP(%iv-`N zZ@ip-g|7l6Hz%j%gcAM}6-nrC8oA$BkOTz^?dakvX?`^=ZkYh%vUE z9+&)K1UTK=ahYiaNn&G5nHUY5niLGus@p5E2@RwZufRvF{@$hW{;{3QhjvEHMvduO z#Wf-@oYU4ht?#uP{N3utVzV49mEc9>*TV_W2TVC`6+oI)zAjy$KJrr=*q##&kobiQ z1vNbya&OVjK`2pdRrM?LuK6BgrLN7H_3m z!qpNKg~87XgCwb#I=Q&0rI*l$wM!qTkXrx1ko5q-f;=R2fImRMwt5Qs{P*p^z@9ex z`2#v(qE&F%MXlHpdO#QEZyZftn4f05ab^f2vjxuFaat2}jke{j?5GrF=WYBR?gS(^ z9SBiNi}anzBDBRc+QqizTTQuJrzm^bNA~A{j%ugXP7McZqJ}65l10({wk++$=e8O{ zxWjG!Qp#5OmI#XRQQM?n6?1ztl6^D40hDJr?4$Wc&O_{*OfMfxe)V0=e{|N?J#fgE>j9jAajze$iN!*yeF%jJU#G1c@@rm zolGW!j?W6Q8pP=lkctNFdfgUMg92wlM4E$aks1??M$~WQfzzzXtS)wKrr2sJeCN4X zY(X^H_c^PzfcO8Bq(Q*p4c_v@F$Y8cHLrH$`pJ2}=#*8%JYdqsqnGqEdBQMpl!Ot04tUGSXTQdsX&GDtjbWD=prcCT9(+ z&UM%lW%Q3yrl1yiYs;LxzIy>2G}EPY6|sBhL&X&RAQrSAV4Tlh2nITR?{6xO9ujGu zr*)^E`>o!c=gT*_@6S&>0POxcXYNQd&HMw6<|#{eSute2C3{&h?Ah|cw56-AP^f8l zT^kvZY$YiH8j)sk7_=;gx)vx-PW`hbSBXJGCTkpt;ap(}G2GY=2bbjABU5)ty%G#x zAi07{Bjhv}>OD#5zh#$0w;-vvC@^}F! z#X$@)zIs1L^E;2xDAwEjaXhTBw2<{&JkF*`;c3<1U@A4MaLPe{M5DGGkL}#{cHL%* zYMG+-Fm0#qzPL#V)TvQVI|?_M>=zVJr9>(6ib*#z8q@mYKXDP`k&A4A};xMK0h=yrMp~JW{L?mE~ph&1Y1a#4%SO)@{ zK2juwynUOC)U*hVlJU17%llUxAJFuKZh3K0gU`aP)pc~bE~mM!i1mi!~LTf>1Wp< zuG+ahp^gH8g8-M$u{HUWh0m^9Rg@cQ{&DAO{PTMudV6c?ka7+AO& z746QylZ&Oj`1aqfu?l&zGtJnpEQOt;OAFq19MXTcI~`ZcoZmyMrIKDFRIDi`FH)w; z8+*8tdevMDv*VtQi|e}CnB_JWs>fhLOH-+Os2Lh!&)Oh2utl{*AwR)QVLS49iTp{6 z;|172Jl!Ml17unF+pd+Ff@jIE-{Oxv)5|pOm@CkHW?{l}b@1>Pe!l}VccX#xp@xgJ zyE<&ep$=*vT=}7vtvif0B?9xw_3Gej7mN*dOHdQPtW5kA5_zGD zpA4tV2*0E^OUimSsV#?Tg#oiQ>%4D@1F5@AHwT8Kgen$bSMHD3sXCkq8^(uo7CWk`mT zuslYq`6Yz;L%wJh$3l1%SZv#QnG3=NZ=BK4yzk#HAPbqXa92;3K5?0kn4TQ`%E%X} z&>Lbt!!QclYKd6+J7Nl@xv!uD%)*bY-;p`y^ZCC<%LEHUi$l5biu!sT3TGGSTPA21 zT8@B&a0lJHVn1I$I3I1I{W9fJAYc+8 zVj8>HvD}&O`TqU2AAb={?eT;0hyL(R{|h23=4fDSZKC32;wWxsVj`P z3J3{M$PwdH!ro*Cn!D&=jnFR>BNGR<<|I8CI@+@658Dy(lhqbhXfPTVecY@L8%`3Q z1Fux2w?2C3th60jI~%OC9BtpNF$QPqcG+Pz96qZJ71_`0o0w_q7|h&O>`6U+^BA&5 zXd5Zp1Xkw~>M%RixTm&OqpNl8Q+ue=92Op_>T~_9UON?ZM2c0aGm=^A4ejrXj3dV9 zhh_bCt-b9`uOX#cFLj!vhZ#lS8Tc47OH>*)y#{O9?AT~KR9LntM|#l#Dlm^8{nZdk zjMl#>ZM%#^nK2TPzLcKxqx24P7R1FPlBy7LSBrRvx>fE$9AJ;7{PQm~^LBX^k#6Zq zw*Z(zJC|`!6_)EFR}8|n8&&Rbj8y028~P~sFXBFRt+tmqH-S3<%N;C&WGH!f3{7cm zy_fCAb9@HqaXa1Y5vFbxWf%#zg6SI$C+Uz5=CTO}e|2fjWkZ;Dx|84Ow~bkI=LW+U zuq;KSv9VMboRvs9)}2PAO|b(JCEC_A0wq{uEj|3x@}*=bOd zwr{TgeCGG>HT<@Zeq8y}vTpwDg#UBvD)BEs@1KP$^3$sh&_joQPn{hjBXmLPJ{tC) z*HS`*2+VtJO{|e$mM^|qv1R*8i(m1`%)}g=SU#T#0KlTM2RSvYUc1fP+va|4;5}Bfz98UvDCpq7}+SMV&;nX zQw~N6qOX{P55{#LQkrZk(e5YGzr|(B;Q;ju;2a`q+S9bsEH@i1{_Y0;hWYn1-79jl z5c&bytD*k)GqrVcHn6t-7kinadiD>B{Tl`ZY@`g|b~pvHh5!gKP4({rp?D0aFd_cN zhHRo4dd5^S6ViN(>(28qZT6E>??aRhc($kP`>@<+lIKS5HdhjVU;>f7<4))E*5|g{ z&d1}D|vpuV^eRj5j|xx9nwaCxXFG?Qbjn~_WSy=N}P0W>MP zG-F%70lX5Xr$a)2i6?i|iMyM|;Jtf*hO?=Jxj12oz&>P=1#h~lf%#fc73M2_(SUM- zf&qnjS80|_Y0lDgl&I?*eMumUklLe_=Td!9G@eR*tcPOgIShJipp3{A10u(4eT~DY zHezEj8V+7m!knn7)W!-5QI3=IvC^as5+TW1@Ern@yX| z7Nn~xVx&fGSr+L%4iohtS3w^{-H1A_5=r&x8}R!YZvp<2T^YFvj8G_vm}5q;^UOJf ztl=X3iL;;^^a#`t{Ae-%5Oq{?M#s6Npj+L(n-*LMI-yMR{)qki!~{5z{&`-iL}lgW zxo+tnvICK=lImjV$Z|O_cYj_PlEYCzu-XBz&XC-JVxUh9;6*z4fuBG+H{voCC;`~GYV|hj%j_&I zDZCj>Q_0RCwFauYoVMiUSB+*Mx`tg)bWmM^SwMA+?lBg12QUF_x2b)b?qb88K-YUd z0dO}3k#QirBV<5%jL$#wlf!60dizu;tsp(7XLdI=eQs?P`tOZYMjVq&jE)qK*6B^$ zBe>VvH5TO>s>izhwJJ$<`a8fakTL!yM^Zfr2hV9`f}}VVUXK39p@G|xYRz{fTI+Yq z20d=)iwjuG9RB$%$^&8#(c0_j0t_C~^|n+c`Apu|x7~;#cS-s=X1|C*YxX3ailhg_|0`g!E&GZJEr?bh#Tpb8siR=JxWKc{#w7g zWznLwi;zLFmM1g8V5-P#RsM@iX>TK$xsWuujcsVR^7TQ@!+vCD<>Bk9tdCo7Mzgq5 zv8d>dK9x8C@Qoh01u@3h0X_`SZluTb@5o;{4{{eF!-4405x8X7hewZWpz z2qEi4UTiXTvsa(0X7kQH{3VMF>W|6;6iTrrYD2fMggFA&-CBEfSqPlQDxqsa>{e2M z(R5PJ7uOooFc|9GU0ELA%m4&4Ja#cQpNw8i8ACAoK6?-px+oBl_yKmenZut#Xumjz zk8p^OV2KY&?5MUwGrBOo?ki`Sxo#?-Q4gw*Sh0k`@ zFTaYK2;}%Zk-68`#5DXU$2#=%YL#S&MTN8bF+!J2VT6x^XBci6O)Q#JfW{YMz) zOBM>t2rSj)n#0a3cjvu}r|k3od6W(SN}V-cL?bi*Iz-8uOcCcsX0L>ZXjLqk zZu2uHq5B|Kt>e+=pPKu=1P@1r9WLgYFq_TNV1p9pu0erHGd!+bBp!qGi+~4A(RsYN@CyXNrC&hxGmW)u5m35OmWwX`I+0yByglO`}HC4nGE^_HUs^&A(uaM zKPj^=qI{&ayOq#z=p&pnx@@k&I1JI>cttJcu@Ihljt?6p^6{|ds`0MoQwp+I{3l6` zB<9S((RpLG^>=Kic`1LnhpW2=Gu!x`m~=y;A`Qk!-w`IN;S8S930#vBVMv2vCKi}u z6<-VPrU0AnE&vzwV(CFC0gnZYcpa-l5T0ZS$P6(?9AM;`Aj~XDvt;Jua=jIgF=Fm? zdp=M$>`phx%+Gu};;-&7T|B1AcC#L4@mW5SV_^1BRbo6;2PWe$r+npRV`yc;T1mo& z+~_?7rA+(Um&o@Tddl zL_hxvWk~a)yY}%j`Y+200D%9$bWHy&;(yj{jpi?Rtz{J66ANw)UyPOm;t6FzY3$hx zcn)Ir79nhFvNa7^a{SHN7XH*|Vlsx`CddPnA&Qvh8aNhEA;mPVv;Ah=k<*u!Zq^7 z<=xs*iQTQOMMcg|(NA_auh@x`3#_LFt=)}%SQppP{E>mu_LgquAWvh<>L7tf9+~rO znwUDS52u)OtY<~!d$;m9+87aO+&`#2ICl@Y>&F{jI=H(K+@3M1$rr=*H^dye#~TyD z!){#Pyfn+|ugUu}G;a~!&&0aqQ59U@UT3|_JuBlYUpT$2+11;}JBJ`{+lQN9T@QFY z5+`t;6(TS0F?OlBTE!@7D`8#URDNqx2t6`GZ{ZgXeS@v%-eJzZOHz18aS|svxII$a zZeFjrJ*$IwX$f-Rzr_G>xbu@euGl)B7pC&S+CmDJBg$BoV~jxSO#>y z33`bupN#LDoW0feZe0%q8un0rYN|eRAnwDHQ6e_)xBTbtoZtTA=Fvk){q}9Os~6mQ zKB80VI_&6iSq`LnK7*kfHZoeX6?WE}8yjuDn=2#JG$+;-TOA1%^=DnXx%w{b=w}tS zQbU3XxtOI8E(!%`64r2`zog;5<0b4i)xBmGP^jiDZ2%HNSxIf3@wKs~uk4%3Mxz;~ zts_S~E4>W+YwI<-*-$U8*^HKDEa8oLbmqGg?3vewnaNg%Mm)W=)lcC_J+1ov^u*N3 zXJ?!BrH-+wGYziJq2Y#vyry6Z>NPgkEk+Ke`^DvNRdb>Q2Nlr#v%O@<5hbflI6EKE z9dWc0-ORk^T}jP!nkJ1imyjdVX@GrjOs%cpgA8-c&FH&$(4od#x6Y&=LiJZPINVyW z0snY$8JW@>tc2}DlrD3StQmA0Twck~@>8dSix9CyQOALcREdxoM$Sw*l!}bXKq9&r zysMWR@%OY24@e`?+#xV2bk{T^C_xSo8v2ZI=lBI*l{RciPwuE>L5@uhz@{!l)rtVlWC>)6(G)1~n=Q|S!{E9~6*fdpa*n z!()-8EpTdj=zr_Lswi;#{TxbtH$8*G=UM`I+icz7sr_SdnHXrv=?iEOF1UL+*6O;% zPw>t^kbW9X@oEXx<97%lBm-9?O_7L!DeD)Me#rwE54t~UBu9VZ zl_I1tBB~>jm@bw0Aljz8! zXBB6ATG6iByKIxs!qr%pz%wgqbg(l{65DP4#v(vqhhL{0b#0C8mq`bnqZ1OwFV z7mlZZJFMACm>h9v^2J9+^_zc1=JjL#qM5ZHaThH&n zXPTsR8(+)cj&>Un{6v*z?@VTLr{TmZ@-fY%*o2G}*G}#!bmqpoo*Ay@U!JI^Q@7gj;Kg-HIrLj4}#ec4~D2~X6vo;ghep-@&yOivYP zC19L0D`jjKy1Yi-SGPAn94(768Tcf$urAf{)1)9W58P`6MA{YG%O?|07!g9(b`8PXG1B1Sh0?HQmeJtP0M$O$hI z{5G`&9XzYhh|y@qsF1GnHN|~^ru~HVf#)lOTSrv=S@DyR$UKQk zjdEPFDz{uHM&UM;=mG!xKvp;xAGHOBo~>_=WFTmh$chpC7c`~7?36h)7$fF~Ii}8q zF|YXxH-Z?d+Q+27Rs3X9S&K3N+)OBxMHn1u(vlrUC6ckBY@@jl+mgr#KQUKo#VeFm zFwNYgv0<%~Wn}KeLeD9e1$S>jhOq&(e*I@L<=I5b(?G(zpqI*WBqf|Zge0&aoDUsC zngMRA_Kt0>La+Erl=Uv_J^p(z=!?XHpenzn$%EA`JIq#yYF?JLDMYiPfM(&Csr#f{ zdd+LJL1by?xz|D8+(fgzRs~(N1k9DSyK@LJygwaYX8dZl0W!I&c^K?7)z{2is;OkE zd$VK-(uH#AUaZrp=1z;O*n=b?QJkxu`Xsw&7yrX0?(CX=I-C#T;yi8a<{E~?vr3W> zQrpPqOW2M+AnZ&p{hqmHZU-;Q(7?- zP8L|Q0RM~sB0w1w53f&Kd*y}ofx@c z5Y6B8qGel+uT1JMot$nT1!Tim6{>oZzJXdyA+4euOLME?5Fd_85Uk%#E*ln%y{u8Q z$|?|R@Hpb~yTVK-Yr_S#%NUy7EBfYGAg>b({J|5b+j-PBpPy$Ns`PaJin4JdRfOaS zE|<HjH%NuJgsd2wOlv>~y=np%=2)$M9LS|>P)zJ+Fei5vYo_N~B0XCn+GM76 z)Xz3tg*FRVFgIl9zpESgdpWAavvVViGlU8|UFY{{gVJskg*I!ZjWyk~OW-Td4(mZ6 zB&SQreAAMqwp}rjy`HsG({l2&q5Y52<@AULVAu~rWI$UbFuZs>Sc*x+XI<+ez%$U)|a^unjpiW0l0 zj1!K0(b6$8LOjzRqQ~K&dfbMIE=TF}XFAi)$+h}5SD3lo z%%Qd>p9se=VtQG{kQ;N`sI)G^u|DN#7{aoEd zkksYP%_X$Rq08);-s6o>CGJ<}v`qs%eYf+J%DQ^2k68C%nvikRsN?$ap--f+vCS`K z#&~)f7!N^;sdUXu54gl3L=LN>FB^tuK=y2e#|hWiWUls__n@L|>xH{%8lIJTd5`w? zSwZbnS;W~DawT4OwSJVdAylbY+u5S+ZH{4hAi2&}Iv~W(UvHg(1GTZRPz`@{SOqzy z(8g&Dz=$PfRV=6FgxN~zo+G8OoPI&d-thcGVR*_^(R8COTM@bq?fDwY{}WhsQS1AK zF6R1t8!RdFmfocpJ6?9Yv~;WYi~XPgs(|>{5})j!AR!voO7y9&cMPo#80A(`za@t>cx<0;qxM@S*m(jYP)dMXr*?q0E`oL;12}VAep179uEr8c<=D zr5?A*C{eJ`z9Ee;E$8)MECqatHkbHH z&Y+ho0B$31MIB-xm&;xyaFCtg<{m~M-QDbY)fQ>Q*Xibb~8ytxZQ?QMf9!%cV zU0_X1@b4d+Pg#R!`OJ~DOrQz3@cpiGy~XSKjZQQ|^4J1puvwKeScrH8o{bscBsowomu z^f12kTvje`yEI3eEXDHJ6L+O{Jv$HVj%IKb|J{IvD*l6IG8WUgDJ*UGz z3!C%>?=dlfSJ>4U88)V+`U-!9r^@AxJBx8R;)J4Fn@`~k>8>v0M9xp90OJElWP&R5 zM#v*vtT}*Gm1^)Bv!s72T3PB0yVIjJW)H7a)ilkAvoaH?)jjb`MP>2z{%Y?}83 zUIwBKn`-MSg)=?R)1Q0z3b>dHE^)D8LFs}6ASG1|daDly_^lOSy&zIIhm*HXm1?VS=_iacG);_I9c zUQH1>i#*?oPIwBMJkzi_*>HoUe}_4o>2(SHWzqQ=;TyhAHS;Enr7!#8;sdlty&(>d zl%5cjri8`2X^Ds`jnw7>A`X|bl=U8n+3LKLy(1dAu8`g@9=5iw$R0qk)w8Vh_Dt^U zIglK}sn^)W7aB(Q>HvrX=rxB z+*L)3DiqpQ_%~|m=44LcD4-bxO3OO*LPjsh%p(k?&jvLp0py57oMH|*IMa(<|{m1(0S|x)?R-mqJ=I;_YUZA>J z62v*eSK;5w!h8J+6Z2~oyGdZ68waWfy09?4fU&m7%u~zi?YPHPgK6LDwphgaYu%0j zurtw)AYOpYKgHBrkX189mlJ`q)w-f|6>IER{5Lk97%P~a-JyCRFjejW@L>n4vt6#hq;!|m;hNE||LK3nw1{bJOy+eBJjK=QqNjI;Q6;Rp5 z&035pZDUZ#%Oa;&_7x0T<7!RW`#YBOj}F380Bq?MjjEhrvlCATPdkCTTl+2efTX$k zH&0zR1n^`C3ef~^sXzJK-)52(T}uTG%OF8yDhT76L~|^+hZ2hiSM*QA9*D5odI1>& z9kV9jC~twA5MwyOx(lsGD_ggYmztXPD`2=_V|ks_FOx!_J8!zM zTzh^cc+=VNZ&(OdN=y4Juw)@8-85lwf_#VMN!Ed(eQiRiLB2^2e`4dp286h@v@`O%_b)Y~A; zv}r6U?zs&@uD_+(_4bwoy7*uozNvp?bXFoB8?l8yG0qsm1JYzIvB_OH4_2G*IIOwT zVl%HX1562vLVcxM_RG*~w_`FbIc!(T=3>r528#%mwwMK}uEhJ()3MEby zQQjzqjWkwfI~;Fuj(Lj=Ug0y`>~C7`w&wzjK(rPw+Hpd~EvQ-ufQOiB4OMpyUKJhw zqEt~jle9d7S~LI~$6Z->J~QJ{Vdn3!c}g9}*KG^Kzr^(7VI5Gk(mHLL{itj_hG?&K4Ws0+T4gLfi3eu$N=`s36geNC?c zm!~}vG6lx9Uf^5M;bWntF<-{p^bruy~f?sk9 zcETAPQZLoJ8JzMMg<-=ju4keY@SY%Wo?u9Gx=j&dfa6LIAB|IrbORLV1-H==Z1zCM zeZcOYpm5>U2fU7V*h;%n`8 zN95QhfD994={1*<2vKLCNF)feKOGk`R#K~G=;rfq}|)s20&MCa65 zUM?xF5!&e0lF%|U!#rD@I{~OsS_?=;s_MQ_b_s=PuWdC)q|UQ&ea)DMRh5>fpQjXe z%9#*x=7{iRCtBKT#H>#v%>77|{4_slZ)XCY{s3j_r{tdpvb#|r|sbS^dU1x70$eJMU!h{Y7Kd{dl}9&vxQl6Jt1a` zHQZrWyY0?!vqf@u-fxU_@+}u(%Wm>0I#KP48tiAPYY!TdW(o|KtVI|EUB9V`CBBNaBLVih7+yMVF|GSoIQD0Jfb{ z!OXq;(>Z?O`1gap(L~bUcp>Lc@Jl-})^=6P%<~~9ywY=$iu8pJ0m*hOPzr~q`23eX zgbs;VOxxENe0UMVeN*>uCn9Gk!4siN-e>x)pIKAbQz!G)TcqIJ0`JBBaX>1-4_XO_-HCS^vr2vjv#7KltDZdyQ{tlWh4$Gm zB>|O1cBDC)yG(sbnc*@w6e%e}r*|IhpXckx&;sQCwGdKH+3oSG-2)Bf#x`@<4ETAr z0My%7RFh6ZLiZ_;X6Mu1YmXx7C$lSZ^}1h;j`EZd6@%JNUe=btBE z%s=Xmo1Ps?8G`}9+6>iaB8bgjUdXT?=trMu|4yLX^m0Dg{m7rpKNJey|EwHI+nN1e zL^>qN%5Fg)dGs4DO~uwIdXImN)QJ*Jhpj7$fq_^`{3fwpztL@WBB}OwQ#Epo-mqMO zsM$UgpFiG&d#)lzEQ{3Q;)&zTw;SzGOah-Dpm{!q7<8*)Ti_;xvV2TYXa}=faXZy? z3y?~GY@kl)>G&EvEijk9y1S`*=zBJSB1iet>0;x1Ai)*`^{pj0JMs)KAM=@UyOGtO z3y0BouW$N&TnwU6!%zS%nIrnANvZF&vB1~P5_d`x-giHuG zPJ;>XkVoghm#kZXRf>qxxEix;2;D1CC~NrbO6NBX!`&_$iXwP~P*c($EVV|669kDO zKoTLZNF4Cskh!Jz5ga9uZ`3o%7Pv`d^;a=cXI|>y;zC3rYPFLQkF*nv(r>SQvD*## z(Vo%^9g`%XwS0t#94zPq;mYGLKu4LU3;txF26?V~A0xZbU4Lmy`)>SoQX^m7fd^*E z+%{R4eN!rIk~K)M&UEzxp9dbY;_I^c} zOc{wlIrN_P(PPqi51k_$>Lt|X6A^|CGYgKAmoI#Li?;Wq%q~q*L7ehZkUrMxW67Jl zhsb~+U?33QS>eqyN{(odAkbopo=Q$Az?L+NZW>j;#~@wCDX?=L5SI|OxI~7!Pli;e zELMFcZtJY3!|=Gr2L4>z8yQ-{To>(f80*#;6`4IAiqUw`=Pg$%C?#1 z_g@hIGerILSU>=P>z{gM|DS91A4cT@PEIB^hSop!uhMo#2G;+tQSpDO_6nOnPWSLU zS;a9m^DFMXR4?*X=}d7l;nXuHk&0|m`NQn%d?8|Ab3A9l9Jh5s120ibWBdB z$5YwsK3;wvp!Kn@)Qae{ef`0#NwlRpQ}k^r>yos_Ne1;xyKLO?4)t_G4eK~wkUS2A&@_;)K0-03XGBzU+5f+uMDxC z(s8!8!RvdC#@`~fx$r)TKdLD6fWEVdEYtV#{ncT-ZMX~eI#UeQ-+H(Z43vVn%Yj9X zLdu9>o%wnWdvzA-#d6Z~vzj-}V3FQ5;axDIZ;i(95IIU=GQ4WuU{tl-{gk!5{l4_d zvvb&uE{%!iFwpymz{wh?bKr1*qzeZb5f6e6m_ozRF&zux2mlK=v_(_s^R6b5lu?_W4W3#<$zeG~Pd)^!4tzhs}-Sx$FJP>)ZGF(hVTH|C3(U zs0PO&*h_ zNA-&qZpTP$$LtIgfiCn07}XDbK#HIXdmv8zdz4TY;ifNIH-0jy(gMSByG2EF~Th#eb_TueZC` zE?3I>UTMpKQ})=C;6p!?G)M6w^u*A57bD?2X`m3X^6;&4%i_m(uGJ3Z5h`nwxM<)H z$I5m?wN>O~8`BGnZ=y^p6;0+%_0K}Dcg|K;+fEi|qoBqvHj(M&aHGqNF48~XqhtU? z^ogwBzRlOfpAJ+Rw7IED8lRbTdBdyEK$gPUpUG}j-M42xDj_&qEAQEtbs>D#dRd7Y z<&TpSZ(quQDHiCFn&0xsrz~4`4tz!CdL8m~HxZM_agu@IrBpyeL1Ft}V$HX_ZqDPm z-f89)pjuEzGdq-PRu`b1m+qBGY{zr_>{6Ss>F|xHZlJj9dt5HD$u`1*WZe)qEIuDSR)%z+|n zatVlhQ?$w#XRS7xUrFE;Y8vMGhQS5*T{ZnY=q1P?w5g$OKJ#M&e??tAmPWHMj3xhS ziGxapy?kn@$~2%ZY;M8Bc@%$pkl%Rvj!?o%agBvpQ-Q61n9kznC4ttrRNQ4%GFR5u zyv%Yo9~yxQJWJSfj z?#HY$y=O~F|2pZs22pu|_&Ajd+D(Mt!nPUG{|1nlvP`=R#kKH zO*s$r_%ss5h1YO7k0bHJ2CXN)Yd6CHn~W!R=SqkWe=&nAZu(Q1G!xgcUilM@YVei@2@a`8he z9@pM`)VB*=e7-MWgLlXlc)t;fF&-AwM{E-EX}pViFn0I0CNw2bNEnN2dj!^4(^zS3 zobUm1uQnpqk_4q{pl*n06=TfK_C>UgurKFjRXsK_LEn};=79`TB12tv6KzwSu*-C8 z;=~ohDLZylHQ|Mpx-?yql>|e=vI1Z!epyUpAcDCp4T|*RV&X`Q$0ogNwy6mFALo^@ z9=&(9txO8V@E!@6^(W0{*~CT>+-MA~vnJULBxCTUW>X5>r7*eXYUT0B6+w@lzw%n> z_VjJ<2qf|(d6jYq2(x$(ZDf!yVkfnbvNmb5c|hhZ^2TV_LBz`9w!e_V*W_(MiA7|= z&EeIIkw*+$Xd!)j8<@_<}A5;~A_>3JT*kX^@}cDoLd>Qj<`Se^wdUa(j0dp+Tl8EptwBm{9OGsdFEq zM`!pjf(Lm(`$e3FLOjqA5LnN5o!}z{ zNf}rJuZh@yUtq&ErjHeGzX4(!luV!jB&;FAP|!R_QHYw#^Z1LwTePAKJ6X&IDNO#; z)#I@Xnnzyij~C@UH~X51JCgQeF0&hTXnuoElz#m{heZRexWc0k4<>0+ClX7%0 zEBqCCld1tD9Zwkr4{?Nor19#E5-YKfB8d?qgR82-Ow2^AuNevly2*tHA|sK!ybYkX zm-sLQH72P&{vEAW6+z~O5d0qd=xW~rua~5a?ymYFSD@8&gV)E5@RNNBAj^C99+Z5Z zR@Pq55mbCQbz+Mn$d_CMW<-+?TU960agEk1J<>d>0K=pF19yN))a~4>m^G&tc*xR+yMD*S=yip-q=H zIlredHpsJV8H(32@Zxc@bX6a21dUV95Th--8pE6C&3F>pk=yv$yd6@Haw;$v4+Fcb zRwn{Qo@0`7aPa2LQOP}j9v>sjOo5Kqvn|`FLizX zB+@-u4Lw|jsvz{p^>n8Vo8H2peIqJJnMN}A)q6%$Tmig7eu^}K2 zrh$X?T|ZMsoh{6pdw1G$_T<`Ds-G=jc;qcGdK4{?dN2-XxjDNbb(7pk|3JUVCU4y; z)?LXR>f+AAu)JEiti_Zy#z5{RgsC}R(@jl%9YZ>zu~hKQ*AxbvhC378-I@{~#%Y`Z zy=a=9YpewPIC+gkEUUwtUL7|RU7=!^Aa}Mk^6uxOgRGA#JXjWLsjFUnix|Mau{hDT z7mn*z1m5g`vP(#tjT0Zy4eAY(br&!RiiXE=ZI!{sE1#^#%x^Z7t1U)b<;%Y}Q9=5v z;wpDCEZ@OE36TWT=|gxigT@VaW9BvHS05;_P(#s z8zI4XFQys}q)<`tkX$WnSarn{3e!s}4(J!=Yf>+Y>cP3f;vr63f2{|S^`_pWc)^5_!R z*(x-fuBxL51@xe!lnDBKi}Br$c$BMZ3%f2Sa6kLabiBS{pq*yj;q|k(86x`PiC{p6 z_bxCW{>Q2BA8~Ggz&0jkrcU+-$ANBsOop*ms>34K9lNYil@}jC;?cYP(m^P}nR6FV zk(M%48Z&%2Rx$A&FhOEirEhY0(dn;-k(qkTU)sFQ`+-ih+s@A8g?r8Pw+}2;35WYf zi}VO`jS`p(tc)$X$a>-#WXoW!phhatC*$}|rk>|wUU71eUJG^$c6_jwX?iSHM@6__ zvV|6%U*$sSXJu9SX?2%M^kK|}a2QJ8AhF{fuXrHZxXsI~O zGKX45!K7p*MCPEQ=gp?eu&#AW*pR{lhQR##P_*{c_DjMGL|3T3-bSJ(o$|M{ytU}> zAV>wq*uE*qFo9KvnA^@juy{x<-u*#2NvkV={Ly}ysKYB-k`K3@K#^S1Bb$8Y#0L0# z`6IkSG&|Z$ODy|VLS+y5pFJx&8tvPmMd8c9FhCyiU8~k6FwkakUd^(_ml8`rnl>JS zZV){9G*)xBqPz^LDqRwyS6w86#D^~xP4($150M)SOZRe9sn=>V#aG0Iy(_^YcPpIz8QYM-#s+n% z@Jd?xQq?Xk6=<3xSY7XYP$$yd&Spu{A#uafiIfy8gRC`o0nk{ezEDjb=q_qRAlR1d zFq^*9Gn)yTG4b}R{!+3hWQ+u3GT~8nwl2S1lpw`s0X_qpxv)g+JIkVKl${sYf_nV~B>Em>M;RlqGb5WVil(89 zs=ld@|#;dq1*vQGz=7--Br-|l) zZ%Xh@v8>B7P?~}?Cg$q9_={59l%m~O&*a6TKsCMAzG&vD>k2WDzJ6!tc!V)+oxF;h zJH;apM=wO?r_+*#;ulohuP=E>^zon}a$NnlcQ{1$SO*i=jnGVcQa^>QOILc)e6;eNTI>os=eaJ{*^DE+~jc zS}TYeOykDmJ=6O%>m`i*>&pO_S;qMySJIyP=}4E&J%#1zju$RpVAkZbEl+p%?ZP^C z*$$2b4t%a(e+%>a>d_f_<JjxI#J1x;=hPd1zFPx=6T$;;X1TD*2(edZ3f46zaAoW>L53vS_J*N8TMB|n+;LD| zC=GkQPpyDY#Am4l49chDv*gojhRj_?63&&8#doW`INATAo(qY#{q}%nf@eTIXmtU< zdB<7YWfyCmBs|c)cK>1)v&M#!yNj#4d$~pVfDWQc_ke1?fw{T1Nce_b`v|Vp5ig(H zJvRD^+ps46^hLX;=e2!2e;w9y1D@!D$c@Jc&%%%IL=+xzw55&2?darw=9g~>P z9>?Kdc$r?6c$m%x2S$sdpPl>GQZ{rC9mPS63*qjCVa?OIBj!fW zm|g?>CVfGXNjOfcyqImXR_(tXS(F{FcoNzKvG5R$IgGaxC@)i(e+$ME}vPVIhd|mx2IIE+f zM?9opQHIVgBWu)^A|RzXw!^??S!x)SZOwZaJkGjc<_}2l^eSBm!eAJG9T>EC6I_sy z?bxzDIAn&K5*mX)$RQzDA?s)-no-XF(g*yl4%+GBf`##bDXJ==AQk*xmnatI;SsLp zP9XTHq5mmS=iWu~9ES>b%Q=1aMa|ya^vj$@qz9S!ih{T8_PD%Sf_QrNKwgrXw9ldm zHRVR98*{C?_XNpJn{abA!oix_mowRMu^2lV-LPi;0+?-F(>^5#OHX-fPED zCu^l7u3E%STI}c4{J2!)9SUlGP_@!d?5W^QJXOI-Ea`hFMKjR7TluLvzC-ozCPn1`Tpy z!vlv@_Z58ILX6>nDjTp-1LlFMx~-%GA`aJvG$?8*Ihn;mH37eK**rmOEwqegf-Ccx zrIX4;{c~RK>XuTXxYo5kMiWMy)!IC{*DHG@E$hx?RwP@+wuad(P1{@%tRkyJRqD)3 zMHHHZ4boqDn>-=DgR5VlhQTpfVy182Gk;A_S8A1-;U1RR>+$62>(MUx@Nox$vTjHq z%QR=j!6Gdyb5wu7y(YUktwMuW5<@jl?m4cv4BODiT5o8qVdC0MBqGr@-YBIwnpZAY znX9(_uQjP}JJ=!~Ve9#5I~rUnN|P_3D$LqZcvBnywYhjlMSFHm`;u9GPla{5QD7(7*6Tb3Svr8;(nuAd81q$*uq6HC_&~je*Ca7hP4sJp0av{M8480wF zxASi7Qv+~@2U%Nu1Ud;s-G4CTVWIPyx!sg&8ZG0Wq zG_}i3C(6_1>q3w!EH7$Kwq8uBp2F2N7}l65mk1p*9v0&+;th=_E-W)E;w}P(j⁢ zv5o9#E7!G0XmdzfsS{efPNi`1b44~SZ4Z8fuX!I}#8g+(wxzQwUT#Xb2(tbY1+EUhGKoT@KEU9Ktl>_0 z%bjDJg;#*gtJZv!-Zs`?^}v5eKmnbjqlvnSzE@_SP|LG_PJ6CYU+6zY6>92%E+ z=j@TZf-iW4(%U{lnYxQA;7Q!b;^brF8n0D>)`q5>|WDDXLrqYU_tKN2>=#@~OE7grMnNh?UOz-O~6 z6%rHy{#h9K0AT+lDC7q4{hw^|q6*Ry;;L%Q@)Ga}$60_q%D)rv(CtS$CQbpq9|y1e zRSrN4;$Jyl{m5bZw`$8TGvb}(LpY{-cQ)fcyJv7l3S52TLXVDsphtv&aPuDk1OzCA z4A^QtC(!11`IsNx_HnSy?>EKpHJWT^wmS~hc^p^zIIh@9f6U@I2 zC=Mve{j2^)mS#U$e{@Q?SO6%LDsXz@SY+=cK_QMmXBIU)j!$ajc-zLx3V60EXJ!qC zi<%2x8Q24YN+&8U@CIlN zrZkcT9yh%LrlGS9`G)KdP(@9Eo-AQz@8GEFWcb7U=a0H^ZVbLmz{+&M7W(nXJ4sN8 zJLR7eeK(K8`2-}j(T7JsO`L!+CvbueT%izanm-^A1Dn{`1Nw`9P?cq;7no+XfC`K(GO9?O^5zNIt4M+M8LM0=7Gz8UA@Z0N+lg+cX)NfazRu z5D)~HA^(u%w^cz+@2@_#S|u>GpB+j4KzQ^&Wcl9f z&hG#bCA(Yk0D&t&aJE^xME^&E-&xGHhXn%}psEIj641H+Nl-}boj;)Zt*t(4wZ5DN z@GXF$bL=&pBq-#vkTkh>7hl%K5|3 z{`Vn9b$iR-SoGENp}bn4;fR3>9sA%X2@1L3aE9yTra;Wb#_`xWwLSLdfu+PAu+o3| zGVnpzPr=ch{uuoHjtw7+_!L_2;knQ!DuDl0R`|%jr+}jFzXtrHIKc323?JO{l&;VF z*L1+}JU7%QJOg|5|Tc|D8fN zJORAg=_vsy{ak|o);@)Yh8Lkcg@$FG3k@ep36BRa^>~UmnRPziS>Z=`Jb2x*Q#`%A zU*i3&Vg?TluO@X0O;r2Jl6LKLUOVhSqg1*qOt^|8*c7 zo(298@+r$k_wQNGHv{|$tW(T8L+4_`FQ{kEW5Jgg{yf7ey4ss_(SNKfz(N9lx&a;< je(UuV8hP?p&}TPdm1I$XmG#(RzlD&B2izSj9sl%y5~4qc diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 3999f7f3f6..ca8f2653b1 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip networkTimeout=10000 zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=3e1af3ae886920c3ac87f7a91f816c0c7c436f276a6eefdb3da152100fef72ae +distributionSha256Sum=9d926787066a081739e8200858338b4a69e837c3a821a33aca9db09dd4a41026 From b038f93f9711f6a48e96fa82f5830534502d58f6 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Fri, 5 Jan 2024 04:31:57 -0500 Subject: [PATCH 074/204] Refactor SSL handler retrieval to use HttpChannel / TranportChannel APIs instead of typecasting (#3917) ### Description This is cherry-pick from https://github.com/opensearch-project/security/pull/3514 to use the channel properties instead of type-casting ### Issues Resolved Closes https://github.com/opensearch-project/security/issues/3911 Is this a backport? If so, please add backport PR # and/or commits # ### Testing The change is covered by existing test suites ### Check List - [X] New functionality includes testing - [X] New functionality has been documented - [X] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). Signed-off-by: Andriy Redko --- .../security/filter/NettyAttribute.java | 17 ++++++++-------- .../security/filter/OpenSearchRequest.java | 15 ++------------ .../transport/SecuritySSLRequestHandler.java | 20 +------------------ 3 files changed, 12 insertions(+), 40 deletions(-) diff --git a/src/main/java/org/opensearch/security/filter/NettyAttribute.java b/src/main/java/org/opensearch/security/filter/NettyAttribute.java index 46bf0296cb..3a035a390b 100644 --- a/src/main/java/org/opensearch/security/filter/NettyAttribute.java +++ b/src/main/java/org/opensearch/security/filter/NettyAttribute.java @@ -12,7 +12,7 @@ import java.util.Optional; -import org.opensearch.http.netty4.Netty4HttpChannel; +import org.opensearch.http.HttpChannel; import org.opensearch.rest.RestRequest; import io.netty.channel.Channel; @@ -25,11 +25,12 @@ public class NettyAttribute { * Gets an attribute value from the request context and clears it from that context */ public static Optional popFrom(final RestRequest request, final AttributeKey attribute) { - if (request.getHttpChannel() instanceof Netty4HttpChannel) { - Channel nettyChannel = ((Netty4HttpChannel) request.getHttpChannel()).getNettyChannel(); - return Optional.ofNullable(nettyChannel.attr(attribute).getAndSet(null)); + final HttpChannel httpChannel = request.getHttpChannel(); + if (httpChannel != null) { + return httpChannel.get("channel", Channel.class).map(channel -> channel.attr(attribute).getAndSet(null)); + } else { + return Optional.empty(); } - return Optional.empty(); } /** @@ -50,9 +51,9 @@ public static Optional peekFrom(final ChannelHandlerContext ctx, final At * Clears an attribute value from the channel handler context */ public static void clearAttribute(final RestRequest request, final AttributeKey attribute) { - if (request.getHttpChannel() instanceof Netty4HttpChannel) { - Channel nettyChannel = ((Netty4HttpChannel) request.getHttpChannel()).getNettyChannel(); - nettyChannel.attr(attribute).set(null); + final HttpChannel httpChannel = request.getHttpChannel(); + if (httpChannel != null) { + httpChannel.get("channel", Channel.class).ifPresent(channel -> channel.attr(attribute).set(null)); } } diff --git a/src/main/java/org/opensearch/security/filter/OpenSearchRequest.java b/src/main/java/org/opensearch/security/filter/OpenSearchRequest.java index 80ede8b2c1..e86012f594 100644 --- a/src/main/java/org/opensearch/security/filter/OpenSearchRequest.java +++ b/src/main/java/org/opensearch/security/filter/OpenSearchRequest.java @@ -17,7 +17,6 @@ import java.util.Optional; import javax.net.ssl.SSLEngine; -import org.opensearch.http.netty4.Netty4HttpChannel; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestRequest.Method; @@ -41,21 +40,11 @@ public Map> getHeaders() { @Override public SSLEngine getSSLEngine() { - if (underlyingRequest == null - || underlyingRequest.getHttpChannel() == null - || !(underlyingRequest.getHttpChannel() instanceof Netty4HttpChannel)) { + if (underlyingRequest == null || underlyingRequest.getHttpChannel() == null) { return null; } - // We look for Ssl_handler called `ssl_http` in the outbound pipeline of Netty channel first, and if its not - // present we look for it in inbound channel. If its present in neither we return null, else we return the sslHandler. - final Netty4HttpChannel httpChannel = (Netty4HttpChannel) underlyingRequest.getHttpChannel(); - SslHandler sslhandler = (SslHandler) httpChannel.getNettyChannel().pipeline().get("ssl_http"); - if (sslhandler == null && httpChannel.inboundPipeline() != null) { - sslhandler = (SslHandler) httpChannel.inboundPipeline().get("ssl_http"); - } - - return sslhandler != null ? sslhandler.engine() : null; + return underlyingRequest.getHttpChannel().get("ssl_http", SslHandler.class).map(SslHandler::engine).orElse(null); } @Override diff --git a/src/main/java/org/opensearch/security/ssl/transport/SecuritySSLRequestHandler.java b/src/main/java/org/opensearch/security/ssl/transport/SecuritySSLRequestHandler.java index 39312e29ad..078c822357 100644 --- a/src/main/java/org/opensearch/security/ssl/transport/SecuritySSLRequestHandler.java +++ b/src/main/java/org/opensearch/security/ssl/transport/SecuritySSLRequestHandler.java @@ -36,13 +36,9 @@ import org.opensearch.security.support.ConfigConstants; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; -import org.opensearch.transport.TaskTransportChannel; -import org.opensearch.transport.TcpChannel; -import org.opensearch.transport.TcpTransportChannel; import org.opensearch.transport.TransportChannel; import org.opensearch.transport.TransportRequest; import org.opensearch.transport.TransportRequestHandler; -import org.opensearch.transport.netty4.Netty4TcpChannel; import io.netty.handler.ssl.SslHandler; @@ -111,21 +107,7 @@ public final void messageReceived(T request, TransportChannel channel, Task task } try { - - Netty4TcpChannel nettyChannel = null; - - if (channel instanceof TaskTransportChannel) { - final TransportChannel inner = ((TaskTransportChannel) channel).getChannel(); - nettyChannel = (Netty4TcpChannel) ((TcpTransportChannel) inner).getChannel(); - } else if (channel instanceof TcpTransportChannel) { - final TcpChannel inner = ((TcpTransportChannel) channel).getChannel(); - nettyChannel = (Netty4TcpChannel) inner; - } else { - throw new Exception("Invalid channel of type " + channel.getClass() + " (" + channel.getChannelType() + ")"); - } - - final SslHandler sslhandler = (SslHandler) nettyChannel.getNettyChannel().pipeline().get("ssl_server"); - + final SslHandler sslhandler = channel.get("ssl_server", SslHandler.class).orElse(null); if (sslhandler == null) { if (SSLConfig.isDualModeEnabled()) { log.info("Communication in dual mode. Skipping SSL handler check"); From 03fd79fe910e209cd092c92ad3a2507b8cc20d07 Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Fri, 5 Jan 2024 15:00:26 -0500 Subject: [PATCH 075/204] Add additional ignore_headers audit configuration setting (#3885) Signed-off-by: Stephen Crawford Signed-off-by: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Co-authored-by: Craig Perkins --- .../test/framework/AuditFilters.java | 9 +++++ .../security/OpenSearchSecurityPlugin.java | 11 +++++- .../security/auditlog/config/AuditConfig.java | 38 +++++++++++++++++-- .../auditlog/impl/AbstractAuditLog.java | 5 --- .../security/auditlog/impl/AuditMessage.java | 9 +++-- .../security/support/ConfigConstants.java | 1 + .../config/AuditConfigFilterTest.java | 4 ++ .../config/AuditConfigSerializeTest.java | 7 ++++ .../auditlog/impl/AuditMessageTest.java | 24 +++++++++--- .../dlic/rest/api/AuditApiActionTest.java | 2 +- 10 files changed, 92 insertions(+), 18 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/test/framework/AuditFilters.java b/src/integrationTest/java/org/opensearch/test/framework/AuditFilters.java index f984becefa..087342eb6f 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/AuditFilters.java +++ b/src/integrationTest/java/org/opensearch/test/framework/AuditFilters.java @@ -34,6 +34,8 @@ public class AuditFilters implements ToXContentObject { private List ignoreRequests; + private List ignoreHeaders; + private List disabledRestCategories; private List disabledTransportCategories; @@ -49,6 +51,7 @@ public AuditFilters() { this.ignoreUsers = Collections.emptyList(); this.ignoreRequests = Collections.emptyList(); + this.ignoreHeaders = Collections.emptyList(); this.disabledRestCategories = Collections.emptyList(); this.disabledTransportCategories = Collections.emptyList(); } @@ -93,6 +96,11 @@ public AuditFilters ignoreRequests(List ignoreRequests) { return this; } + public AuditFilters ignoreHeaders(List ignoreHeaders) { + this.ignoreHeaders = ignoreHeaders; + return this; + } + public AuditFilters disabledRestCategories(List disabledRestCategories) { this.disabledRestCategories = disabledRestCategories; return this; @@ -114,6 +122,7 @@ public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params xContentBuilder.field("exclude_sensitive_headers", excludeSensitiveHeaders); xContentBuilder.field("ignore_users", ignoreUsers); xContentBuilder.field("ignore_requests", ignoreRequests); + xContentBuilder.field("ignore_headers", ignoreHeaders); xContentBuilder.field("disabled_rest_categories", disabledRestCategories); xContentBuilder.field("disabled_transport_categories", disabledTransportCategories); xContentBuilder.endObject(); diff --git a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java index 3c04816c32..b0263e06d4 100644 --- a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java +++ b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java @@ -1352,7 +1352,7 @@ public List> getSettings() { Function.identity(), Property.NodeScope ) - ); // not filtered here + ); settings.add( Setting.listSetting( ConfigConstants.OPENDISTRO_SECURITY_AUDIT_IGNORE_REQUESTS, @@ -1361,6 +1361,14 @@ public List> getSettings() { Property.NodeScope ) ); // not filtered here + settings.add( + Setting.listSetting( + ConfigConstants.SECURITY_AUDIT_IGNORE_HEADERS, + Collections.emptyList(), + Function.identity(), + Property.NodeScope + ) + ); settings.add( Setting.boolSetting( ConfigConstants.OPENDISTRO_SECURITY_AUDIT_RESOLVE_BULK_REQUESTS, @@ -1393,6 +1401,7 @@ public List> getSettings() { Property.NodeScope ); case IGNORE_REQUESTS: + case IGNORE_HEADERS: return Setting.listSetting( filterEntry.getKeyWithNamespace(), Collections.emptyList(), diff --git a/src/main/java/org/opensearch/security/auditlog/config/AuditConfig.java b/src/main/java/org/opensearch/security/auditlog/config/AuditConfig.java index 2cffd93dfa..7b173099b5 100644 --- a/src/main/java/org/opensearch/security/auditlog/config/AuditConfig.java +++ b/src/main/java/org/opensearch/security/auditlog/config/AuditConfig.java @@ -62,7 +62,8 @@ * "ignore_users" : [ * "kibanaserver" * ], - * "ignore_requests" : [ ] + * "ignore_requests" : [ ], + * "ignore_headers" : [ ], * }, * "compliance" : { * "enabled": true, @@ -82,6 +83,7 @@ public class AuditConfig { public static final List DEFAULT_IGNORED_USERS = Collections.singletonList("kibanaserver"); + private static Set FIELDS = DefaultObjectMapper.getFields(AuditConfig.class); private AuditConfig() { @@ -138,8 +140,11 @@ public static class Filter { private final Set ignoredAuditUsers; @JsonProperty("ignore_requests") private final Set ignoredAuditRequests; + @JsonProperty("ignore_headers") + private final Set ignoredCustomHeaders; private final WildcardMatcher ignoredAuditUsersMatcher; private final WildcardMatcher ignoredAuditRequestsMatcher; + private final WildcardMatcher ignoredCustomHeadersMatcher; private final Set disabledRestCategories; private final Set disabledTransportCategories; @@ -153,6 +158,7 @@ public static class Filter { final boolean excludeSensitiveHeaders, final Set ignoredAuditUsers, final Set ignoredAuditRequests, + final Set ignoredCustomHeaders, final Set disabledRestCategories, final Set disabledTransportCategories ) { @@ -166,6 +172,8 @@ public static class Filter { this.ignoredAuditUsersMatcher = WildcardMatcher.from(ignoredAuditUsers); this.ignoredAuditRequests = ignoredAuditRequests; this.ignoredAuditRequestsMatcher = WildcardMatcher.from(ignoredAuditRequests); + this.ignoredCustomHeaders = ignoredCustomHeaders; + this.ignoredCustomHeadersMatcher = WildcardMatcher.from(ignoredCustomHeaders); this.disabledRestCategories = disabledRestCategories; this.disabledTransportCategories = disabledTransportCategories; } @@ -183,7 +191,8 @@ public enum FilterEntries { ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_TRANSPORT_CATEGORIES ), IGNORE_USERS("ignore_users", ConfigConstants.OPENDISTRO_SECURITY_AUDIT_IGNORE_USERS), - IGNORE_REQUESTS("ignore_requests", ConfigConstants.OPENDISTRO_SECURITY_AUDIT_IGNORE_REQUESTS); + IGNORE_REQUESTS("ignore_requests", ConfigConstants.OPENDISTRO_SECURITY_AUDIT_IGNORE_REQUESTS), + IGNORE_HEADERS("ignore_headers", ConfigConstants.SECURITY_AUDIT_IGNORE_HEADERS); private final String key; private final String legacyKeyWithNamespace; @@ -246,6 +255,9 @@ public static Filter from(Map properties) throws JsonProcessingE final Set ignoreAuditRequests = ImmutableSet.copyOf( getOrDefault(properties, FilterEntries.IGNORE_REQUESTS.getKey(), Collections.emptyList()) ); + final Set ignoreHeaders = ImmutableSet.copyOf( + getOrDefault(properties, FilterEntries.IGNORE_HEADERS.getKey(), Collections.emptyList()) + ); return new Filter( isRestApiAuditEnabled, @@ -256,6 +268,7 @@ public static Filter from(Map properties) throws JsonProcessingE excludeSensitiveHeaders, ignoredAuditUsers, ignoreAuditRequests, + ignoreHeaders, disabledRestCategories, disabledTransportCategories ); @@ -290,7 +303,7 @@ public static Filter from(Settings settings) { ); final Set ignoredAuditUsers = fromSettingStringSet(settings, FilterEntries.IGNORE_USERS, DEFAULT_IGNORED_USERS); final Set ignoreAuditRequests = fromSettingStringSet(settings, FilterEntries.IGNORE_REQUESTS, Collections.emptyList()); - + final Set ignoreHeaders = fromSettingStringSet(settings, FilterEntries.IGNORE_HEADERS, Collections.emptyList()); return new Filter( isRestApiAuditEnabled, isTransportAuditEnabled, @@ -300,6 +313,7 @@ public static Filter from(Settings settings) { excludeSensitiveHeaders, ignoredAuditUsers, ignoreAuditRequests, + ignoreHeaders, disabledRestCategories, disabledTransportCategories ); @@ -403,6 +417,21 @@ WildcardMatcher getIgnoredAuditRequestsMatcher() { return ignoredAuditRequestsMatcher; } + @VisibleForTesting + WildcardMatcher getIgnoredCustomHeadersMatcher() { + return ignoredCustomHeadersMatcher; + } + + /** + * Check if the specified header is excluded from the audit + * + * @param header + * @return true if header should be excluded + */ + public boolean shouldExcludeHeader(String header) { + return ignoredCustomHeadersMatcher.test(header); + } + /** * Check if request is excluded from audit * @param action @@ -440,6 +469,7 @@ public void log(Logger logger) { logger.info("Index resolution is {} during request auditing.", resolveIndices ? "enabled" : "disabled"); logger.info("Sensitive headers auditing is {}.", excludeSensitiveHeaders ? "enabled" : "disabled"); logger.info("Auditing requests from {} users is disabled.", ignoredAuditUsersMatcher); + logger.info("Auditing request headers {} is disabled.", ignoredCustomHeadersMatcher); } @Override @@ -465,6 +495,8 @@ public String toString() { + ignoredAuditUsersMatcher + ", ignoreAuditRequests=" + ignoredAuditRequestsMatcher + + ", ignoredCustomHeaders=" + + ignoredCustomHeadersMatcher + '}'; } } diff --git a/src/main/java/org/opensearch/security/auditlog/impl/AbstractAuditLog.java b/src/main/java/org/opensearch/security/auditlog/impl/AbstractAuditLog.java index d97adc358b..e5f314cd29 100644 --- a/src/main/java/org/opensearch/security/auditlog/impl/AbstractAuditLog.java +++ b/src/main/java/org/opensearch/security/auditlog/impl/AbstractAuditLog.java @@ -927,11 +927,6 @@ boolean checkRestFilter(final AuditCategory category, final String effectiveUser } return false; } - - // check rest audit enabled - // check category enabled - // check action - // check ignoreAuditUsers } protected abstract void save(final AuditMessage msg); diff --git a/src/main/java/org/opensearch/security/auditlog/impl/AuditMessage.java b/src/main/java/org/opensearch/security/auditlog/impl/AuditMessage.java index 8b24a554d1..b57becc359 100644 --- a/src/main/java/org/opensearch/security/auditlog/impl/AuditMessage.java +++ b/src/main/java/org/opensearch/security/auditlog/impl/AuditMessage.java @@ -356,12 +356,15 @@ public void addRestParams(Map params) { } } - public void addRestHeaders(Map> headers, boolean excludeSensitiveHeaders) { + public void addRestHeaders(Map> headers, boolean excludeSensitiveHeaders, AuditConfig.Filter filter) { if (headers != null && !headers.isEmpty()) { final Map> headersClone = new HashMap<>(headers); if (excludeSensitiveHeaders) { headersClone.keySet().removeIf(AUTHORIZATION_HEADER); } + if (filter != null) { + headersClone.entrySet().removeIf(entry -> filter.shouldExcludeHeader(entry.getKey())); + } auditInfo.put(REST_REQUEST_HEADERS, headersClone); } } @@ -376,14 +379,14 @@ void addRestRequestInfo(final SecurityRequest request, final AuditConfig.Filter if (request != null) { final String path = request.path().toString(); addPath(path); - addRestHeaders(request.getHeaders(), filter.shouldExcludeSensitiveHeaders()); + addRestHeaders(request.getHeaders(), filter.shouldExcludeSensitiveHeaders(), filter); addRestParams(request.params()); addRestMethod(request.method()); if (filter.shouldLogRequestBody()) { if (!(request instanceof OpenSearchRequest)) { - // The request body is only avaliable on some request sources + // The request body is only available on some request sources return; } diff --git a/src/main/java/org/opensearch/security/support/ConfigConstants.java b/src/main/java/org/opensearch/security/support/ConfigConstants.java index f10dedade3..d4383c05de 100644 --- a/src/main/java/org/opensearch/security/support/ConfigConstants.java +++ b/src/main/java/org/opensearch/security/support/ConfigConstants.java @@ -165,6 +165,7 @@ public class ConfigConstants { ); public static final String OPENDISTRO_SECURITY_AUDIT_IGNORE_USERS = "opendistro_security.audit.ignore_users"; public static final String OPENDISTRO_SECURITY_AUDIT_IGNORE_REQUESTS = "opendistro_security.audit.ignore_requests"; + public static final String SECURITY_AUDIT_IGNORE_HEADERS = "plugins.security.audit.ignore_headers"; public static final String OPENDISTRO_SECURITY_AUDIT_RESOLVE_BULK_REQUESTS = "opendistro_security.audit.resolve_bulk_requests"; public static final boolean OPENDISTRO_SECURITY_AUDIT_SSL_VERIFY_HOSTNAMES_DEFAULT = true; public static final boolean OPENDISTRO_SECURITY_AUDIT_SSL_ENABLE_SSL_CLIENT_AUTH_DEFAULT = false; diff --git a/src/test/java/org/opensearch/security/auditlog/config/AuditConfigFilterTest.java b/src/test/java/org/opensearch/security/auditlog/config/AuditConfigFilterTest.java index e40e65549f..a28d940862 100644 --- a/src/test/java/org/opensearch/security/auditlog/config/AuditConfigFilterTest.java +++ b/src/test/java/org/opensearch/security/auditlog/config/AuditConfigFilterTest.java @@ -57,6 +57,7 @@ public void testDefault() { assertTrue(auditConfigFilter.shouldExcludeSensitiveHeaders()); assertSame(WildcardMatcher.NONE, auditConfigFilter.getIgnoredAuditRequestsMatcher()); assertEquals(defaultIgnoredUserMatcher, auditConfigFilter.getIgnoredAuditUsersMatcher()); + assertSame(WildcardMatcher.NONE, auditConfigFilter.getIgnoredCustomHeadersMatcher()); assertEquals(auditConfigFilter.getDisabledRestCategories(), defaultDisabledCategories); assertEquals(auditConfigFilter.getDisabledTransportCategories(), defaultDisabledCategories); } @@ -73,6 +74,7 @@ public void testConfig() { .put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_EXCLUDE_SENSITIVE_HEADERS, false) .putList(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_IGNORE_REQUESTS, "test-request") .putList(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_IGNORE_USERS, "test-user") + .putList(ConfigConstants.SECURITY_AUDIT_IGNORE_HEADERS, "test-header") .putList( ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_REST_CATEGORIES, BAD_HEADERS.toString(), @@ -95,6 +97,7 @@ public void testConfig() { assertFalse(auditConfigFilter.shouldExcludeSensitiveHeaders()); assertEquals(WildcardMatcher.from(Collections.singleton("test-user")), auditConfigFilter.getIgnoredAuditUsersMatcher()); assertEquals(WildcardMatcher.from(Collections.singleton("test-request")), auditConfigFilter.getIgnoredAuditRequestsMatcher()); + assertEquals(WildcardMatcher.from(Collections.singleton("test-header")), auditConfigFilter.getIgnoredCustomHeadersMatcher()); assertEquals(auditConfigFilter.getDisabledRestCategories(), EnumSet.of(BAD_HEADERS, SSL_EXCEPTION)); assertEquals(auditConfigFilter.getDisabledTransportCategories(), EnumSet.of(FAILED_LOGIN, MISSING_PRIVILEGES)); } @@ -121,6 +124,7 @@ public void testEmpty() { final Settings settings = Settings.builder() .putList(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_IGNORE_USERS, Collections.emptyList()) .putList(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_IGNORE_REQUESTS, Collections.emptyList()) + .putList(ConfigConstants.SECURITY_AUDIT_IGNORE_HEADERS, Collections.emptyList()) .putList(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_REST_CATEGORIES, Collections.emptyList()) .putList(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_TRANSPORT_CATEGORIES, Collections.emptyList()) .build(); diff --git a/src/test/java/org/opensearch/security/auditlog/config/AuditConfigSerializeTest.java b/src/test/java/org/opensearch/security/auditlog/config/AuditConfigSerializeTest.java index 0b50c2ac20..b0b93afc54 100644 --- a/src/test/java/org/opensearch/security/auditlog/config/AuditConfigSerializeTest.java +++ b/src/test/java/org/opensearch/security/auditlog/config/AuditConfigSerializeTest.java @@ -72,6 +72,7 @@ public void testDefaultSerialize() throws IOException { .field("exclude_sensitive_headers", true) .field("ignore_users", Collections.singletonList("kibanaserver")) .field("ignore_requests", Collections.emptyList()) + .field("ignore_headers", Collections.emptyList()) .endObject() .startObject("compliance") .field("enabled", true) @@ -107,6 +108,7 @@ public void testDefaultDeserialize() throws IOException { assertTrue(audit.shouldExcludeSensitiveHeaders()); assertSame(WildcardMatcher.NONE, audit.getIgnoredAuditRequestsMatcher()); assertEquals(DEFAULT_IGNORED_USER, audit.getIgnoredAuditUsersMatcher()); + assertEquals(WildcardMatcher.NONE, audit.getIgnoredCustomHeadersMatcher()); assertFalse(compliance.shouldLogExternalConfig()); assertFalse(compliance.shouldLogInternalConfig()); assertFalse(compliance.shouldLogReadMetadataOnly()); @@ -133,6 +135,7 @@ public void testDeserialize() throws IOException { .field("exclude_sensitive_headers", true) .field("ignore_users", Collections.singletonList("test-user-1")) .field("ignore_requests", Collections.singletonList("test-request")) + .field("ignore_headers", Collections.singletonList("test-headers")) .endObject() .startObject("compliance") .field("enabled", true) @@ -196,6 +199,7 @@ public void testSerialize() throws IOException { true, ImmutableSet.of("ignore-user-1", "ignore-user-2"), ImmutableSet.of("ignore-request-1"), + ImmutableSet.of("test-header"), EnumSet.of(AuditCategory.FAILED_LOGIN, AuditCategory.GRANTED_PRIVILEGES), EnumSet.of(AUTHENTICATED) ); @@ -227,6 +231,7 @@ public void testSerialize() throws IOException { .field("exclude_sensitive_headers", true) .field("ignore_users", ImmutableList.of("ignore-user-1", "ignore-user-2")) .field("ignore_requests", Collections.singletonList("ignore-request-1")) + .field("ignore_headers", Collections.singletonList("test-header")) .endObject() .startObject("compliance") .field("enabled", true) @@ -269,6 +274,7 @@ public void testNullSerialize() throws IOException { .field("exclude_sensitive_headers", true) .field("ignore_users", ImmutableList.of("kibanaserver")) .field("ignore_requests", Collections.emptyList()) + .field("ignore_headers", Collections.emptyList()) .endObject() .startObject("compliance") .field("enabled", true) @@ -287,6 +293,7 @@ public void testNullSerialize() throws IOException { // act final String json = objectMapper.writeValueAsString(auditConfig); // assert + assertTrue(compareJson(jsonBuilder.toString(), json)); } diff --git a/src/test/java/org/opensearch/security/auditlog/impl/AuditMessageTest.java b/src/test/java/org/opensearch/security/auditlog/impl/AuditMessageTest.java index d915c02e55..3b7fc916ef 100644 --- a/src/test/java/org/opensearch/security/auditlog/impl/AuditMessageTest.java +++ b/src/test/java/org/opensearch/security/auditlog/impl/AuditMessageTest.java @@ -28,6 +28,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.security.auditlog.AuditLog; +import org.opensearch.security.auditlog.config.AuditConfig; import org.opensearch.security.securityconf.impl.CType; import static org.junit.Assert.assertEquals; @@ -60,32 +61,45 @@ public class AuditMessageTest { ); private AuditMessage message; + private AuditConfig auditConfig; @Before public void setUp() { final ClusterService clusterServiceMock = mock(ClusterService.class); when(clusterServiceMock.localNode()).thenReturn(mock(DiscoveryNode.class)); when(clusterServiceMock.getClusterName()).thenReturn(mock(ClusterName.class)); + auditConfig = mock(AuditConfig.class); + final AuditConfig.Filter auditFilter = mock(AuditConfig.Filter.class); + when(auditConfig.getFilter()).thenReturn(auditFilter); message = new AuditMessage(AuditCategory.AUTHENTICATED, clusterServiceMock, AuditLog.Origin.REST, AuditLog.Origin.REST); } @Test - public void testRestHeadersAreFiltered() { - message.addRestHeaders(TEST_REST_HEADERS, true); + public void testAuthorizationRestHeadersAreFiltered() { + when(auditConfig.getFilter().shouldExcludeHeader("test-header")).thenReturn(false); + message.addRestHeaders(TEST_REST_HEADERS, true, auditConfig.getFilter()); assertEquals(message.getAsMap().get(AuditMessage.REST_REQUEST_HEADERS), ImmutableMap.of("test-header", ImmutableList.of("test-4"))); } + @Test + public void testCustomRestHeadersAreFiltered() { + when(auditConfig.getFilter().shouldExcludeHeader("test-header")).thenReturn(true); + message.addRestHeaders(TEST_REST_HEADERS, true, auditConfig.getFilter()); + assertEquals(message.getAsMap().get(AuditMessage.REST_REQUEST_HEADERS), Map.of()); + } + @Test public void testRestHeadersNull() { - message.addRestHeaders(null, true); + message.addRestHeaders(null, true, null); assertNull(message.getAsMap().get(AuditMessage.REST_REQUEST_HEADERS)); - message.addRestHeaders(Collections.emptyMap(), true); + message.addRestHeaders(Collections.emptyMap(), true, null); assertNull(message.getAsMap().get(AuditMessage.REST_REQUEST_HEADERS)); } @Test public void testRestHeadersAreNotFiltered() { - message.addRestHeaders(TEST_REST_HEADERS, false); + when(auditConfig.getFilter().shouldExcludeHeader("test-header")).thenReturn(false); + message.addRestHeaders(TEST_REST_HEADERS, false, null); assertEquals(message.getAsMap().get(AuditMessage.REST_REQUEST_HEADERS), TEST_REST_HEADERS); } diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/AuditApiActionTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/AuditApiActionTest.java index b512ae2228..b3d916e8ed 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/AuditApiActionTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/AuditApiActionTest.java @@ -682,7 +682,7 @@ private String getTestPayload() { + "\"enable_rest\":true,\"disabled_rest_categories\":[\"AUTHENTICATED\"]," + "\"enable_transport\":true,\"disabled_transport_categories\":[\"SSL_EXCEPTION\"]," + "\"resolve_bulk_requests\":true,\"log_request_body\":true,\"resolve_indices\":true,\"exclude_sensitive_headers\":true," - + "\"ignore_users\":[\"test-user-1\"],\"ignore_requests\":[\"test-request\"]}," + + "\"ignore_users\":[\"test-user-1\"],\"ignore_requests\":[\"test-request\"], \"ignore_headers\":[\"\"]}," + "\"compliance\":{" + "\"enabled\":true," + "\"internal_config\":true,\"external_config\":true," From 045d4ef8a5d9bb8470e03c1b4d7cc68847b986cb Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Fri, 5 Jan 2024 15:16:54 -0500 Subject: [PATCH 076/204] Allow TransportConfigUpdateAction when security config initialization has completed (#3810) ### Description Introduces another variable on DynamicConfigFactory called `bgThreadComplete` that behaves differently than the `initialized` variable. `bgThreadComplete` is a flag that signals to TransportConfigUpdateAction that it can start accepting updates. There are 2 ways the security index can be created from scratch: 1. If `plugins.security.allow_default_init_securityindex` is set to **true** it will create the security index and load all yaml files 2. If `plugins.security.allow_default_init_securityindex` is set to **false**, the security index is not created on bootstrap and requires a user to run securityadmin to initialize security. When securityadmin is utilized, the cluster does depend on [TransportConfigUpdateAction](https://github.com/opensearch-project/security/blob/main/src/main/java/org/opensearch/security/tools/SecurityAdmin.java#L975-L977) to initialize security so there still needs to be an avenue where this can update the config before `initialized` is set to **true** This PR sets `bgThreadComplete` to **false** on node startup and explicitly sets it to **true** once its ok for TransportConfigUpdateAction to start accepting transport actions. In case 2) up above, it can be set to **true** before DynamicConfigFactory is `initialized` so that it can accept requests from securityadmin. * Category (Enhancement, New feature, Bug fix, Test fix, Refactoring, Maintenance, Documentation) Bug fix ### Issues Resolved - Resolves https://github.com/opensearch-project/security/issues/3204 ### Check List - [X] New functionality includes testing - [ ] New functionality has been documented - [X] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: Craig Perkins Signed-off-by: Peter Nied Signed-off-by: Peter Nied Co-authored-by: Peter Nied Co-authored-by: Peter Nied --- .../security/ConfigurationFiles.java | 5 +- .../security/SecurityAdminLauncher.java | 18 + .../SecurityConfigurationBootstrapTests.java | 159 +++++++++ .../security/SecurityConfigurationTests.java | 2 +- .../security/api/CreateResetPasswordTest.java | 2 +- .../http/OnBehalfOfJwtAuthenticationTest.java | 6 +- .../test/framework/cluster/LocalCluster.java | 14 +- .../security/OpenSearchSecurityPlugin.java | 8 + .../TransportConfigUpdateAction.java | 6 +- .../ConfigurationRepository.java | 320 ++++++++++-------- .../securityconf/DynamicConfigFactory.java | 1 - .../security/support/ConfigConstants.java | 2 + .../InitializationIntegrationTests.java | 6 - .../ConfigurationRepositoryTest.java | 8 +- .../security/test/SingleClusterTest.java | 12 - 15 files changed, 396 insertions(+), 173 deletions(-) create mode 100644 src/integrationTest/java/org/opensearch/security/SecurityConfigurationBootstrapTests.java diff --git a/src/integrationTest/java/org/opensearch/security/ConfigurationFiles.java b/src/integrationTest/java/org/opensearch/security/ConfigurationFiles.java index 287bc139b1..f3b7613aa1 100644 --- a/src/integrationTest/java/org/opensearch/security/ConfigurationFiles.java +++ b/src/integrationTest/java/org/opensearch/security/ConfigurationFiles.java @@ -31,12 +31,13 @@ public static Path createConfigurationDirectory() { String[] configurationFiles = { "config.yml", "action_groups.yml", - "config.yml", "internal_users.yml", + "nodes_dn.yml", "roles.yml", "roles_mapping.yml", "security_tenants.yml", - "tenants.yml" }; + "tenants.yml", + "whitelist.yml" }; for (String fileName : configurationFiles) { Path configFileDestination = tempDirectory.resolve(fileName); copyResourceToFile(fileName, configFileDestination.toFile()); diff --git a/src/integrationTest/java/org/opensearch/security/SecurityAdminLauncher.java b/src/integrationTest/java/org/opensearch/security/SecurityAdminLauncher.java index 164b2cb714..81460d3d91 100644 --- a/src/integrationTest/java/org/opensearch/security/SecurityAdminLauncher.java +++ b/src/integrationTest/java/org/opensearch/security/SecurityAdminLauncher.java @@ -10,6 +10,7 @@ package org.opensearch.security; import java.io.File; +import java.nio.file.Path; import org.opensearch.security.tools.SecurityAdmin; import org.opensearch.test.framework.certificate.TestCertificates; @@ -44,4 +45,21 @@ public int updateRoleMappings(File roleMappingsConfigurationFile) throws Excepti return SecurityAdmin.execute(commandLineArguments); } + + public int runSecurityAdmin(Path configurationFolder) throws Exception { + String[] commandLineArguments = { + "-cacert", + certificates.getRootCertificate().getAbsolutePath(), + "-cert", + certificates.getAdminCertificate().getAbsolutePath(), + "-key", + certificates.getAdminKey(null).getAbsolutePath(), + "-nhnv", + "-p", + String.valueOf(port), + "-cd", + configurationFolder.toString() }; + + return SecurityAdmin.execute(commandLineArguments); + } } diff --git a/src/integrationTest/java/org/opensearch/security/SecurityConfigurationBootstrapTests.java b/src/integrationTest/java/org/opensearch/security/SecurityConfigurationBootstrapTests.java new file mode 100644 index 0000000000..5b83e0d6d0 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/SecurityConfigurationBootstrapTests.java @@ -0,0 +1,159 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.security; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.Map; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import com.google.common.collect.ImmutableMap; +import org.apache.commons.io.FileUtils; +import org.awaitility.Awaitility; +import org.junit.AfterClass; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.security.securityconf.impl.CType; +import org.opensearch.security.support.ConfigConstants; +import org.opensearch.security.support.ConfigHelper; +import org.opensearch.test.framework.TestSecurityConfig.User; +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.ContextHeaderDecoratorClient; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; + +import static org.apache.http.HttpStatus.SC_SERVICE_UNAVAILABLE; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.security.configuration.ConfigurationRepository.DEFAULT_CONFIG_VERSION; +import static org.opensearch.security.support.ConfigConstants.OPENDISTRO_SECURITY_DEFAULT_CONFIG_INDEX; +import static org.opensearch.security.support.ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX; +import static org.opensearch.security.support.ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST; +import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_ROLES_ENABLED; +import static org.opensearch.security.support.ConfigConstants.SECURITY_UNSUPPORTED_DELAY_INITIALIZATION_SECONDS; +import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS; + +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class SecurityConfigurationBootstrapTests { + + private final static Path configurationFolder = ConfigurationFiles.createConfigurationDirectory(); + private static final User USER_ADMIN = new User("admin").roles(ALL_ACCESS); + + private static LocalCluster createCluster(final Map nodeSettings) { + var cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS) + .loadConfigurationIntoIndex(false) + .defaultConfigurationInitDirectory(configurationFolder.toString()) + .nodeSettings( + ImmutableMap.builder() + .put(SECURITY_RESTAPI_ROLES_ENABLED, List.of("user_" + USER_ADMIN.getName() + "__" + ALL_ACCESS.getName())) + .putAll(nodeSettings) + .build() + ) + .build(); + + cluster.before(); // normally invoked by JUnit rules when run as a class rule - this starts the cluster + return cluster; + } + + @AfterClass + public static void cleanConfigurationDirectory() throws IOException { + FileUtils.deleteDirectory(configurationFolder.toFile()); + } + + @Test + public void testInitializeWithSecurityAdminWhenNoBackgroundInitialization() throws Exception { + final var nodeSettings = ImmutableMap.builder() + .put(SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, false) + .put(SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, false) + .build(); + try (final LocalCluster cluster = createCluster(nodeSettings)) { + try (final TestRestClient client = cluster.getRestClient(USER_ADMIN)) { + final var rolesMapsResponse = client.get("_plugins/_security/api/rolesmapping/readall"); + assertThat(rolesMapsResponse.getStatusCode(), equalTo(SC_SERVICE_UNAVAILABLE)); + assertThat(rolesMapsResponse.getBody(), containsString("OpenSearch Security not initialized")); + } + + final var securityAdminLauncher = new SecurityAdminLauncher(cluster.getHttpPort(), cluster.getTestCertificates()); + final int exitCode = securityAdminLauncher.runSecurityAdmin(configurationFolder); + assertThat(exitCode, equalTo(0)); + + try (final TestRestClient client = cluster.getRestClient(USER_ADMIN)) { + Awaitility.await() + .alias("Waiting for rolemapping 'readall' availability.") + .until(() -> client.get("_plugins/_security/api/rolesmapping/readall").getStatusCode(), equalTo(200)); + } + } + } + + @Test + public void shouldStillLoadSecurityConfigDuringBootstrapAndActiveConfigUpdateRequests() throws Exception { + final var nodeSettings = ImmutableMap.builder() + .put(SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, true) + .put(SECURITY_UNSUPPORTED_DELAY_INITIALIZATION_SECONDS, 5) + .build(); + try (final LocalCluster cluster = createCluster(nodeSettings)) { + try (final TestRestClient client = cluster.getRestClient(USER_ADMIN)) { + cluster.getInternalNodeClient() + .admin() + .cluster() + .health(new ClusterHealthRequest(OPENDISTRO_SECURITY_DEFAULT_CONFIG_INDEX).waitForGreenStatus()) + .actionGet(); + + // Make sure the cluster is unavalaible to authenticate with the security plugin even though it is green + final var authResponseWhenUnconfigured = client.getAuthInfo(); + authResponseWhenUnconfigured.assertStatusCode(503); + + final var internalNodeClient = new ContextHeaderDecoratorClient( + cluster.getInternalNodeClient(), + Map.of(ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER, "true") + ); + final var filesToUpload = ImmutableMap.builder() + .put("action_groups.yml", CType.ACTIONGROUPS) + .put("config.yml", CType.CONFIG) + .put("roles.yml", CType.ROLES) + .put("tenants.yml", CType.TENANTS) + .build(); + + final String defaultInitDirectory = System.getProperty("security.default_init.dir") + "/"; + filesToUpload.forEach((fileName, ctype) -> { + try { + ConfigHelper.uploadFile( + internalNodeClient, + defaultInitDirectory + fileName, + OPENDISTRO_SECURITY_DEFAULT_CONFIG_INDEX, + ctype, + DEFAULT_CONFIG_VERSION + ); + } catch (final Exception ex) { + throw new RuntimeException(ex); + } + }); + + Awaitility.await().alias("Load default configuration").pollInterval(Duration.ofMillis(100)).until(() -> { + // After the configuration has been loaded, the rest clients should be able to connect successfully + cluster.triggerConfigurationReloadForCTypes( + internalNodeClient, + List.of(CType.ACTIONGROUPS, CType.CONFIG, CType.ROLES, CType.TENANTS), + true + ); + try (final TestRestClient freshClient = cluster.getRestClient(USER_ADMIN)) { + return client.getAuthInfo().getStatusCode(); + } + }, equalTo(200)); + } + } + } +} diff --git a/src/integrationTest/java/org/opensearch/security/SecurityConfigurationTests.java b/src/integrationTest/java/org/opensearch/security/SecurityConfigurationTests.java index 76ea02494e..6b04737d18 100644 --- a/src/integrationTest/java/org/opensearch/security/SecurityConfigurationTests.java +++ b/src/integrationTest/java/org/opensearch/security/SecurityConfigurationTests.java @@ -70,7 +70,7 @@ public class SecurityConfigurationTests { SECURITY_RESTAPI_ROLES_ENABLED, List.of("user_" + USER_ADMIN.getName() + "__" + ALL_ACCESS.getName()), SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, - true + false ) ) .build(); diff --git a/src/integrationTest/java/org/opensearch/security/api/CreateResetPasswordTest.java b/src/integrationTest/java/org/opensearch/security/api/CreateResetPasswordTest.java index 44f8dca20b..8a7795e90f 100644 --- a/src/integrationTest/java/org/opensearch/security/api/CreateResetPasswordTest.java +++ b/src/integrationTest/java/org/opensearch/security/api/CreateResetPasswordTest.java @@ -63,7 +63,7 @@ public class CreateResetPasswordTest { SECURITY_RESTAPI_ROLES_ENABLED, List.of("user_" + USER_ADMIN.getName() + "__" + ALL_ACCESS.getName()), SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, - true, + false, ConfigConstants.SECURITY_RESTAPI_PASSWORD_VALIDATION_REGEX, CUSTOM_PASSWORD_REGEX, ConfigConstants.SECURITY_RESTAPI_PASSWORD_VALIDATION_ERROR_MESSAGE, diff --git a/src/integrationTest/java/org/opensearch/security/http/OnBehalfOfJwtAuthenticationTest.java b/src/integrationTest/java/org/opensearch/security/http/OnBehalfOfJwtAuthenticationTest.java index 78af7ffc05..7210c53ad4 100644 --- a/src/integrationTest/java/org/opensearch/security/http/OnBehalfOfJwtAuthenticationTest.java +++ b/src/integrationTest/java/org/opensearch/security/http/OnBehalfOfJwtAuthenticationTest.java @@ -48,7 +48,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; -import static org.opensearch.security.support.ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX; +import static org.opensearch.security.support.ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST; import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_ADMIN_ENABLED; import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_ROLES_ENABLED; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; @@ -128,8 +128,8 @@ private static OnBehalfOfConfig defaultOnBehalfOfConfig() { .users(ADMIN_USER, OBO_USER, OBO_USER_NO_PERM, HOST_MAPPING_OBO_USER) .nodeSettings( Map.of( - SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, - true, + SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, + false, SECURITY_RESTAPI_ROLES_ENABLED, ADMIN_USER.getRoleNames(), SECURITY_RESTAPI_ADMIN_ENABLED, diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index 64207ead5b..217ce99a81 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -133,7 +133,7 @@ public String getSnapshotDirPath() { } @Override - public void before() throws Throwable { + public void before() { if (localOpenSearchCluster == null) { for (LocalCluster dependency : clusterDependencies) { if (!dependency.isStarted()) { @@ -155,12 +155,12 @@ public void before() throws Throwable { @Override protected void after() { - System.clearProperty(INIT_CONFIGURATION_DIR); close(); } @Override public void close() { + System.clearProperty(INIT_CONFIGURATION_DIR); if (localOpenSearchCluster != null && localOpenSearchCluster.isStarted()) { try { localOpenSearchCluster.destroy(); @@ -297,6 +297,16 @@ private static void triggerConfigurationReload(Client client) { } } + public void triggerConfigurationReloadForCTypes(Client client, List cTypes, boolean ignoreFailures) { + ConfigUpdateResponse configUpdateResponse = client.execute( + ConfigUpdateAction.INSTANCE, + new ConfigUpdateRequest(cTypes.stream().map(CType::toLCString).toArray(String[]::new)) + ).actionGet(); + if (!ignoreFailures && configUpdateResponse.hasFailures()) { + throw new RuntimeException("ConfigUpdateResponse produced failures: " + configUpdateResponse.failures()); + } + } + public CertificateData getAdminCertificate() { return testCertificates.getAdminCertificateData(); } diff --git a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java index b0263e06d4..569380582b 100644 --- a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java +++ b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java @@ -1793,6 +1793,14 @@ public List> getSettings() { Property.Filtered ) ); + settings.add( + Setting.intSetting( + ConfigConstants.SECURITY_UNSUPPORTED_DELAY_INITIALIZATION_SECONDS, + 0, + Property.NodeScope, + Property.Filtered + ) + ); // system integration settings.add( diff --git a/src/main/java/org/opensearch/security/action/configupdate/TransportConfigUpdateAction.java b/src/main/java/org/opensearch/security/action/configupdate/TransportConfigUpdateAction.java index 64149a7c97..6f1f99a434 100644 --- a/src/main/java/org/opensearch/security/action/configupdate/TransportConfigUpdateAction.java +++ b/src/main/java/org/opensearch/security/action/configupdate/TransportConfigUpdateAction.java @@ -125,8 +125,10 @@ protected ConfigUpdateResponse newResponse( @Override protected ConfigUpdateNodeResponse nodeOperation(final NodeConfigUpdateRequest request) { - configurationRepository.reloadConfiguration(CType.fromStringValues((request.request.getConfigTypes()))); - backendRegistry.get().invalidateCache(); + boolean didReload = configurationRepository.reloadConfiguration(CType.fromStringValues((request.request.getConfigTypes()))); + if (didReload) { + backendRegistry.get().invalidateCache(); + } return new ConfigUpdateNodeResponse(clusterService.localNode(), request.request.getConfigTypes(), null); } diff --git a/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java b/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java index e7f375bef4..dfbeb16cb3 100644 --- a/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java +++ b/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java @@ -28,6 +28,8 @@ import java.io.File; import java.nio.file.Path; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; @@ -37,10 +39,11 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Supplier; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -86,13 +89,13 @@ public class ConfigurationRepository { private final List configurationChangedListener; private final ConfigurationLoaderSecurity7 cl; private final Settings settings; + private final Path configPath; private final ClusterService clusterService; private final AuditLog auditLog; private final ThreadPool threadPool; private DynamicConfigFactory dynamicConfigFactory; - private static final int DEFAULT_CONFIG_VERSION = 2; - private final Thread bgThread; - private final AtomicBoolean installDefaultConfig = new AtomicBoolean(); + public static final int DEFAULT_CONFIG_VERSION = 2; + private final CompletableFuture initalizeConfigTask = new CompletableFuture<>(); private final boolean acceptInvalid; private ConfigurationRepository( @@ -108,6 +111,7 @@ private ConfigurationRepository( ConfigConstants.OPENDISTRO_SECURITY_DEFAULT_CONFIG_INDEX ); this.settings = settings; + this.configPath = configPath; this.client = client; this.threadPool = threadPool; this.clusterService = clusterService; @@ -117,148 +121,150 @@ private ConfigurationRepository( cl = new ConfigurationLoaderSecurity7(client, threadPool, settings, clusterService); configCache = CacheBuilder.newBuilder().build(); + } - bgThread = new Thread(() -> { - try { - LOGGER.info("Background init thread started. Install default config?: " + installDefaultConfig.get()); - // wait for the cluster here until it will finish managed node election - while (clusterService.state().blocks().hasGlobalBlockWithStatus(RestStatus.SERVICE_UNAVAILABLE)) { - LOGGER.info("Wait for cluster to be available ..."); - TimeUnit.SECONDS.sleep(1); - } + private void initalizeClusterConfiguration(final boolean installDefaultConfig) { + try { + LOGGER.info("Background init thread started. Install default config?: " + installDefaultConfig); + // wait for the cluster here until it will finish managed node election + while (clusterService.state().blocks().hasGlobalBlockWithStatus(RestStatus.SERVICE_UNAVAILABLE)) { + LOGGER.info("Wait for cluster to be available ..."); + TimeUnit.SECONDS.sleep(1); + } - if (installDefaultConfig.get()) { + if (installDefaultConfig) { - try { - String lookupDir = System.getProperty("security.default_init.dir"); - final String cd = lookupDir != null - ? (lookupDir + "/") - : new Environment(settings, configPath).configDir().toAbsolutePath().toString() + "/opensearch-security/"; - File confFile = new File(cd + "config.yml"); - if (confFile.exists()) { - final ThreadContext threadContext = threadPool.getThreadContext(); - try (StoredContext ctx = threadContext.stashContext()) { - threadContext.putHeader(ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER, "true"); - - createSecurityIndexIfAbsent(); - waitForSecurityIndexToBeAtLeastYellow(); - - ConfigHelper.uploadFile(client, cd + "config.yml", securityIndex, CType.CONFIG, DEFAULT_CONFIG_VERSION); - ConfigHelper.uploadFile(client, cd + "roles.yml", securityIndex, CType.ROLES, DEFAULT_CONFIG_VERSION); - ConfigHelper.uploadFile( - client, - cd + "roles_mapping.yml", - securityIndex, - CType.ROLESMAPPING, - DEFAULT_CONFIG_VERSION - ); - ConfigHelper.uploadFile( - client, - cd + "internal_users.yml", - securityIndex, - CType.INTERNALUSERS, - DEFAULT_CONFIG_VERSION - ); - ConfigHelper.uploadFile( - client, - cd + "action_groups.yml", - securityIndex, - CType.ACTIONGROUPS, - DEFAULT_CONFIG_VERSION - ); - if (DEFAULT_CONFIG_VERSION == 2) { - ConfigHelper.uploadFile( - client, - cd + "tenants.yml", - securityIndex, - CType.TENANTS, - DEFAULT_CONFIG_VERSION - ); - } - final boolean populateEmptyIfFileMissing = true; - ConfigHelper.uploadFile( - client, - cd + "nodes_dn.yml", - securityIndex, - CType.NODESDN, - DEFAULT_CONFIG_VERSION, - populateEmptyIfFileMissing - ); - ConfigHelper.uploadFile( - client, - cd + "whitelist.yml", - securityIndex, - CType.WHITELIST, - DEFAULT_CONFIG_VERSION, - populateEmptyIfFileMissing - ); - ConfigHelper.uploadFile( - client, - cd + "allowlist.yml", - securityIndex, - CType.ALLOWLIST, - DEFAULT_CONFIG_VERSION, - populateEmptyIfFileMissing - ); - - // audit.yml is not packaged by default - final String auditConfigPath = cd + "audit.yml"; - if (new File(auditConfigPath).exists()) { - ConfigHelper.uploadFile(client, auditConfigPath, securityIndex, CType.AUDIT, DEFAULT_CONFIG_VERSION); - } + try { + String lookupDir = System.getProperty("security.default_init.dir"); + final String cd = lookupDir != null + ? (lookupDir + "/") + : new Environment(settings, configPath).configDir().toAbsolutePath().toString() + "/opensearch-security/"; + File confFile = new File(cd + "config.yml"); + if (confFile.exists()) { + final ThreadContext threadContext = threadPool.getThreadContext(); + try (StoredContext ctx = threadContext.stashContext()) { + threadContext.putHeader(ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER, "true"); + + createSecurityIndexIfAbsent(); + waitForSecurityIndexToBeAtLeastYellow(); + + final int initializationDelaySeconds = settings.getAsInt( + ConfigConstants.SECURITY_UNSUPPORTED_DELAY_INITIALIZATION_SECONDS, + 0 + ); + if (initializationDelaySeconds > 0) { + LOGGER.error("Test setting loaded to delay initialization for {} seconds", initializationDelaySeconds); + TimeUnit.SECONDS.sleep(initializationDelaySeconds); + } + + ConfigHelper.uploadFile(client, cd + "config.yml", securityIndex, CType.CONFIG, DEFAULT_CONFIG_VERSION); + ConfigHelper.uploadFile(client, cd + "roles.yml", securityIndex, CType.ROLES, DEFAULT_CONFIG_VERSION); + ConfigHelper.uploadFile( + client, + cd + "roles_mapping.yml", + securityIndex, + CType.ROLESMAPPING, + DEFAULT_CONFIG_VERSION + ); + ConfigHelper.uploadFile( + client, + cd + "internal_users.yml", + securityIndex, + CType.INTERNALUSERS, + DEFAULT_CONFIG_VERSION + ); + ConfigHelper.uploadFile( + client, + cd + "action_groups.yml", + securityIndex, + CType.ACTIONGROUPS, + DEFAULT_CONFIG_VERSION + ); + if (DEFAULT_CONFIG_VERSION == 2) { + ConfigHelper.uploadFile(client, cd + "tenants.yml", securityIndex, CType.TENANTS, DEFAULT_CONFIG_VERSION); + } + final boolean populateEmptyIfFileMissing = true; + ConfigHelper.uploadFile( + client, + cd + "nodes_dn.yml", + securityIndex, + CType.NODESDN, + DEFAULT_CONFIG_VERSION, + populateEmptyIfFileMissing + ); + ConfigHelper.uploadFile( + client, + cd + "whitelist.yml", + securityIndex, + CType.WHITELIST, + DEFAULT_CONFIG_VERSION, + populateEmptyIfFileMissing + ); + ConfigHelper.uploadFile( + client, + cd + "allowlist.yml", + securityIndex, + CType.ALLOWLIST, + DEFAULT_CONFIG_VERSION, + populateEmptyIfFileMissing + ); + + // audit.yml is not packaged by default + final String auditConfigPath = cd + "audit.yml"; + if (new File(auditConfigPath).exists()) { + ConfigHelper.uploadFile(client, auditConfigPath, securityIndex, CType.AUDIT, DEFAULT_CONFIG_VERSION); } - } else { - LOGGER.error("{} does not exist", confFile.getAbsolutePath()); } - } catch (Exception e) { - LOGGER.error("Cannot apply default config (this is maybe not an error!)", e); + } else { + LOGGER.error("{} does not exist", confFile.getAbsolutePath()); } + } catch (Exception e) { + LOGGER.error("Cannot apply default config (this is maybe not an error!)", e); } + } - while (!dynamicConfigFactory.isInitialized()) { + while (!dynamicConfigFactory.isInitialized()) { + try { + LOGGER.debug("Try to load config ..."); + reloadConfiguration(Arrays.asList(CType.values()), true); + break; + } catch (Exception e) { + LOGGER.debug("Unable to load configuration due to {}", String.valueOf(ExceptionUtils.getRootCause(e))); try { - LOGGER.debug("Try to load config ..."); - reloadConfiguration(Arrays.asList(CType.values())); + Thread.sleep(3000); + } catch (InterruptedException e1) { + Thread.currentThread().interrupt(); + LOGGER.debug("Thread was interrupted so we cancel initialization"); break; - } catch (Exception e) { - LOGGER.debug("Unable to load configuration due to {}", String.valueOf(ExceptionUtils.getRootCause(e))); - try { - Thread.sleep(3000); - } catch (InterruptedException e1) { - Thread.currentThread().interrupt(); - LOGGER.debug("Thread was interrupted so we cancel initialization"); - break; - } } } + } - final Set deprecatedAuditKeysInSettings = AuditConfig.getDeprecatedKeys(settings); + final Set deprecatedAuditKeysInSettings = AuditConfig.getDeprecatedKeys(settings); + if (!deprecatedAuditKeysInSettings.isEmpty()) { + LOGGER.warn( + "Following keys {} are deprecated in opensearch settings. They will be removed in plugin v2.0.0.0", + deprecatedAuditKeysInSettings + ); + } + final boolean isAuditConfigDocPresentInIndex = cl.isAuditConfigDocPresentInIndex(); + if (isAuditConfigDocPresentInIndex) { if (!deprecatedAuditKeysInSettings.isEmpty()) { - LOGGER.warn( - "Following keys {} are deprecated in opensearch settings. They will be removed in plugin v2.0.0.0", - deprecatedAuditKeysInSettings - ); + LOGGER.warn("Audit configuration settings found in both index and opensearch settings (deprecated)"); } - final boolean isAuditConfigDocPresentInIndex = cl.isAuditConfigDocPresentInIndex(); - if (isAuditConfigDocPresentInIndex) { - if (!deprecatedAuditKeysInSettings.isEmpty()) { - LOGGER.warn("Audit configuration settings found in both index and opensearch settings (deprecated)"); - } - LOGGER.info("Hot-reloading of audit configuration is enabled"); - } else { - LOGGER.info( - "Hot-reloading of audit configuration is disabled. Using configuration with defaults from opensearch settings. Populate the configuration in index using audit.yml or securityadmin to enable it." - ); - auditLog.setConfig(AuditConfig.from(settings)); - } - - LOGGER.info("Node '{}' initialized", clusterService.localNode().getName()); - - } catch (Exception e) { - LOGGER.error("Unexpected exception while initializing node " + e, e); + LOGGER.info("Hot-reloading of audit configuration is enabled"); + } else { + LOGGER.info( + "Hot-reloading of audit configuration is disabled. Using configuration with defaults from opensearch settings. Populate the configuration in index using audit.yml or securityadmin to enable it." + ); + auditLog.setConfig(AuditConfig.from(settings)); } - }); + LOGGER.info("Node '{}' initialized", clusterService.localNode().getName()); + + } catch (Exception e) { + LOGGER.error("Unexpected exception while initializing node " + e, e); + } } private boolean createSecurityIndexIfAbsent() { @@ -306,27 +312,37 @@ private void waitForSecurityIndexToBeAtLeastYellow() { } } - public void initOnNodeStart() { + public CompletableFuture initOnNodeStart() { + final boolean installDefaultConfig = settings.getAsBoolean(ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, false); + + final Supplier> startInitialization = () -> { + new Thread(() -> { + initalizeClusterConfiguration(installDefaultConfig); + initalizeConfigTask.complete(null); + }).start(); + return initalizeConfigTask.thenApply(result -> installDefaultConfig); + }; try { - if (settings.getAsBoolean(ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, false)) { + if (installDefaultConfig) { LOGGER.info("Will attempt to create index {} and default configs if they are absent", securityIndex); - installDefaultConfig.set(true); - bgThread.start(); + return startInitialization.get(); } else if (settings.getAsBoolean(ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, true)) { LOGGER.info( "Will not attempt to create index {} and default configs if they are absent. Use securityadmin to initialize cluster", securityIndex ); - bgThread.start(); + return startInitialization.get(); } else { LOGGER.info( "Will not attempt to create index {} and default configs if they are absent. Will not perform background initialization", securityIndex ); + initalizeConfigTask.complete(null); + return initalizeConfigTask.thenApply(result -> installDefaultConfig); } } catch (Throwable e2) { LOGGER.error("Error during node initialization: {}", e2, e2); - bgThread.start(); + return startInitialization.get(); } } @@ -372,16 +388,26 @@ public SecurityDynamicConfiguration getConfiguration(CType configurationType) private final Lock LOCK = new ReentrantLock(); - public void reloadConfiguration(Collection configTypes) throws ConfigUpdateAlreadyInProgressException { + public boolean reloadConfiguration(final Collection configTypes) throws ConfigUpdateAlreadyInProgressException { + return reloadConfiguration(configTypes, false); + } + + private boolean reloadConfiguration(final Collection configTypes, final boolean fromBackgroundThread) + throws ConfigUpdateAlreadyInProgressException { + if (!fromBackgroundThread && !initalizeConfigTask.isDone()) { + LOGGER.warn("Unable to reload configuration, initalization thread has not yet completed."); + return false; + } try { if (LOCK.tryLock(60, TimeUnit.SECONDS)) { try { reloadConfiguration0(configTypes, this.acceptInvalid); + return true; } finally { LOCK.unlock(); } } else { - throw new ConfigUpdateAlreadyInProgressException("A config update is already imn progress"); + throw new ConfigUpdateAlreadyInProgressException("A config update is already in progress"); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -489,7 +515,23 @@ public static int getDefaultConfigVersion() { return ConfigurationRepository.DEFAULT_CONFIG_VERSION; } - public AtomicBoolean getInstallDefaultConfig() { - return installDefaultConfig; + private class AccessControllerWrappedThread extends Thread { + private final Thread innerThread; + + public AccessControllerWrappedThread(Thread innerThread) { + this.innerThread = innerThread; + } + + @Override + public void run() { + AccessController.doPrivileged(new PrivilegedAction() { + + @Override + public Void run() { + innerThread.run(); + return null; + } + }); + } } } diff --git a/src/main/java/org/opensearch/security/securityconf/DynamicConfigFactory.java b/src/main/java/org/opensearch/security/securityconf/DynamicConfigFactory.java index ed61481885..f046b4c114 100644 --- a/src/main/java/org/opensearch/security/securityconf/DynamicConfigFactory.java +++ b/src/main/java/org/opensearch/security/securityconf/DynamicConfigFactory.java @@ -315,7 +315,6 @@ public void onChange(Map> typeToConfig) { } initialized.set(true); - } private static ConfigV6 getConfigV6(SecurityDynamicConfiguration sdc) { diff --git a/src/main/java/org/opensearch/security/support/ConfigConstants.java b/src/main/java/org/opensearch/security/support/ConfigConstants.java index d4383c05de..3060e1b2dc 100644 --- a/src/main/java/org/opensearch/security/support/ConfigConstants.java +++ b/src/main/java/org/opensearch/security/support/ConfigConstants.java @@ -281,6 +281,8 @@ public enum RolesMappingResolution { // Illegal Opcodes from here on public static final String SECURITY_UNSUPPORTED_DISABLE_REST_AUTH_INITIALLY = "plugins.security.unsupported.disable_rest_auth_initially"; + public static final String SECURITY_UNSUPPORTED_DELAY_INITIALIZATION_SECONDS = + "plugins.security.unsupported.delay_initialization_seconds"; public static final String SECURITY_UNSUPPORTED_DISABLE_INTERTRANSPORT_AUTH_INITIALLY = "plugins.security.unsupported.disable_intertransport_auth_initially"; public static final String SECURITY_UNSUPPORTED_PASSIVE_INTERTRANSPORT_AUTH_INITIALLY = diff --git a/src/test/java/org/opensearch/security/InitializationIntegrationTests.java b/src/test/java/org/opensearch/security/InitializationIntegrationTests.java index 79ab0c020b..7545822620 100644 --- a/src/test/java/org/opensearch/security/InitializationIntegrationTests.java +++ b/src/test/java/org/opensearch/security/InitializationIntegrationTests.java @@ -302,12 +302,6 @@ public void testInvalidDefaultConfig() throws Exception { HttpStatus.SC_SERVICE_UNAVAILABLE, rh.executeGetRequest("", encodeBasicHeader("admin", "admin")).getStatusCode() ); - - ClusterHelper.updateDefaultDirectory(defaultInitDirectory); - restart(Settings.EMPTY, null, settings, false); - rh = nonSslRestHelper(); - Thread.sleep(10000); - Assert.assertEquals(HttpStatus.SC_OK, rh.executeGetRequest("", encodeBasicHeader("admin", "admin")).getStatusCode()); } finally { ClusterHelper.resetSystemProperties(); } diff --git a/src/test/java/org/opensearch/security/configuration/ConfigurationRepositoryTest.java b/src/test/java/org/opensearch/security/configuration/ConfigurationRepositoryTest.java index c8f41433e0..5ce1873405 100644 --- a/src/test/java/org/opensearch/security/configuration/ConfigurationRepositoryTest.java +++ b/src/test/java/org/opensearch/security/configuration/ConfigurationRepositoryTest.java @@ -81,9 +81,9 @@ public void initOnNodeStart_withSecurityIndexCreationEnabledShouldSetInstallDefa ConfigurationRepository configRepository = createConfigurationRepository(settings); - configRepository.initOnNodeStart(); + final var result = configRepository.initOnNodeStart(); - assertThat(configRepository.getInstallDefaultConfig().get(), is(true)); + assertThat(result.join(), is(true)); } @Test @@ -92,9 +92,9 @@ public void initOnNodeStart_withSecurityIndexNotCreatedShouldNotSetInstallDefaul ConfigurationRepository configRepository = createConfigurationRepository(settings); - configRepository.initOnNodeStart(); + final var result = configRepository.initOnNodeStart(); - assertThat(configRepository.getInstallDefaultConfig().get(), is(false)); + assertThat(result.join(), is(false)); } @Test diff --git a/src/test/java/org/opensearch/security/test/SingleClusterTest.java b/src/test/java/org/opensearch/security/test/SingleClusterTest.java index 2839e1e283..cdde57a5c0 100644 --- a/src/test/java/org/opensearch/security/test/SingleClusterTest.java +++ b/src/test/java/org/opensearch/security/test/SingleClusterTest.java @@ -83,18 +83,6 @@ protected void setup( setup(initTransportClientSettings, dynamicSecuritySettings, nodeOverride, initSecurityIndex, ClusterConfiguration.DEFAULT); } - protected void restart( - Settings initTransportClientSettings, - DynamicSecurityConfig dynamicSecuritySettings, - Settings nodeOverride, - boolean initOpendistroSecurityIndex - ) throws Exception { - clusterInfo = clusterHelper.startCluster(minimumSecuritySettings(ccs(nodeOverride)), ClusterConfiguration.DEFAULT); - if (initOpendistroSecurityIndex && dynamicSecuritySettings != null) { - initialize(clusterHelper, clusterInfo, dynamicSecuritySettings); - } - } - private Settings ccs(Settings nodeOverride) throws Exception { if (remoteClusterHelper != null) { Assert.assertNull("No remote clusters", remoteClusterInfo); From df3465aec7c795773aa4dfe7c65084d9a2d82a36 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jan 2024 13:46:34 +0000 Subject: [PATCH 077/204] Bump com.google.errorprone:error_prone_annotations from 2.24.0 to 2.24.1 (#3929) Bumps [com.google.errorprone:error_prone_annotations](https://github.com/google/error-prone) from 2.24.0 to 2.24.1.

Release notes

Sourced from com.google.errorprone:error_prone_annotations's releases.

Error Prone 2.24.1

Changes:

Full Changelog: https://github.com/google/error-prone/compare/v2.24.0...v2.24.1

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.errorprone:error_prone_annotations&package-manager=gradle&previous-version=2.24.0&new-version=2.24.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 3a368bc8a7..e71bba82ec 100644 --- a/build.gradle +++ b/build.gradle @@ -499,7 +499,7 @@ configurations { // For integrationTest force "org.apache.httpcomponents:httpclient:4.5.14" force "org.apache.httpcomponents:httpcore:4.4.16" - force "com.google.errorprone:error_prone_annotations:2.24.0" + force "com.google.errorprone:error_prone_annotations:2.24.1" force "org.checkerframework:checker-qual:3.42.0" force "ch.qos.logback:logback-classic:1.2.13" } @@ -609,7 +609,7 @@ dependencies { runtimeOnly 'com.eclipsesource.minimal-json:minimal-json:0.9.5' runtimeOnly 'commons-codec:commons-codec:1.16.0' runtimeOnly 'org.cryptacular:cryptacular:1.2.6' - compileOnly 'com.google.errorprone:error_prone_annotations:2.24.0' + compileOnly 'com.google.errorprone:error_prone_annotations:2.24.1' runtimeOnly 'com.sun.istack:istack-commons-runtime:4.2.0' runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.1' runtimeOnly 'org.ow2.asm:asm:9.6' From 649a2fc43a3470b3c8c31fca0aec2ff6885e2ae2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jan 2024 13:47:12 +0000 Subject: [PATCH 078/204] Bump com.google.googlejavaformat:google-java-format from 1.19.1 to 1.19.2 (#3930) Bumps [com.google.googlejavaformat:google-java-format](https://github.com/google/google-java-format) from 1.19.1 to 1.19.2.
Release notes

Sourced from com.google.googlejavaformat:google-java-format's releases.

v1.19.2

Changes

  • Improve support for string templates (#1010)
  • Handle var in record patterns (#1020)

Full Changelog: https://github.com/google/google-java-format/compare/v1.19.1...v1.19.2

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.googlejavaformat:google-java-format&package-manager=gradle&previous-version=1.19.1&new-version=1.19.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index e71bba82ec..89fce7faf9 100644 --- a/build.gradle +++ b/build.gradle @@ -740,7 +740,7 @@ dependencies { integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5" //spotless - implementation('com.google.googlejavaformat:google-java-format:1.19.1') { + implementation('com.google.googlejavaformat:google-java-format:1.19.2') { exclude group: 'com.google.guava' } } From f217fa85a45100de730025307924715accfd85f8 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Wed, 10 Jan 2024 15:47:22 -0500 Subject: [PATCH 079/204] Fix Bug with Install demo configuration running in cluster mode with -y (#3935) Previous behavior prior to 2.11 would be that init security and cluster mode would only be set if they were explicitly passed in. There is a small bug that is setting them to be true if -y is passed in. This is causing a few failures with duplicate keys in integration tests when -y is passed in. https://github.com/opensearch-project/security/blob/2.11/tools/install_demo_configuration.sh#L73-L98 --------- Signed-off-by: Derek Ho --- .../security/tools/democonfig/Installer.java | 3 --- .../security/tools/democonfig/InstallerTests.java | 4 ++-- .../democonfig/SecuritySettingsConfigurerTests.java | 12 ++++++++++++ 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/opensearch/security/tools/democonfig/Installer.java b/src/main/java/org/opensearch/security/tools/democonfig/Installer.java index 61acd7e4c9..864607a9c6 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/Installer.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/Installer.java @@ -212,9 +212,6 @@ void gatherUserInputs() { cluster_mode = confirmAction(scanner, "Enable cluster mode?"); } } - } else { - initsecurity = true; - cluster_mode = true; } } diff --git a/src/test/java/org/opensearch/security/tools/democonfig/InstallerTests.java b/src/test/java/org/opensearch/security/tools/democonfig/InstallerTests.java index 06c6edf734..268bd9ea0e 100644 --- a/src/test/java/org/opensearch/security/tools/democonfig/InstallerTests.java +++ b/src/test/java/org/opensearch/security/tools/democonfig/InstallerTests.java @@ -193,8 +193,8 @@ public void testGatherInputs_withAssumeYes() { installer.gatherUserInputs(); - assertThat(installer.initsecurity, is(true)); - assertThat(installer.cluster_mode, is(true)); + assertThat(installer.initsecurity, is(false)); + assertThat(installer.cluster_mode, is(false)); } @Test diff --git a/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java b/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java index 27ba150a78..280d704fb8 100644 --- a/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java +++ b/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java @@ -241,6 +241,18 @@ public void testIsStringAlreadyPresentInFile_isPresent() throws IOException { assertThat(isKeyPresentInYMLFile(installer.OPENSEARCH_CONF_FILE, str2), is(equalTo(false))); } + @Test + public void testAssumeYesDoesNotInitializeClusterMode() throws IOException { + String nodeName = "node.name"; // cluster_mode + String securityIndex = "plugins.security.allow_default_init_securityindex"; // init_security + + installer.assumeyes = true; + securitySettingsConfigurer.writeSecurityConfigToOpenSearchYML(); + + assertThat(isKeyPresentInYMLFile(installer.OPENSEARCH_CONF_FILE, nodeName), is(false)); + assertThat(isKeyPresentInYMLFile(installer.OPENSEARCH_CONF_FILE, securityIndex), is(false)); + } + @Test public void testCreateSecurityAdminDemoScriptAndGetSecurityAdminCommands() throws IOException { String demoPath = installer.OPENSEARCH_CONF_DIR + "securityadmin_demo" + installer.FILE_EXTENSION; From b8a315a321927a8f4490fb5a7023663ffc184e31 Mon Sep 17 00:00:00 2001 From: Owais Kazi Date: Thu, 11 Jan 2024 05:24:01 -0800 Subject: [PATCH 080/204] Add permission for get workflow step (#3937) Description Adds permission for a new Get Workflow Step API. https://github.com/opensearch-project/flow-framework/pull/394 ### Check List - [ ] New functionality includes testing - [ ] New functionality has been documented - [ ] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). Signed-off-by: owaiskazi19 --- config/roles.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/roles.yml b/config/roles.yml index e307ee7de3..ee9a61f1d5 100644 --- a/config/roles.yml +++ b/config/roles.yml @@ -396,3 +396,4 @@ flow_framework_read_access: - 'cluster:admin/opensearch/flow_framework/workflow/search' - 'cluster:admin/opensearch/flow_framework/workflow_state/get' - 'cluster:admin/opensearch/flow_framework/workflow_state/search' + - 'cluster:admin/opensearch/flow_framework/workflow_step/get' From fdfe087ab0dae3366d4c57db6ab2289059dbc707 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 11 Jan 2024 11:55:57 -0600 Subject: [PATCH 081/204] Add additional logging around `testShouldSearchAll` tests (#3932) ### Description - Improve error message logging on mismatch in search response. #### Example of new error message: ``` Suite: Test class org.opensearch.security.DlsIntegrationTests 2> java.lang.AssertionError: Expected: Search hit with index <5> should contain field "artist"" with value equal to ""yes" but: Unexpected match in SearchResponse {"took":101,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":6,"relation":"eq"},"max_score":null,"hits":[{"_index":"first-test-index","_id":"INDEX_1_S1","_score":null,"_source":{"artist":"First artist","lyrics":"Very deep subject","stars":1,"genre":"rock","title":"Magnum Opus"},"sort":["INDEX_1_S1"]},{"_index":"first-test-index","_id":"INDEX_1_S2","_score":null,"_source":{"artist":"String","lyrics":"Once upon a time","stars":2,"genre":"blues","title":"Song 1+1"},"sort":["INDEX_1_S2"]},{"_index":"first-test-index","_id":"INDEX_1_S3","_score":null,"_source":{"artist":"Twins","lyrics":"giant nonsense","stars":3,"genre":"jazz","title":"Next song"},"sort":["INDEX_1_S3"]},{"_index":"first-test-index","_id":"INDEX_1_S4","_score":null,"_source":{"artist":"No!","lyrics":"Much too much","stars":4,"genre":"rock","title":"Poison"},"sort":["INDEX_1_S4"]},{"_index":"first-test-index","_id":"INDEX_1_S5","_score":null,"_source":{"artist":"yes","lyrics":"Little to little","stars":5,"genre":"blues","title":"Affirmative"},"sort":["INDEX_1_S5"]},{"_index":"first-test-index","_id":"INDEX_1_S6","_score":null,"_source":{"artist":"unknown","lyrics":"confidential secret classified","stars":6,"genre":"jazz","title":"confidential"},"sort":["INDEX_1_S6"]}]}} Field value is equal to "unknown" at __randomizedtesting.SeedInfo.seed([2CB66291B43A24:7C34555162F056C]:0) at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:20) at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:6) at org.opensearch.security.DlsIntegrationTests.testShouldSearchAll(DlsIntegrationTests.java:294) ``` ### Issues Resolved - Resolved https://github.com/opensearch-project/security/issues/3830 ### Testing Ran the suite of integration tests with [10x](https://github.com/opensearch-project/security/actions/runs/7464876335?pr=3932) with no failures. ### Check List - [ ] ~New functionality includes testing~ - [ ] ~New functionality has been documented~ - [x] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: Peter Nied --- .../matcher/SearchHitContainsFieldWithValueMatcher.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitContainsFieldWithValueMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitContainsFieldWithValueMatcher.java index c92924ebfe..7d1a6b3251 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitContainsFieldWithValueMatcher.java +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitContainsFieldWithValueMatcher.java @@ -35,6 +35,7 @@ class SearchHitContainsFieldWithValueMatcher extends TypeSafeDiagnosingMatche @Override protected boolean matchesSafely(SearchResponse searchResponse, Description mismatchDescription) { + mismatchDescription.appendText("Unexpected match in SearchResponse " + searchResponse.toString() + "\n"); Long numberOfHits = readTotalHits(searchResponse); if (numberOfHits == null) { mismatchDescription.appendText("Total number of hits is unknown."); From 37d3ca28492c29ea2e2a13ce1f95c43d9207a31e Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 11 Jan 2024 11:56:23 -0600 Subject: [PATCH 082/204] HeapBasedRateTracker uses time provider to allow simluating of time in unit tests (#3934) Signed-off-by: Peter Nied --- .../ratetracking/HeapBasedRateTracker.java | 10 ++++++- .../limiting/HeapBasedRateTrackerTest.java | 27 ++++++++++--------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/opensearch/security/util/ratetracking/HeapBasedRateTracker.java b/src/main/java/org/opensearch/security/util/ratetracking/HeapBasedRateTracker.java index 40b1f622d0..46aa577254 100644 --- a/src/main/java/org/opensearch/security/util/ratetracking/HeapBasedRateTracker.java +++ b/src/main/java/org/opensearch/security/util/ratetracking/HeapBasedRateTracker.java @@ -18,8 +18,10 @@ package org.opensearch.security.util.ratetracking; import java.util.Arrays; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.function.LongSupplier; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -33,16 +35,22 @@ public class HeapBasedRateTracker implements RateTracker cache; + private final LongSupplier timeProvider; private final long timeWindowMs; private final int maxTimeOffsets; public HeapBasedRateTracker(long timeWindowMs, int allowedTries, int maxEntries) { + this(timeWindowMs, allowedTries, maxEntries, null); + } + + public HeapBasedRateTracker(long timeWindowMs, int allowedTries, int maxEntries, LongSupplier timeProvider) { if (allowedTries < 2) { throw new IllegalArgumentException("allowedTries must be >= 2"); } this.timeWindowMs = timeWindowMs; this.maxTimeOffsets = allowedTries > 2 ? allowedTries - 2 : 0; + this.timeProvider = Optional.ofNullable(timeProvider).orElse(System::currentTimeMillis); this.cache = CacheBuilder.newBuilder() .expireAfterAccess(this.timeWindowMs, TimeUnit.MILLISECONDS) .maximumSize(maxEntries) @@ -89,7 +97,7 @@ private class ClientRecord { private short timeOffsetEnd = -1; synchronized boolean track() { - long timestamp = System.currentTimeMillis(); + long timestamp = timeProvider.getAsLong(); if (this.startTime == -1 || timestamp - getMostRecent() >= timeWindowMs) { this.startTime = timestamp; diff --git a/src/test/java/org/opensearch/security/auth/limiting/HeapBasedRateTrackerTest.java b/src/test/java/org/opensearch/security/auth/limiting/HeapBasedRateTrackerTest.java index c92c328564..aaae27e8c3 100644 --- a/src/test/java/org/opensearch/security/auth/limiting/HeapBasedRateTrackerTest.java +++ b/src/test/java/org/opensearch/security/auth/limiting/HeapBasedRateTrackerTest.java @@ -17,7 +17,9 @@ package org.opensearch.security.auth.limiting; -import org.junit.Ignore; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongSupplier; + import org.junit.Test; import org.opensearch.security.util.ratetracking.HeapBasedRateTracker; @@ -27,9 +29,12 @@ public class HeapBasedRateTrackerTest { + private final AtomicLong currentTime = new AtomicLong(1); + private LongSupplier timeProvider = () -> currentTime.getAndAdd(1); + @Test public void simpleTest() throws Exception { - HeapBasedRateTracker tracker = new HeapBasedRateTracker<>(100, 5, 100_000); + HeapBasedRateTracker tracker = new HeapBasedRateTracker<>(100, 5, 100_000, timeProvider); assertFalse(tracker.track("a")); assertFalse(tracker.track("a")); @@ -40,9 +45,8 @@ public void simpleTest() throws Exception { } @Test - @Ignore // https://github.com/opensearch-project/security/issues/2193 public void expiryTest() throws Exception { - HeapBasedRateTracker tracker = new HeapBasedRateTracker<>(100, 5, 100_000); + HeapBasedRateTracker tracker = new HeapBasedRateTracker<>(100, 5, 100_000, timeProvider); assertFalse(tracker.track("a")); assertFalse(tracker.track("a")); @@ -58,20 +62,20 @@ public void expiryTest() throws Exception { assertFalse(tracker.track("c")); - Thread.sleep(50); + currentTime.addAndGet(50); assertFalse(tracker.track("c")); assertFalse(tracker.track("c")); assertFalse(tracker.track("c")); - Thread.sleep(55); + currentTime.addAndGet(55); assertFalse(tracker.track("c")); assertTrue(tracker.track("c")); assertFalse(tracker.track("a")); - Thread.sleep(55); + currentTime.addAndGet(55); assertFalse(tracker.track("c")); assertFalse(tracker.track("c")); assertTrue(tracker.track("c")); @@ -79,21 +83,20 @@ public void expiryTest() throws Exception { } @Test - @Ignore // https://github.com/opensearch-project/security/issues/2193 public void maxTwoTriesTest() throws Exception { - HeapBasedRateTracker tracker = new HeapBasedRateTracker<>(100, 2, 100_000); + HeapBasedRateTracker tracker = new HeapBasedRateTracker<>(100, 2, 100_000, timeProvider); assertFalse(tracker.track("a")); assertTrue(tracker.track("a")); assertFalse(tracker.track("b")); - Thread.sleep(50); + currentTime.addAndGet(50); assertTrue(tracker.track("b")); - Thread.sleep(55); + currentTime.addAndGet(55); assertTrue(tracker.track("b")); - Thread.sleep(105); + currentTime.addAndGet(105); assertFalse(tracker.track("b")); assertTrue(tracker.track("b")); From b905999f81093fc96db8371df5f96badabb9ff38 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 11 Jan 2024 11:56:34 -0600 Subject: [PATCH 083/204] Add logging for test LdapServer actions (#3933) Signed-off-by: Peter Nied --- .../amazon/dlic/auth/ldap/srv/LdapServer.java | 40 ++++++++++++------- src/test/resources/log4j2-test.properties | 6 +++ 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/test/java/com/amazon/dlic/auth/ldap/srv/LdapServer.java b/src/test/java/com/amazon/dlic/auth/ldap/srv/LdapServer.java index 36bb37494d..64cc1f8ab0 100644 --- a/src/test/java/com/amazon/dlic/auth/ldap/srv/LdapServer.java +++ b/src/test/java/com/amazon/dlic/auth/ldap/srv/LdapServer.java @@ -23,9 +23,12 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; +import java.util.logging.Handler; +import java.util.logging.LogRecord; import com.google.common.io.CharStreams; import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -153,9 +156,8 @@ private synchronized int configureAndStartServer(String... ldifFiles) throws Exc config.setEnforceAttributeSyntaxCompliance(false); config.setEnforceSingleStructuralObjectClass(false); - // config.setLDAPDebugLogHandler(DEBUG_HANDLER); - // config.setAccessLogHandler(DEBUG_HANDLER); - // config.addAdditionalBindCredentials(configuration.getBindDn(), configuration.getPassword()); + config.setLDAPDebugLogHandler(new ServerLogger()); + config.setAccessLogHandler(new ServerLogger()); server = new InMemoryDirectoryServer(config); @@ -214,25 +216,33 @@ private int loadLdifFiles(String... ldifFiles) throws Exception { return ldifLoadCount; } - /* private static class DebugHandler extends Handler { - private final static Logger LOG = LogManager.getLogger(DebugHandler.class); + private static class ServerLogger extends Handler { + final Logger logger = LogManager.getLogger(ServerLogger.class); @Override - public void publish(LogRecord logRecord) { - //LOG.debug(ToStringBuilder.reflectionToString(logRecord, ToStringStyle.MULTI_LINE_STYLE)); + public void publish(final LogRecord logRecord) { + logger.log(toLog4jLevel(logRecord.getLevel()), logRecord.getMessage(), logRecord.getThrown()); } @Override - public void flush() { - - } + public void flush() {} @Override - public void close() throws SecurityException { - + public void close() throws SecurityException {} + + private Level toLog4jLevel(java.util.logging.Level javaLevel) { + switch (javaLevel.getName()) { + case "SEVERE": + return Level.ERROR; + case "WARNING": + return Level.WARN; + case "INFO": + return Level.INFO; + case "CONFIG": + return Level.DEBUG; + default: + return Level.TRACE; + } } } - - private static final DebugHandler DEBUG_HANDLER = new DebugHandler(); - */ } diff --git a/src/test/resources/log4j2-test.properties b/src/test/resources/log4j2-test.properties index 3d22ca3765..866b68325c 100644 --- a/src/test/resources/log4j2-test.properties +++ b/src/test/resources/log4j2-test.properties @@ -17,6 +17,12 @@ rootLogger.level = warn rootLogger.appenderRef.console.ref = console rootLogger.appenderRef.file.ref = LOGFILE +# For troubleshooting com.amazon.dlic.auth.ldap.* test cases +logger.ldapServerLogger.name = com.amazon.dlic.auth.ldap.srv.LdapServer.ServerLogger +logger.ldapServerLogger.level = info +logger.ldapAuthBackend.name = com.amazon.dlic.auth.ldap.backend.LDAPAuthorizationBackend +logger.ldapAuthBackend.level = debug + #logger.resolver.name = org.opensearch.security.resolver #logger.resolver.level = trace From d734b2e2a5e5b6b33782380a1abc1fb41f862114 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jan 2024 08:36:57 -0500 Subject: [PATCH 084/204] Bump com.diffplug.spotless from 6.23.3 to 6.24.0 (#3946) Bumps com.diffplug.spotless from 6.23.3 to 6.24.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.diffplug.spotless&package-manager=gradle&previous-version=6.23.3&new-version=6.24.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 89fce7faf9..ea810c1e17 100644 --- a/build.gradle +++ b/build.gradle @@ -62,7 +62,7 @@ plugins { id 'idea' id 'jacoco' id 'maven-publish' - id 'com.diffplug.spotless' version '6.23.3' + id 'com.diffplug.spotless' version '6.24.0' id 'checkstyle' id 'com.netflix.nebula.ospackage' version "11.6.0" id "org.gradle.test-retry" version "1.5.8" From 7273936149609c843e52ea831e67feda574e84fb Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Tue, 16 Jan 2024 14:13:18 -0800 Subject: [PATCH 085/204] Update security analytics roles to include custom log type cluster permissions (#3951) Signed-off-by: Subhobrata Dey --- config/roles.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/roles.yml b/config/roles.yml index ee9a61f1d5..ccd1759355 100644 --- a/config/roles.yml +++ b/config/roles.yml @@ -346,6 +346,7 @@ security_analytics_read_access: - 'cluster:admin/opensearch/securityanalytics/detector/get' - 'cluster:admin/opensearch/securityanalytics/detector/search' - 'cluster:admin/opensearch/securityanalytics/findings/get' + - 'cluster:admin/opensearch/securityanalytics/logtype/search' - 'cluster:admin/opensearch/securityanalytics/mapping/get' - 'cluster:admin/opensearch/securityanalytics/mapping/view/get' - 'cluster:admin/opensearch/securityanalytics/rule/get' @@ -359,6 +360,7 @@ security_analytics_full_access: - 'cluster:admin/opensearch/securityanalytics/correlations/*' - 'cluster:admin/opensearch/securityanalytics/detector/*' - 'cluster:admin/opensearch/securityanalytics/findings/*' + - 'cluster:admin/opensearch/securityanalytics/logtype/*' - 'cluster:admin/opensearch/securityanalytics/mapping/*' - 'cluster:admin/opensearch/securityanalytics/rule/*' index_permissions: From 6047d6626ec8adfb5d770d3955be5ffbcb0c907b Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Wed, 17 Jan 2024 11:42:41 -0600 Subject: [PATCH 086/204] Protect config object from concurrent modification issues (#3945) Signed-off-by: Peter Nied --- .../impl/SecurityDynamicConfiguration.java | 77 ++++++++++++------- 1 file changed, 51 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/opensearch/security/securityconf/impl/SecurityDynamicConfiguration.java b/src/main/java/org/opensearch/security/securityconf/impl/SecurityDynamicConfiguration.java index 938ee23c1e..bba44b5e28 100644 --- a/src/main/java/org/opensearch/security/securityconf/impl/SecurityDynamicConfiguration.java +++ b/src/main/java/org/opensearch/security/securityconf/impl/SecurityDynamicConfiguration.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -56,6 +57,8 @@ public class SecurityDynamicConfiguration implements ToXContent { @JsonIgnore private final Map centries = new HashMap<>(); + @JsonIgnore + private final Object modificationLock = new Object(); private long seqNo = -1; private long primaryTerm = -1; private CType ctype; @@ -158,23 +161,33 @@ void setCEntries(String key, T value) { @JsonAnyGetter public Map getCEntries() { - return centries; + synchronized (modificationLock) { + return new HashMap<>(centries); + } } @JsonIgnore public void removeHidden() { - for (Entry entry : new HashMap(centries).entrySet()) { - if (entry.getValue() instanceof Hideable && ((Hideable) entry.getValue()).isHidden()) { - centries.remove(entry.getKey()); + synchronized (modificationLock) { + final Iterator> iterator = centries.entrySet().iterator(); + while (iterator.hasNext()) { + final var entry = iterator.next(); + if (entry.getValue() instanceof Hideable && ((Hideable) entry.getValue()).isHidden()) { + iterator.remove(); + } } } } @JsonIgnore public void removeStatic() { - for (Entry entry : new HashMap(centries).entrySet()) { - if (entry.getValue() instanceof StaticDefinable && ((StaticDefinable) entry.getValue()).isStatic()) { - centries.remove(entry.getKey()); + synchronized (modificationLock) { + final Iterator> iterator = centries.entrySet().iterator(); + while (iterator.hasNext()) { + final var entry = iterator.next(); + if (entry.getValue() instanceof StaticDefinable && ((StaticDefinable) entry.getValue()).isStatic()) { + iterator.remove(); + } } } } @@ -189,20 +202,26 @@ public void clearHashes() { } public void removeOthers(String key) { - T tmp = this.centries.get(key); - this.centries.clear(); - this.centries.put(key, tmp); + synchronized (modificationLock) { + T tmp = this.centries.get(key); + this.centries.clear(); + this.centries.put(key, tmp); + } } @JsonIgnore public T putCEntry(String key, T value) { - return centries.put(key, value); + synchronized (modificationLock) { + return centries.put(key, value); + } } @JsonIgnore @SuppressWarnings("unchecked") public void putCObject(String key, Object value) { - centries.put(key, (T) value); + synchronized (modificationLock) { + centries.put(key, (T) value); + } } @JsonIgnore @@ -286,36 +305,42 @@ public SecurityDynamicConfiguration deepClone() { @JsonIgnore public void remove(String key) { - centries.remove(key); + synchronized (modificationLock) { + centries.remove(key); + } } @JsonIgnore public void remove(List keySet) { - keySet.stream().forEach(this::remove); + synchronized (modificationLock) { + keySet.stream().forEach(centries::remove); + } } @SuppressWarnings({ "rawtypes", "unchecked" }) public boolean add(SecurityDynamicConfiguration other) { - if (other.ctype == null || !other.ctype.equals(this.ctype)) { - return false; - } + synchronized (modificationLock) { + if (other.ctype == null || !other.ctype.equals(this.ctype)) { + return false; + } - if (other.getImplementingClass() == null || !other.getImplementingClass().equals(this.getImplementingClass())) { - return false; - } + if (other.getImplementingClass() == null || !other.getImplementingClass().equals(this.getImplementingClass())) { + return false; + } - if (other.version != this.version) { - return false; - } + if (other.version != this.version) { + return false; + } - this.centries.putAll(other.centries); - return true; + this.centries.putAll(other.centries); + return true; + } } @JsonIgnore @SuppressWarnings({ "rawtypes" }) public boolean containsAny(SecurityDynamicConfiguration other) { - return !Collections.disjoint(this.centries.keySet(), other.centries.keySet()); + return !Collections.disjoint(this.getCEntries().keySet(), other.getCEntries().keySet()); } public boolean isHidden(String resourceName) { From 09051f47b7a42741c5aaa0789497bb43b156648e Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Wed, 17 Jan 2024 11:43:50 -0600 Subject: [PATCH 087/204] Add test coverage for ComplianceConfig (#3952) Signed-off-by: Peter Nied --- .../security/compliance/ComplianceConfig.java | 37 +++++-- .../compliance/ComplianceConfigTest.java | 96 +++++++++++++++++++ .../config/AuditConfigSerializeTest.java | 1 + 3 files changed, 128 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/opensearch/security/compliance/ComplianceConfig.java b/src/main/java/org/opensearch/security/compliance/ComplianceConfig.java index edc5248781..b149f2604a 100644 --- a/src/main/java/org/opensearch/security/compliance/ComplianceConfig.java +++ b/src/main/java/org/opensearch/security/compliance/ComplianceConfig.java @@ -30,8 +30,10 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.function.Supplier; import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.CacheBuilder; @@ -73,11 +75,11 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public class ComplianceConfig { + public static Set FIELDS = DefaultObjectMapper.getFields(ComplianceConfig.class); private static final Logger log = LogManager.getLogger(ComplianceConfig.class); public static final ComplianceConfig DEFAULT = ComplianceConfig.from(Settings.EMPTY); private static final int CACHE_SIZE = 1000; private static final String INTERNAL_OPENSEARCH = "internal_opensearch"; - public static Set FIELDS = DefaultObjectMapper.getFields(ComplianceConfig.class); private final boolean logExternalConfig; private final boolean logInternalConfig; @@ -104,6 +106,7 @@ public class ComplianceConfig { private final DateTimeFormatter auditLogPattern; private final String auditLogIndex; private final boolean enabled; + private final Supplier dateProvider; private ComplianceConfig( final boolean enabled, @@ -118,7 +121,8 @@ private ComplianceConfig( final Set ignoredComplianceUsersForWrite, final String securityIndex, final String destinationType, - final String destinationIndex + final String destinationIndex, + final Supplier dateProvider ) { this.enabled = enabled; this.logExternalConfig = logExternalConfig; @@ -148,6 +152,11 @@ private ComplianceConfig( try { auditLogPattern = DateTimeFormat.forPattern(destinationIndex); // throws IllegalArgumentException if no pattern } catch (IllegalArgumentException e) { + log.warn( + "Unable to translate {} as a DateTimeFormat, will instead treat this as a static audit log index name. Error: {}", + destinationIndex, + e.getMessage() + ); // no pattern auditLogIndex = destinationIndex; } catch (Exception e) { @@ -163,6 +172,8 @@ public WildcardMatcher load(String index) throws Exception { return WildcardMatcher.from(getFieldsForIndex(index)); } }); + + this.dateProvider = Optional.ofNullable(dateProvider).orElse(() -> DateTime.now(DateTimeZone.UTC)); } @VisibleForTesting @@ -177,6 +188,7 @@ public ComplianceConfig( final boolean logDiffsForWrite, final List watchedWriteIndicesPatterns, final Set ignoredComplianceUsersForWrite, + final Supplier dateProvider, Settings settings ) { this( @@ -195,7 +207,8 @@ public ComplianceConfig( settings.get( ConfigConstants.SECURITY_AUDIT_CONFIG_DEFAULT_PREFIX + ConfigConstants.SECURITY_AUDIT_OPENSEARCH_INDEX, "'security-auditlog-'YYYY.MM.dd" - ) + ), + dateProvider ); } @@ -253,6 +266,7 @@ public static ComplianceConfig from(Map properties, @JacksonInje logDiffsForWrite, watchedWriteIndicesPatterns, ignoredComplianceUsersForWrite, + null, settings ); } @@ -263,6 +277,16 @@ public static ComplianceConfig from(Map properties, @JacksonInje * @return compliance configuration */ public static ComplianceConfig from(Settings settings) { + return ComplianceConfig.from(settings, null); + } + + /** + * Create compliance configuration from Settings defined in opensearch.yml + * @param settings settings + * @param dateProvider how the current date/time is evalated for audit logs that rollover + * @return compliance configuration + */ + public static ComplianceConfig from(Settings settings, Supplier dateProvider) { final boolean logExternalConfig = settings.getAsBoolean( ConfigConstants.OPENDISTRO_SECURITY_COMPLIANCE_HISTORY_EXTERNAL_CONFIG_ENABLED, false @@ -326,6 +350,7 @@ public static ComplianceConfig from(Settings settings) { logDiffsForWrite, watchedWriteIndices, ignoredComplianceUsersForWrite, + dateProvider, settings ); } @@ -469,7 +494,7 @@ private String getExpandedIndexName(DateTimeFormatter indexPattern, String index if (indexPattern == null) { return index; } - return indexPattern.print(DateTime.now(DateTimeZone.UTC)); + return indexPattern.print(dateProvider.get()); } /** @@ -507,7 +532,7 @@ public boolean writeHistoryEnabledForIndex(String index) { * @return true/false */ public boolean readHistoryEnabledForIndex(String index) { - if (!this.isEnabled()) { + if (index == null || !this.isEnabled()) { return false; } // if security index (internal index) check if internal config logging is enabled @@ -529,7 +554,7 @@ public boolean readHistoryEnabledForIndex(String index) { * @return true/false */ public boolean readHistoryEnabledForField(String index, String field) { - if (!this.isEnabled()) { + if (index == null || !this.isEnabled()) { return false; } // if security index (internal index) check if internal config logging is enabled diff --git a/src/test/java/org/opensearch/security/auditlog/compliance/ComplianceConfigTest.java b/src/test/java/org/opensearch/security/auditlog/compliance/ComplianceConfigTest.java index 467475212b..302d26bb00 100644 --- a/src/test/java/org/opensearch/security/auditlog/compliance/ComplianceConfigTest.java +++ b/src/test/java/org/opensearch/security/auditlog/compliance/ComplianceConfigTest.java @@ -12,8 +12,12 @@ package org.opensearch.security.auditlog.compliance; import java.util.Collections; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import com.google.common.collect.ImmutableSet; +import org.apache.logging.log4j.Logger; import org.junit.Test; import org.opensearch.common.settings.Settings; @@ -21,10 +25,22 @@ import org.opensearch.security.support.ConfigConstants; import org.opensearch.security.support.WildcardMatcher; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.mockito.Mockito; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; public class ComplianceConfigTest { @@ -136,4 +152,84 @@ public void testEmpty() { assertSame(WildcardMatcher.NONE, complianceConfig.getIgnoredComplianceUsersForReadMatcher()); assertSame(WildcardMatcher.NONE, complianceConfig.getIgnoredComplianceUsersForWriteMatcher()); } + + @Test + public void testLogState() { + // arrange + final var logger = Mockito.mock(Logger.class); + final ComplianceConfig complianceConfig = ComplianceConfig.from(Settings.EMPTY); + // act + complianceConfig.log(logger); + // assert: don't validate content, but ensure message's logged is generally consistant + verify(logger, times(6)).info(anyString(), anyString()); + verify(logger, times(1)).info(anyString(), isNull(String.class)); + verify(logger, times(1)).info(anyString(), any(Map.class)); + verify(logger, times(3)).info(anyString(), any(WildcardMatcher.class)); + verifyNoMoreInteractions(logger); + } + + @Test + public void testReadWriteHistoryEnabledForIndex_rollingIndex() { + // arrange + final var date = new AtomicReference(); + final Consumer setYear = (year) -> { date.set(new DateTime(year, 1, 1, 1, 2, DateTimeZone.UTC)); }; + final ComplianceConfig complianceConfig = ComplianceConfig.from( + Settings.builder() + .put( + ConfigConstants.SECURITY_AUDIT_CONFIG_DEFAULT_PREFIX + ConfigConstants.SECURITY_AUDIT_OPENSEARCH_INDEX, + "'audit-log-index'-YYYY-MM-dd" + ) + .put(ConfigConstants.SECURITY_AUDIT_TYPE_DEFAULT, "internal_opensearch") + .putList(ConfigConstants.OPENDISTRO_SECURITY_COMPLIANCE_HISTORY_READ_WATCHED_FIELDS, "*") + .putList(ConfigConstants.OPENDISTRO_SECURITY_COMPLIANCE_HISTORY_WRITE_WATCHED_INDICES, "*") + .build(), + date::get + ); + + // act: Don't log for null indices + assertThat(complianceConfig.readHistoryEnabledForIndex(null), equalTo(false)); + assertThat(complianceConfig.writeHistoryEnabledForIndex(null), equalTo(false)); + // act: Don't log for the security indices + assertThat(complianceConfig.readHistoryEnabledForIndex(complianceConfig.getSecurityIndex()), equalTo(false)); + assertThat(complianceConfig.writeHistoryEnabledForIndex(complianceConfig.getSecurityIndex()), equalTo(false)); + + // act: Don't log for the current audit log + setYear.accept(1337); + assertThat(complianceConfig.readHistoryEnabledForIndex("audit-log-index-1337-01-01"), equalTo(false)); + assertThat(complianceConfig.writeHistoryEnabledForIndex("audit-log-index-1337-01-01"), equalTo(false)); + + // act: Log for current audit log when it does not match the date + setYear.accept(2048); + // See https://github.com/opensearch-project/security/issues/3950 + // assertThat(complianceConfig.readHistoryEnabledForIndex("audit-log-index-1337-01-01"), equalTo(true)); + assertThat(complianceConfig.writeHistoryEnabledForIndex("audit-log-index-1337-01-01"), equalTo(true)); + + // act: Log for any matching index + assertThat(complianceConfig.readHistoryEnabledForIndex("my-data"), equalTo(true)); + assertThat(complianceConfig.writeHistoryEnabledForIndex("my-data"), equalTo(true)); + } + + @Test + public void testReadWriteHistoryEnabledForIndex_staticIndex() { + // arrange + final ComplianceConfig complianceConfig = ComplianceConfig.from( + Settings.builder() + .put( + ConfigConstants.SECURITY_AUDIT_CONFIG_DEFAULT_PREFIX + ConfigConstants.SECURITY_AUDIT_OPENSEARCH_INDEX, + "audit-log-index" + ) + .put(ConfigConstants.SECURITY_AUDIT_TYPE_DEFAULT, "internal_opensearch") + .putList(ConfigConstants.OPENDISTRO_SECURITY_COMPLIANCE_HISTORY_READ_WATCHED_FIELDS, "*") + .putList(ConfigConstants.OPENDISTRO_SECURITY_COMPLIANCE_HISTORY_WRITE_WATCHED_INDICES, "*") + .build() + ); + + // act: Don't log for the static audit log + assertThat(complianceConfig.readHistoryEnabledForIndex(complianceConfig.getAuditLogIndex()), equalTo(false)); + assertThat(complianceConfig.writeHistoryEnabledForIndex(complianceConfig.getAuditLogIndex()), equalTo(false)); + + // act: Log for any matching index + assertThat(complianceConfig.readHistoryEnabledForIndex("my-data"), equalTo(true)); + assertThat(complianceConfig.writeHistoryEnabledForIndex("my-data"), equalTo(true)); + } } diff --git a/src/test/java/org/opensearch/security/auditlog/config/AuditConfigSerializeTest.java b/src/test/java/org/opensearch/security/auditlog/config/AuditConfigSerializeTest.java index b0b93afc54..04cea3dc05 100644 --- a/src/test/java/org/opensearch/security/auditlog/config/AuditConfigSerializeTest.java +++ b/src/test/java/org/opensearch/security/auditlog/config/AuditConfigSerializeTest.java @@ -214,6 +214,7 @@ public void testSerialize() throws IOException { false, Collections.singletonList("test-write-watch-index"), Collections.singleton("test-user-2"), + null, Settings.EMPTY ); final AuditConfig auditConfig = new AuditConfig(true, audit, compliance); From 037bc20b212fae282e4eefc379ff6631f49db6cf Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Thu, 18 Jan 2024 10:10:34 -0500 Subject: [PATCH 088/204] Improve code coverage for SSLNettyTransport class (#3953) ### Description [Describe what this change achieves] This change increases code coverage for the SecuritySSLNettyTransport class. In the middle of 12/23, a few unit tests were added to give coverage to different parts of the class. This change builds on these existing changes. ### Issues Resolved Box three of https://github.com/opensearch-project/security/issues/3137 Signed-off-by: Stephen Crawford --- .../transport/SecuritySSLNettyTransport.java | 15 +- .../SecuritySSLNettyTransportTests.java | 132 ++++++++++++++---- 2 files changed, 120 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/opensearch/security/ssl/transport/SecuritySSLNettyTransport.java b/src/main/java/org/opensearch/security/ssl/transport/SecuritySSLNettyTransport.java index 242c7c56ed..5be3424528 100644 --- a/src/main/java/org/opensearch/security/ssl/transport/SecuritySSLNettyTransport.java +++ b/src/main/java/org/opensearch/security/ssl/transport/SecuritySSLNettyTransport.java @@ -39,6 +39,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.ExceptionsHelper; +import org.opensearch.OpenSearchSecurityException; import org.opensearch.Version; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.network.NetworkService; @@ -103,6 +104,11 @@ public SecuritySSLNettyTransport( this.SSLConfig = SSLConfig; } + // This allows for testing log messages + Logger getLogger() { + return logger; + } + @Override public void onException(TcpChannel channel, Exception e) { @@ -113,8 +119,11 @@ public void onException(TcpChannel channel, Exception e) { } errorHandler.logError(cause, false); - logger.error("Exception during establishing a SSL connection: " + cause, cause); + getLogger().error("Exception during establishing a SSL connection: " + cause, cause); + if (channel == null || !channel.isOpen()) { + throw new OpenSearchSecurityException("The provided TCP channel is invalid.", e); + } super.onException(channel, e); } @@ -156,7 +165,7 @@ public final void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) th } errorHandler.logError(cause, false); - logger.error("Exception during establishing a SSL connection: " + cause, cause); + getLogger().error("Exception during establishing a SSL connection: " + cause, cause); super.exceptionCaught(ctx, cause); } @@ -291,7 +300,7 @@ public final void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) th } errorHandler.logError(cause, false); - logger.error("Exception during establishing a SSL connection: " + cause, cause); + getLogger().error("Exception during establishing a SSL connection: " + cause, cause); super.exceptionCaught(ctx, cause); } diff --git a/src/test/java/org/opensearch/security/ssl/transport/SecuritySSLNettyTransportTests.java b/src/test/java/org/opensearch/security/ssl/transport/SecuritySSLNettyTransportTests.java index 27705988d8..32e0f48fac 100644 --- a/src/test/java/org/opensearch/security/ssl/transport/SecuritySSLNettyTransportTests.java +++ b/src/test/java/org/opensearch/security/ssl/transport/SecuritySSLNettyTransportTests.java @@ -11,10 +11,14 @@ package org.opensearch.security.ssl.transport; -import org.junit.Assert; +import java.util.Collections; + +import org.apache.logging.log4j.Logger; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.opensearch.OpenSearchSecurityException; import org.opensearch.Version; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.network.NetworkService; @@ -28,15 +32,27 @@ import org.opensearch.security.ssl.transport.SecuritySSLNettyTransport.SSLServerChannelInitializer; import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.FakeTcpChannel; import org.opensearch.transport.SharedGroupFactory; +import org.opensearch.transport.TcpChannel; import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.DecoderException; import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class SecuritySSLNettyTransportTests { @@ -45,16 +61,12 @@ public class SecuritySSLNettyTransportTests { @Mock private ThreadPool threadPool; @Mock - private NetworkService networkService; - @Mock private PageCacheRecycler pageCacheRecycler; @Mock private NamedWriteableRegistry namedWriteableRegistry; @Mock private CircuitBreakerService circuitBreakerService; @Mock - private SharedGroupFactory sharedGroupFactory; - @Mock private Tracer trace; @Mock private SecurityKeyStore ossks; @@ -63,55 +75,127 @@ public class SecuritySSLNettyTransportTests { @Mock private DiscoveryNode discoveryNode; + // This initializes all the above mocks + @Rule + public MockitoRule rule = MockitoJUnit.rule(); + + private NetworkService networkService; + private SharedGroupFactory sharedGroupFactory; + private Logger mockLogger; private SSLConfig sslConfig; private SecuritySSLNettyTransport securitySSLNettyTransport; + Throwable testCause = new Throwable("Test Cause"); @Before public void setup() { - sslConfig = new SSLConfig(Settings.EMPTY); + networkService = new NetworkService(Collections.emptyList()); + sharedGroupFactory = new SharedGroupFactory(Settings.EMPTY); - securitySSLNettyTransport = new SecuritySSLNettyTransport( - Settings.EMPTY, - version, - threadPool, - networkService, - pageCacheRecycler, - namedWriteableRegistry, - circuitBreakerService, - ossks, - sslExceptionHandler, - sharedGroupFactory, - sslConfig, - trace + sslConfig = new SSLConfig(Settings.EMPTY); + mockLogger = mock(Logger.class); + + securitySSLNettyTransport = spy( + new SecuritySSLNettyTransport( + Settings.EMPTY, + version, + threadPool, + networkService, + pageCacheRecycler, + namedWriteableRegistry, + circuitBreakerService, + ossks, + sslExceptionHandler, + sharedGroupFactory, + sslConfig, + trace + ) ); } @Test public void OnException_withNullChannelShouldThrowException() { - NullPointerException exception = new NullPointerException("Test Exception"); + OpenSearchSecurityException exception = new OpenSearchSecurityException("The provided TCP channel is invalid"); + assertThrows(OpenSearchSecurityException.class, () -> securitySSLNettyTransport.onException(null, exception)); + } + + @Test + public void OnException_withClosedChannelShouldThrowException() { + + TcpChannel channel = new FakeTcpChannel(); + channel.close(); + OpenSearchSecurityException exception = new OpenSearchSecurityException("The provided TCP channel is invalid"); + assertThrows(OpenSearchSecurityException.class, () -> securitySSLNettyTransport.onException(channel, exception)); + } + + @Test + public void OnException_withNullExceptionShouldSucceed() { + + TcpChannel channel = new FakeTcpChannel(); + securitySSLNettyTransport.onException(channel, null); + verify(securitySSLNettyTransport, times(1)).onException(channel, null); + channel.close(); + } - Assert.assertThrows(NullPointerException.class, () -> securitySSLNettyTransport.onException(null, exception)); + @Test + public void OnException_withDecoderExceptionShouldGetCause() { + when(securitySSLNettyTransport.getLogger()).thenReturn(mockLogger); + DecoderException exception = new DecoderException("Test Exception", testCause); + TcpChannel channel = new FakeTcpChannel(); + securitySSLNettyTransport.onException(channel, exception); + verify(mockLogger, times(1)).error("Exception during establishing a SSL connection: " + exception.getCause(), exception.getCause()); } @Test public void getServerChannelInitializer_shouldReturnValidServerChannel() { ChannelHandler channelHandler = securitySSLNettyTransport.getServerChannelInitializer("test-server-channel"); - assertThat(channelHandler, is(notNullValue())); assertThat(channelHandler, is(instanceOf(SSLServerChannelInitializer.class))); } @Test public void getClientChannelInitializer_shouldReturnValidClientChannel() { - ChannelHandler channelHandler = securitySSLNettyTransport.getClientChannelInitializer(discoveryNode); - assertThat(channelHandler, is(notNullValue())); assertThat(channelHandler, is(instanceOf(SSLClientChannelInitializer.class))); } + @Test + public void exceptionWithServerChannelHandlerContext_nonNullDecoderExceptionShouldGetCause() throws Exception { + when(securitySSLNettyTransport.getLogger()).thenReturn(mockLogger); + Throwable exception = new DecoderException("Test Exception", testCause); + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + securitySSLNettyTransport.getServerChannelInitializer(discoveryNode.getName()).exceptionCaught(ctx, exception); + verify(mockLogger, times(1)).error("Exception during establishing a SSL connection: " + exception.getCause(), exception.getCause()); + } + + @Test + public void exceptionWithServerChannelHandlerContext_nonNullCauseOnlyShouldNotGetCause() throws Exception { + when(securitySSLNettyTransport.getLogger()).thenReturn(mockLogger); + Throwable exception = new OpenSearchSecurityException("Test Exception", testCause); + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + securitySSLNettyTransport.getServerChannelInitializer(discoveryNode.getName()).exceptionCaught(ctx, exception); + verify(mockLogger, times(1)).error("Exception during establishing a SSL connection: " + exception, exception); + } + + @Test + public void exceptionWithClientChannelHandlerContext_nonNullDecoderExceptionShouldGetCause() throws Exception { + when(securitySSLNettyTransport.getLogger()).thenReturn(mockLogger); + Throwable exception = new DecoderException("Test Exception", testCause); + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + securitySSLNettyTransport.getClientChannelInitializer(discoveryNode).exceptionCaught(ctx, exception); + verify(mockLogger, times(1)).error("Exception during establishing a SSL connection: " + exception.getCause(), exception.getCause()); + } + + @Test + public void exceptionWithClientChannelHandlerContext_nonNullCauseOnlyShouldNotGetCause() throws Exception { + when(securitySSLNettyTransport.getLogger()).thenReturn(mockLogger); + Throwable exception = new OpenSearchSecurityException("Test Exception", testCause); + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + securitySSLNettyTransport.getClientChannelInitializer(discoveryNode).exceptionCaught(ctx, exception); + verify(mockLogger, times(1)).error("Exception during establishing a SSL connection: " + exception, exception); + } } From 9eeeb848b51db30e04d3eeb3b4c6de44cf013154 Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Thu, 18 Jan 2024 12:21:45 -0500 Subject: [PATCH 089/204] Remove Third-party.txt file (#3907) ### Description The Security plugin is one of only two plugins (alongside the SQL plugin) which has its own THIRD-PARTY.txt file. This file is not automatically generated and serves no obvious use. This changes follows the suggestions on the linked issue (below) and removes the file. I requested a legal review for this change just in case and will update this PR with a comment once I have confirmation it is fine to merge. ### Issues Resolved - resolves https://github.com/opensearch-project/security/issues/1833 ### Check List - [ ] ~New functionality includes testing~ - [ ] ~New functionality has been documented~ - [x] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). Signed-off-by: Stephen Crawford --- THIRD-PARTY.txt | 71 ------------------------------------------------- 1 file changed, 71 deletions(-) delete mode 100644 THIRD-PARTY.txt diff --git a/THIRD-PARTY.txt b/THIRD-PARTY.txt deleted file mode 100644 index d8a027321b..0000000000 --- a/THIRD-PARTY.txt +++ /dev/null @@ -1,71 +0,0 @@ - -Lists of 69 third-party dependencies. - (The Apache Software License, Version 2.0) HPPC Collections (com.carrotsearch:hppc:0.7.1 - http://labs.carrotsearch.com/hppc.html/hppc) - (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.8.10 - https://github.com/FasterXML/jackson-core) - (The Apache Software License, Version 2.0) Jackson dataformat: CBOR (com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.8.10 - http://github.com/FasterXML/jackson-dataformats-binary) - (The Apache Software License, Version 2.0) Jackson dataformat: Smile (com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.8.10 - http://github.com/FasterXML/jackson-dataformats-binary) - (The Apache Software License, Version 2.0) Jackson-dataformat-YAML (com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.8.10 - https://github.com/FasterXML/jackson) - (The Apache Software License, Version 2.0) OpenDistro Security SSL (com.amazon.opendistroforelasticsearch:opendistro-elasticsearch-security-ssl:0.0.8.0 - https://github.com/opendistro-for-elasticsearch/security-ssl) - (Apache License 2.0) compiler (com.github.spullara.mustache.java:compiler:0.9.3 - http://github.com/spullara/mustache.java) - (The Apache Software License, Version 2.0) FindBugs-jsr305 (com.google.code.findbugs:jsr305:1.3.9 - http://findbugs.sourceforge.net/) - (Apache 2.0) error-prone annotations (com.google.errorprone:error_prone_annotations:2.0.18 - http://nexus.sonatype.org/oss-repository-hosting.html/error_prone_parent/error_prone_annotations) - (The Apache Software License, Version 2.0) Guava: Google Core Libraries for Java (com.google.guava:guava:23.0 - https://github.com/google/guava/guava) - (The Apache Software License, Version 2.0) J2ObjC Annotations (com.google.j2objc:j2objc-annotations:1.1 - https://github.com/google/j2objc/) - (The Apache Software License, Version 2.0) T-Digest (com.tdunning:t-digest:3.0 - https://github.com/tdunning/t-digest) - (Lesser General Public License (LGPL)) JTS Topology Suite (com.vividsolutions:jts:1.13 - http://sourceforge.net/projects/jts-topo-suite) - (Apache License, Version 2.0) Apache Commons CLI (commons-cli:commons-cli:1.3.1 - http://commons.apache.org/proper/commons-cli/) - (Apache License, Version 2.0) Apache Commons Codec (commons-codec:commons-codec:1.10 - http://commons.apache.org/proper/commons-codec/) - (Apache License, Version 2.0) Apache Commons IO (commons-io:commons-io:2.6 - http://commons.apache.org/proper/commons-io/) - (The Apache Software License, Version 2.0) Commons Logging (commons-logging:commons-logging:1.1.3 - http://commons.apache.org/proper/commons-logging/) - (Apache License, Version 2.0) Netty/Buffer (io.netty:netty-buffer:4.1.16.Final - http://netty.io/netty-buffer/) - (Apache License, Version 2.0) Netty/Codec (io.netty:netty-codec:4.1.16.Final - http://netty.io/netty-codec/) - (Apache License, Version 2.0) Netty/Codec/HTTP (io.netty:netty-codec-http:4.1.16.Final - http://netty.io/netty-codec-http/) - (Apache License, Version 2.0) Netty/Common (io.netty:netty-common:4.1.16.Final - http://netty.io/netty-common/) - (Apache License, Version 2.0) Netty/Handler (io.netty:netty-handler:4.1.16.Final - http://netty.io/netty-handler/) - (Apache License, Version 2.0) Netty/Resolver (io.netty:netty-resolver:4.1.16.Final - http://netty.io/netty-resolver/) - (Apache License, Version 2.0) Netty/TomcatNative [OpenSSL - Dynamic] (io.netty:netty-tcnative:2.0.7.Final - https://github.com/netty/netty-tcnative/netty-tcnative/) - (Apache License, Version 2.0) Netty/Transport (io.netty:netty-transport:4.1.16.Final - http://netty.io/netty-transport/) - (Apache 2) Joda-Time (joda-time:joda-time:2.9.9 - http://www.joda.org/joda-time/) - (Eclipse Public License 1.0) JUnit (junit:junit:4.12 - http://junit.org) - (The MIT License) JOpt Simple (net.sf.jopt-simple:jopt-simple:5.0.2 - http://pholser.github.io/jopt-simple) - (Apache License, Version 2.0) Apache HttpAsyncClient (org.apache.httpcomponents:httpasyncclient:4.1.2 - http://hc.apache.org/httpcomponents-asyncclient) - (Apache License, Version 2.0) Apache HttpClient (org.apache.httpcomponents:httpclient:4.5.2 - http://hc.apache.org/httpcomponents-client) - (Apache License, Version 2.0) Apache HttpCore (org.apache.httpcomponents:httpcore:4.4.5 - http://hc.apache.org/httpcomponents-core-ga) - (Apache License, Version 2.0) Apache HttpCore NIO (org.apache.httpcomponents:httpcore-nio:4.4.5 - http://hc.apache.org/httpcomponents-core-ga) - (Apache License, Version 2.0) Apache Log4j API (org.apache.logging.log4j:log4j-api:2.9.1 - https://logging.apache.org/log4j/2.x/log4j-api/) - (Apache License, Version 2.0) Apache Log4j Core (org.apache.logging.log4j:log4j-core:2.9.1 - https://logging.apache.org/log4j/2.x/log4j-core/) - (Apache 2) Lucene Common Analyzers (org.apache.lucene:lucene-analyzers-common:7.2.1 - http://lucene.apache.org/lucene-parent/lucene-analyzers-common) - (Apache 2) Lucene Memory (org.apache.lucene:lucene-backward-codecs:7.2.1 - http://lucene.apache.org/lucene-parent/lucene-backward-codecs) - (Apache 2) Lucene Core (org.apache.lucene:lucene-core:7.2.1 - http://lucene.apache.org/lucene-parent/lucene-core) - (Apache 2) Lucene Grouping (org.apache.lucene:lucene-grouping:7.2.1 - http://lucene.apache.org/lucene-parent/lucene-grouping) - (Apache 2) Lucene Highlighter (org.apache.lucene:lucene-highlighter:7.2.1 - http://lucene.apache.org/lucene-parent/lucene-highlighter) - (Apache 2) Lucene Join (org.apache.lucene:lucene-join:7.2.1 - http://lucene.apache.org/lucene-parent/lucene-join) - (Apache 2) Lucene Memory (org.apache.lucene:lucene-memory:7.2.1 - http://lucene.apache.org/lucene-parent/lucene-memory) - (Apache 2) Lucene Miscellaneous (org.apache.lucene:lucene-misc:7.2.1 - http://lucene.apache.org/lucene-parent/lucene-misc) - (Apache 2) Lucene Queries (org.apache.lucene:lucene-queries:7.2.1 - http://lucene.apache.org/lucene-parent/lucene-queries) - (Apache 2) Lucene QueryParsers (org.apache.lucene:lucene-queryparser:7.2.1 - http://lucene.apache.org/lucene-parent/lucene-queryparser) - (Apache 2) Lucene Sandbox (org.apache.lucene:lucene-sandbox:7.2.1 - http://lucene.apache.org/lucene-parent/lucene-sandbox) - (Apache 2) Lucene Spatial (org.apache.lucene:lucene-spatial:7.2.1 - http://lucene.apache.org/lucene-parent/lucene-spatial) - (Apache 2) Lucene Spatial Extras (org.apache.lucene:lucene-spatial-extras:7.2.1 - http://lucene.apache.org/lucene-parent/lucene-spatial-extras) - (Apache 2) Lucene Spatial 3D (org.apache.lucene:lucene-spatial3d:7.2.1 - http://lucene.apache.org/lucene-parent/lucene-spatial3d) - (Apache 2) Lucene Suggest (org.apache.lucene:lucene-suggest:7.2.1 - http://lucene.apache.org/lucene-parent/lucene-suggest) - (Apache Software License, Version 1.1) (Bouncy Castle Licence) Bouncy Castle OpenPGP API (org.bouncycastle:bcpg-jdk15on:1.59 - http://www.bouncycastle.org/java.html) - (Bouncy Castle Licence) Bouncy Castle Provider (org.bouncycastle:bcprov-jdk15on:1.59 - http://www.bouncycastle.org/java.html) - (MIT license) Animal Sniffer Annotations (org.codehaus.mojo:animal-sniffer-annotations:1.14 - http://mojo.codehaus.org/animal-sniffer/animal-sniffer-annotations) - (The Apache Software License, Version 2.0) server (org.elasticsearch:elasticsearch:6.2.0 - https://github.com/elastic/elasticsearch) - (The Apache Software License, Version 2.0) cli (org.elasticsearch:elasticsearch-cli:6.2.0 - https://github.com/elastic/elasticsearch) - (The Apache Software License, Version 2.0) elasticsearch-core (org.elasticsearch:elasticsearch-core:6.2.0 - https://github.com/elastic/elasticsearch) - (The Apache Software License, Version 2.0) java native access (net.java.dev.jna:jna:5.5.0 - https://github.com/java-native-access/jna) - (The Apache Software License, Version 2.0) Elasticsearch SecureSM (org.elasticsearch:securesm:1.2 - http://nexus.sonatype.org/oss-repository-hosting.html/securesm) - (The Apache Software License, Version 2.0) rest (org.elasticsearch.client:elasticsearch-rest-client:6.2.0 - https://github.com/elastic/elasticsearch) - (The Apache Software License, Version 2.0) aggs-matrix-stats (org.elasticsearch.plugin:aggs-matrix-stats-client:6.2.0 - https://github.com/elastic/elasticsearch) - (The Apache Software License, Version 2.0) lang-mustache (org.elasticsearch.plugin:lang-mustache-client:6.2.0 - https://github.com/elastic/elasticsearch) - (The Apache Software License, Version 2.0) parent-join (org.elasticsearch.plugin:parent-join-client:6.2.0 - https://github.com/elastic/elasticsearch) - (The Apache Software License, Version 2.0) percolator (org.elasticsearch.plugin:percolator-client:6.2.0 - https://github.com/elastic/elasticsearch) - (The Apache Software License, Version 2.0) reindex (org.elasticsearch.plugin:reindex-client:6.2.0 - https://github.com/elastic/elasticsearch) - (The Apache Software License, Version 2.0) transport-netty4 (org.elasticsearch.plugin:transport-netty4-client:6.2.0 - https://github.com/elastic/elasticsearch) - (New BSD License) Hamcrest All (org.hamcrest:hamcrest-all:1.3 - https://github.com/hamcrest/JavaHamcrest/hamcrest-all) - (New BSD License) Hamcrest Core (org.hamcrest:hamcrest-core:1.3 - https://github.com/hamcrest/JavaHamcrest/hamcrest-core) - (Public Domain, per Creative Commons CC0) HdrHistogram (org.hdrhistogram:HdrHistogram:2.1.9 - http://hdrhistogram.github.io/HdrHistogram/) - (The Apache Software License, Version 2.0) Spatial4J (org.locationtech.spatial4j:spatial4j:0.6 - http://www.locationtech.org/projects/locationtech.spatial4j) - (Apache License, Version 2.0) SnakeYAML (org.yaml:snakeyaml:1.17 - http://www.snakeyaml.org) From 15f5b7b8514c5ef6ba4771e4c182086cf5c21362 Mon Sep 17 00:00:00 2001 From: Prabhas Kurapati <66924475+prabhask5@users.noreply.github.com> Date: Fri, 19 Jan 2024 06:48:41 -0800 Subject: [PATCH 090/204] Validate 409s occur when multiple config updates happen simultaneously (#3909) ### Description Added new TenantAsyncActionTest.java file to put api unit tests for the /api/tenants api call- additionally wrote new unit test to test for parallel PUT /tenant/{name} calls, that they successfully return a 409 call and retry the call. * Category (Enhancement, New feature, Bug fix, Test fix, Refactoring, Maintenance, Documentation) Test fix * Why these changes are required? There was no test that checked for the behavior mentioned in the linked ticket, so previously we didn't know if the issue still existed or not (it apparently was fixed before), to make sure this doesn't happen again for this particular case, I decided to write a concrete test for it. * What is the old behavior before changes and new behavior after changes? Test was added, there's no behavior change? ### Issues Resolved - #1402 - added unit test to cover bug in this issue ### Testing Added new tenant action unit test to test parallel tenant api calls. ### Check List - [x] New functionality includes testing - [x] New functionality has been documented - [x] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: Prabhas Kurapati --- .../security/SecurityConfigurationTests.java | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/integrationTest/java/org/opensearch/security/SecurityConfigurationTests.java b/src/integrationTest/java/org/opensearch/security/SecurityConfigurationTests.java index 6b04737d18..3889fa2a3c 100644 --- a/src/integrationTest/java/org/opensearch/security/SecurityConfigurationTests.java +++ b/src/integrationTest/java/org/opensearch/security/SecurityConfigurationTests.java @@ -13,8 +13,13 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.apache.http.HttpStatus; import org.awaitility.Awaitility; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -24,6 +29,7 @@ import org.junit.runner.RunWith; import org.opensearch.client.Client; +import org.opensearch.test.framework.AsyncActions; import org.opensearch.test.framework.TestSecurityConfig.Role; import org.opensearch.test.framework.TestSecurityConfig.User; import org.opensearch.test.framework.certificate.TestCertificates; @@ -33,6 +39,8 @@ import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.opensearch.security.support.ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST; @@ -229,4 +237,39 @@ public void shouldUseSecurityAdminTool() throws Exception { .until(() -> client.get("_plugins/_security/api/rolesmapping/readall").getStatusCode(), equalTo(200)); } } + + @Test + public void testParallelTenantPutRequests() throws Exception { + final String TENANT_ENDPOINT = "_plugins/_security/api/tenants/tenant1"; + final String TENANT_BODY = "{\"description\":\"create new tenant\"}"; + final String TENANT_BODY_TWO = "{\"description\":\"update tenant\"}"; + + try (TestRestClient client = cluster.getRestClient(USER_ADMIN)) { + + final CountDownLatch countDownLatch = new CountDownLatch(1); + final List> conflictingRequests = AsyncActions.generate(() -> { + countDownLatch.await(); + return client.putJson(TENANT_ENDPOINT, TENANT_BODY); + }, 4, 4); + + // Make sure all requests start at the same time + countDownLatch.countDown(); + + AtomicInteger numCreatedResponses = new AtomicInteger(); + AsyncActions.getAll(conflictingRequests, 1, TimeUnit.SECONDS).forEach((response) -> { + assertThat(response.getStatusCode(), anyOf(equalTo(HttpStatus.SC_CREATED), equalTo(HttpStatus.SC_CONFLICT))); + if (response.getStatusCode() == HttpStatus.SC_CREATED) numCreatedResponses.getAndIncrement(); + }); + assertThat(numCreatedResponses.get(), equalTo(1)); // should only be one 201 + + TestRestClient.HttpResponse getResponse = client.get(TENANT_ENDPOINT); // make sure the one 201 works + assertThat(getResponse.getBody(), containsString("create new tenant")); + + TestRestClient.HttpResponse updateResponse = client.putJson(TENANT_ENDPOINT, TENANT_BODY_TWO); + assertThat(updateResponse.getStatusCode(), equalTo(HttpStatus.SC_OK)); + + getResponse = client.get(TENANT_ENDPOINT); // make sure update works + assertThat(getResponse.getBody(), containsString("update tenant")); + } + } } From 881ed584e1497894d0f5dc99faf1d19ab7df9f37 Mon Sep 17 00:00:00 2001 From: Prabhas Kurapati <66924475+prabhask5@users.noreply.github.com> Date: Fri, 19 Jan 2024 14:09:05 -0800 Subject: [PATCH 091/204] [Enhancement-3191] `transport_enabled` setting on an auth domain and authorizer may be unnecessary after transport client removal (#3939) Signed-off-by: Prabhas Kurapati --- .../security/http/LdapAuthenticationTest.java | 1 - .../http/LdapTlsAuthenticationTest.java | 1 - .../test/framework/AuthzDomain.java | 8 -- .../securityconf/DynamicConfigModelV6.java | 20 +--- .../securityconf/DynamicConfigModelV7.java | 20 +--- .../securityconf/impl/v6/ConfigV6.java | 57 +++++++++-- .../securityconf/impl/v7/ConfigV7.java | 58 +++++++++-- .../security/setting/DeprecatedSettings.java | 15 +++ .../setting/DeprecatedSettingsTest.java | 97 +++++++++++++++++++ .../resources/restapi/invalid_config.json | 6 -- .../resources/restapi/security_config.json | 6 -- .../resources/restapi/securityconfig.json | 7 -- .../restapi/securityconfig_nondefault.json | 8 -- 13 files changed, 213 insertions(+), 91 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/http/LdapAuthenticationTest.java b/src/integrationTest/java/org/opensearch/security/http/LdapAuthenticationTest.java index dbb1724a55..b4a3717287 100644 --- a/src/integrationTest/java/org/opensearch/security/http/LdapAuthenticationTest.java +++ b/src/integrationTest/java/org/opensearch/security/http/LdapAuthenticationTest.java @@ -109,7 +109,6 @@ public class LdapAuthenticationTest { .rolesMapping(new RolesMapping(ALL_ACCESS).backendRoles(CN_GROUP_ADMIN)) .authz( new AuthzDomain("ldap_roles").httpEnabled(true) - .transportEnabled(true) .authorizationBackend( new AuthorizationBackend("ldap").config( () -> new LdapAuthorizationConfigBuilder().hosts(List.of("localhost:" + embeddedLDAPServer.getLdapNonTlsPort())) diff --git a/src/integrationTest/java/org/opensearch/security/http/LdapTlsAuthenticationTest.java b/src/integrationTest/java/org/opensearch/security/http/LdapTlsAuthenticationTest.java index bac79ffd12..32265f4b81 100644 --- a/src/integrationTest/java/org/opensearch/security/http/LdapTlsAuthenticationTest.java +++ b/src/integrationTest/java/org/opensearch/security/http/LdapTlsAuthenticationTest.java @@ -156,7 +156,6 @@ public class LdapTlsAuthenticationTest { ) .authz( new AuthzDomain("ldap_roles").httpEnabled(true) - .transportEnabled(true) .authorizationBackend( new AuthorizationBackend("ldap").config( () -> new LdapAuthorizationConfigBuilder().hosts(List.of("localhost:" + embeddedLDAPServer.getLdapTlsPort())) diff --git a/src/integrationTest/java/org/opensearch/test/framework/AuthzDomain.java b/src/integrationTest/java/org/opensearch/test/framework/AuthzDomain.java index 5ccf1f9ee0..d56344d5d2 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/AuthzDomain.java +++ b/src/integrationTest/java/org/opensearch/test/framework/AuthzDomain.java @@ -25,8 +25,6 @@ public class AuthzDomain implements ToXContentObject { private boolean httpEnabled; - private boolean transportEnabled; - private AuthorizationBackend authorizationBackend; public AuthzDomain(String id) { @@ -52,17 +50,11 @@ public AuthzDomain authorizationBackend(AuthorizationBackend authorizationBacken return this; } - public AuthzDomain transportEnabled(boolean transportEnabled) { - this.transportEnabled = transportEnabled; - return this; - } - @Override public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { xContentBuilder.startObject(); xContentBuilder.field("description", description); xContentBuilder.field("http_enabled", httpEnabled); - xContentBuilder.field("transport_enabled", transportEnabled); xContentBuilder.field("authorization_backend", authorizationBackend); xContentBuilder.endObject(); return xContentBuilder; diff --git a/src/main/java/org/opensearch/security/securityconf/DynamicConfigModelV6.java b/src/main/java/org/opensearch/security/securityconf/DynamicConfigModelV6.java index e5308aa574..b652893bdd 100644 --- a/src/main/java/org/opensearch/security/securityconf/DynamicConfigModelV6.java +++ b/src/main/java/org/opensearch/security/securityconf/DynamicConfigModelV6.java @@ -68,8 +68,6 @@ public class DynamicConfigModelV6 extends DynamicConfigModel { private final Path configPath; private SortedSet restAuthDomains; private Set restAuthorizers; - private SortedSet transportAuthDomains; - private Set transportAuthorizers; private List destroyableComponents; private final InternalAuthenticationBackend iab; @@ -216,8 +214,6 @@ private void buildAAA() { final SortedSet restAuthDomains0 = new TreeSet<>(); final Set restAuthorizers0 = new HashSet<>(); - final SortedSet transportAuthDomains0 = new TreeSet<>(); - final Set transportAuthorizers0 = new HashSet<>(); final List destroyableComponents0 = new LinkedList<>(); final List ipAuthFailureListeners0 = new ArrayList<>(); final Multimap authBackendFailureListeners0 = ArrayListMultimap.create(); @@ -229,9 +225,8 @@ private void buildAAA() { for (final Entry ad : authzDyn.getDomains().entrySet()) { final boolean enabled = ad.getValue().enabled; final boolean httpEnabled = enabled && ad.getValue().http_enabled; - final boolean transportEnabled = enabled && ad.getValue().transport_enabled; - if (httpEnabled || transportEnabled) { + if (httpEnabled) { try { final String authzBackendClazz = ad.getValue().authorization_backend.type; @@ -264,10 +259,6 @@ private void buildAAA() { restAuthorizers0.add(authorizationBackend); } - if (transportEnabled) { - transportAuthorizers0.add(authorizationBackend); - } - if (authorizationBackend instanceof Destroyable) { destroyableComponents0.add((Destroyable) authorizationBackend); } @@ -282,9 +273,8 @@ private void buildAAA() { for (final Entry ad : authcDyn.getDomains().entrySet()) { final boolean enabled = ad.getValue().enabled; final boolean httpEnabled = enabled && ad.getValue().http_enabled; - final boolean transportEnabled = enabled && ad.getValue().transport_enabled; - if (httpEnabled || transportEnabled) { + if (httpEnabled) { try { AuthenticationBackend authenticationBackend; final String authBackendClazz = ad.getValue().authentication_backend.type; @@ -343,10 +333,6 @@ private void buildAAA() { restAuthDomains0.add(_ad); } - if (transportEnabled) { - transportAuthDomains0.add(_ad); - } - if (httpAuthenticator instanceof Destroyable) { destroyableComponents0.add((Destroyable) httpAuthenticator); } @@ -365,9 +351,7 @@ private void buildAAA() { List originalDestroyableComponents = destroyableComponents; restAuthDomains = Collections.unmodifiableSortedSet(restAuthDomains0); - transportAuthDomains = Collections.unmodifiableSortedSet(transportAuthDomains0); restAuthorizers = Collections.unmodifiableSet(restAuthorizers0); - transportAuthorizers = Collections.unmodifiableSet(transportAuthorizers0); destroyableComponents = Collections.unmodifiableList(destroyableComponents0); diff --git a/src/main/java/org/opensearch/security/securityconf/DynamicConfigModelV7.java b/src/main/java/org/opensearch/security/securityconf/DynamicConfigModelV7.java index 0de83f2e2e..91bb59db64 100644 --- a/src/main/java/org/opensearch/security/securityconf/DynamicConfigModelV7.java +++ b/src/main/java/org/opensearch/security/securityconf/DynamicConfigModelV7.java @@ -76,8 +76,6 @@ public class DynamicConfigModelV7 extends DynamicConfigModel { private final Path configPath; private SortedSet restAuthDomains; private Set restAuthorizers; - private SortedSet transportAuthDomains; - private Set transportAuthorizers; private List destroyableComponents; private final InternalAuthenticationBackend iab; @@ -234,8 +232,6 @@ private void buildAAA() { final SortedSet restAuthDomains0 = new TreeSet<>(); final Set restAuthorizers0 = new HashSet<>(); - final SortedSet transportAuthDomains0 = new TreeSet<>(); - final Set transportAuthorizers0 = new HashSet<>(); final List destroyableComponents0 = new LinkedList<>(); final List ipAuthFailureListeners0 = new ArrayList<>(); final Multimap authBackendFailureListeners0 = ArrayListMultimap.create(); @@ -246,9 +242,8 @@ private void buildAAA() { for (final Entry ad : authzDyn.getDomains().entrySet()) { final boolean httpEnabled = ad.getValue().http_enabled; - final boolean transportEnabled = ad.getValue().transport_enabled; - if (httpEnabled || transportEnabled) { + if (httpEnabled) { try { final String authzBackendClazz = ad.getValue().authorization_backend.type; @@ -281,10 +276,6 @@ private void buildAAA() { restAuthorizers0.add(authorizationBackend); } - if (transportEnabled) { - transportAuthorizers0.add(authorizationBackend); - } - if (authorizationBackend instanceof Destroyable) { destroyableComponents0.add((Destroyable) authorizationBackend); } @@ -298,9 +289,8 @@ private void buildAAA() { for (final Entry ad : authcDyn.getDomains().entrySet()) { final boolean httpEnabled = ad.getValue().http_enabled; - final boolean transportEnabled = ad.getValue().transport_enabled; - if (httpEnabled || transportEnabled) { + if (httpEnabled) { try { AuthenticationBackend authenticationBackend; final String authBackendClazz = ad.getValue().authentication_backend.type; @@ -359,10 +349,6 @@ private void buildAAA() { restAuthDomains0.add(_ad); } - if (transportEnabled) { - transportAuthDomains0.add(_ad); - } - if (httpAuthenticator instanceof Destroyable) { destroyableComponents0.add((Destroyable) httpAuthenticator); } @@ -398,9 +384,7 @@ private void buildAAA() { List originalDestroyableComponents = destroyableComponents; restAuthDomains = Collections.unmodifiableSortedSet(restAuthDomains0); - transportAuthDomains = Collections.unmodifiableSortedSet(transportAuthDomains0); restAuthorizers = Collections.unmodifiableSet(restAuthorizers0); - transportAuthorizers = Collections.unmodifiableSet(transportAuthorizers0); destroyableComponents = Collections.unmodifiableList(destroyableComponents0); diff --git a/src/main/java/org/opensearch/security/securityconf/impl/v6/ConfigV6.java b/src/main/java/org/opensearch/security/securityconf/impl/v6/ConfigV6.java index 0c95e56bd1..c5b954675b 100644 --- a/src/main/java/org/opensearch/security/securityconf/impl/v6/ConfigV6.java +++ b/src/main/java/org/opensearch/security/securityconf/impl/v6/ConfigV6.java @@ -38,9 +38,12 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException; import org.opensearch.security.DefaultObjectMapper; import org.opensearch.security.auth.internal.InternalAuthenticationBackend; +import org.opensearch.security.setting.DeprecatedSettings; public class ConfigV6 { @@ -224,8 +227,6 @@ public static class AuthcDomain { @JsonInclude(JsonInclude.Include.NON_NULL) public boolean http_enabled = true; @JsonInclude(JsonInclude.Include.NON_NULL) - public boolean transport_enabled = true; - @JsonInclude(JsonInclude.Include.NON_NULL) public boolean enabled = true; public int order = 0; public HttpAuthenticator http_authenticator = new HttpAuthenticator(); @@ -235,8 +236,6 @@ public static class AuthcDomain { public String toString() { return "AuthcDomain [http_enabled=" + http_enabled - + ", transport_enabled=" - + transport_enabled + ", enabled=" + enabled + ", order=" @@ -248,6 +247,31 @@ public String toString() { + "]"; } + @JsonAnySetter + public void unknownPropertiesHandler(String name, Object value) throws JsonMappingException { + switch (name) { + case "transport_enabled": + DeprecatedSettings.logCustomDeprecationMessage( + String.format( + "In AuthcDomain, using http_authenticator=%s, authentication_backend=%s", + http_authenticator, + authentication_backend + ), + name + ); + break; + default: + throw new UnrecognizedPropertyException( + null, + "Unrecognized field " + name + " present in the input data for AuthcDomain config", + null, + AuthcDomain.class, + name, + null + ); + } + } + } public static class HttpAuthenticator { @@ -337,8 +361,6 @@ public static class AuthzDomain { @JsonInclude(JsonInclude.Include.NON_NULL) public boolean http_enabled = true; @JsonInclude(JsonInclude.Include.NON_NULL) - public boolean transport_enabled = true; - @JsonInclude(JsonInclude.Include.NON_NULL) public boolean enabled = true; public AuthzBackend authorization_backend = new AuthzBackend(); @@ -346,8 +368,6 @@ public static class AuthzDomain { public String toString() { return "AuthzDomain [http_enabled=" + http_enabled - + ", transport_enabled=" - + transport_enabled + ", enabled=" + enabled + ", authorization_backend=" @@ -355,6 +375,27 @@ public String toString() { + "]"; } + @JsonAnySetter + public void unknownPropertiesHandler(String name, Object value) throws JsonMappingException { + switch (name) { + case "transport_enabled": + DeprecatedSettings.logCustomDeprecationMessage( + String.format("In AuthzDomain, using authorization_backend=%s", authorization_backend), + name + ); + break; + default: + throw new UnrecognizedPropertyException( + null, + "Unrecognized field " + name + " present in the input data for AuthzDomain config", + null, + AuthzDomain.class, + name, + null + ); + } + } + } public static class OnBehalfOfSettings { diff --git a/src/main/java/org/opensearch/security/securityconf/impl/v7/ConfigV7.java b/src/main/java/org/opensearch/security/securityconf/impl/v7/ConfigV7.java index faeb5d2432..4028719379 100644 --- a/src/main/java/org/opensearch/security/securityconf/impl/v7/ConfigV7.java +++ b/src/main/java/org/opensearch/security/securityconf/impl/v7/ConfigV7.java @@ -39,10 +39,13 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException; import org.opensearch.security.DefaultObjectMapper; import org.opensearch.security.auth.internal.InternalAuthenticationBackend; import org.opensearch.security.securityconf.impl.v6.ConfigV6; +import org.opensearch.security.setting.DeprecatedSettings; public class ConfigV7 { @@ -293,7 +296,6 @@ public static class AuthcDomain { @JsonInclude(JsonInclude.Include.NON_NULL) public boolean http_enabled = true; @JsonInclude(JsonInclude.Include.NON_NULL) - public boolean transport_enabled = true; // public boolean enabled= true; public int order = 0; public HttpAuthenticator http_authenticator = new HttpAuthenticator(); @@ -307,10 +309,8 @@ public AuthcDomain() { public AuthcDomain(ConfigV6.AuthcDomain v6) { super(); http_enabled = v6.http_enabled && v6.enabled; - transport_enabled = v6.transport_enabled && v6.enabled; // if(v6.enabled)vv { // http_enabled = true; - // transport_enabled = true; // } order = v6.order; http_authenticator = new HttpAuthenticator(v6.http_authenticator); @@ -322,8 +322,6 @@ public AuthcDomain(ConfigV6.AuthcDomain v6) { public String toString() { return "AuthcDomain [http_enabled=" + http_enabled - + ", transport_enabled=" - + transport_enabled + ", order=" + order + ", http_authenticator=" @@ -335,6 +333,31 @@ public String toString() { + "]"; } + @JsonAnySetter + public void unknownPropertiesHandler(String name, Object value) throws JsonMappingException { + switch (name) { + case "transport_enabled": + DeprecatedSettings.logCustomDeprecationMessage( + String.format( + "In AuthcDomain, using http_authenticator=%s, authentication_backend=%s", + http_authenticator, + authentication_backend + ), + name + ); + break; + default: + throw new UnrecognizedPropertyException( + null, + "Unrecognized field " + name + " present in the input data for AuthcDomain config", + null, + AuthcDomain.class, + name, + null + ); + } + } + } public static class HttpAuthenticator { @@ -451,8 +474,6 @@ public String toString() { public static class AuthzDomain { @JsonInclude(JsonInclude.Include.NON_NULL) public boolean http_enabled = true; - @JsonInclude(JsonInclude.Include.NON_NULL) - public boolean transport_enabled = true; public AuthzBackend authorization_backend = new AuthzBackend(); public String description; @@ -462,7 +483,6 @@ public AuthzDomain() { public AuthzDomain(ConfigV6.AuthzDomain v6) { http_enabled = v6.http_enabled && v6.enabled; - transport_enabled = v6.transport_enabled && v6.enabled; authorization_backend = new AuthzBackend(v6.authorization_backend); description = "Migrated from v6"; } @@ -471,8 +491,6 @@ public AuthzDomain(ConfigV6.AuthzDomain v6) { public String toString() { return "AuthzDomain [http_enabled=" + http_enabled - + ", transport_enabled=" - + transport_enabled + ", authorization_backend=" + authorization_backend + ", description=" @@ -480,6 +498,26 @@ public String toString() { + "]"; } + @JsonAnySetter + public void unknownPropertiesHandler(String name, Object value) throws JsonMappingException { + switch (name) { + case "transport_enabled": + DeprecatedSettings.logCustomDeprecationMessage( + String.format("In AuthzDomain, using authorization_backend=%s", authorization_backend), + name + ); + break; + default: + throw new UnrecognizedPropertyException( + null, + "Unrecognized field " + name + " present in the input data for AuthzDomain config", + null, + AuthzDomain.class, + name, + null + ); + } + } } public static class OnBehalfOfSettings { diff --git a/src/main/java/org/opensearch/security/setting/DeprecatedSettings.java b/src/main/java/org/opensearch/security/setting/DeprecatedSettings.java index b415dc7c7f..91eb96abdd 100644 --- a/src/main/java/org/opensearch/security/setting/DeprecatedSettings.java +++ b/src/main/java/org/opensearch/security/setting/DeprecatedSettings.java @@ -5,6 +5,7 @@ package org.opensearch.security.setting; +import org.opensearch.Version; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Settings; @@ -28,4 +29,18 @@ public static void checkForDeprecatedSetting(final Settings settings, final Stri ); } } + + /** + * Logs that a specific setting is deprecated, including a specific supplemental message parameter containing information that details where this setting can be removed from. Should be used in cases where a setting is not supported by the codebase and processing it would introduce errors on setup. + */ + public static void logCustomDeprecationMessage(final String deprecationLocationInformation, final String deprecatedSettingKey) { + DEPRECATION_LOGGER.deprecate( + deprecatedSettingKey, + "In OpenSearch " + + Version.CURRENT + + " the setting '{}' is deprecated, it should be removed from the relevant config file using the following location information: " + + deprecationLocationInformation, + deprecatedSettingKey + ); + } } diff --git a/src/test/java/org/opensearch/security/setting/DeprecatedSettingsTest.java b/src/test/java/org/opensearch/security/setting/DeprecatedSettingsTest.java index a0f9558228..3fa8e45816 100644 --- a/src/test/java/org/opensearch/security/setting/DeprecatedSettingsTest.java +++ b/src/test/java/org/opensearch/security/setting/DeprecatedSettingsTest.java @@ -5,17 +5,22 @@ package org.opensearch.security.setting; +import com.fasterxml.jackson.databind.JsonMappingException; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.opensearch.Version; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Settings; +import org.opensearch.security.securityconf.impl.CType; +import org.opensearch.security.support.ConfigHelper; import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; +import static org.opensearch.security.configuration.ConfigurationRepository.DEFAULT_CONFIG_VERSION; import static org.opensearch.security.setting.DeprecatedSettings.checkForDeprecatedSetting; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; @@ -61,4 +66,96 @@ public void testCheckForDeprecatedSettingFoundLegacy() { verify(logger).deprecate(eq("legacyKey"), anyString(), any(), any()); } + + @Test + public void testForTransportEnabledDeprecationMessageOnYamlLoad() throws Exception { + ConfigHelper.fromYamlString( + "---\n" + + "_meta:\n" + + " type: \"config\"\n" + + " config_version: 2\n" + + "config:\n" + + " dynamic:\n" + + " authc:\n" + + " authentication_domain_kerb:\n" + + " http_enabled: false\n" + + " transport_enabled: false\n" + + " order: 3\n" + + " http_authenticator:\n" + + " challenge: true\n" + + " type: \"kerberos\"\n" + + " config: {}\n" + + " authentication_backend:\n" + + " type: \"noop\"\n" + + " config: {}\n" + + " description: \"Migrated from v6\"\n" + + " authz:\n" + + " roles_from_xxx:\n" + + " http_enabled: false\n" + + " transport_enabled: false\n" + + " authorization_backend:\n" + + " type: \"xxx\"\n" + + " config: {}\n" + + " description: \"Migrated from v6\"", + CType.CONFIG, + DEFAULT_CONFIG_VERSION, + 0, + 0 + ); + verify(logger).deprecate( + "transport_enabled", + "In OpenSearch " + + Version.CURRENT + + " the setting '{}' is deprecated, it should be removed from the relevant config file using the following location information: In AuthcDomain, using http_authenticator=HttpAuthenticator [challenge=true, type=null, config={}], authentication_backend=AuthcBackend [type=org.opensearch.security.auth.internal.InternalAuthenticationBackend, config={}]", + "transport_enabled" + ); + verify(logger).deprecate( + "transport_enabled", + "In OpenSearch " + + Version.CURRENT + + " the setting '{}' is deprecated, it should be removed from the relevant config file using the following location information: In AuthzDomain, using authorization_backend=AuthzBackend [type=noop, config={}]", + "transport_enabled" + ); + } + + @Test + public void testForExceptionOnUnknownAuthcAuthzSettingsOnYamlLoad() throws Exception { + try { + ConfigHelper.fromYamlString( + "---\n" + + "_meta:\n" + + " type: \"config\"\n" + + " config_version: 2\n" + + "config:\n" + + " dynamic:\n" + + " authc:\n" + + " authentication_domain_kerb:\n" + + " http_enabled: false\n" + + " unknown_property: false\n" + + " order: 3\n" + + " http_authenticator:\n" + + " challenge: true\n" + + " type: \"kerberos\"\n" + + " config: {}\n" + + " authentication_backend:\n" + + " type: \"noop\"\n" + + " config: {}\n" + + " description: \"Migrated from v6\"\n" + + " authz:\n" + + " roles_from_xxx:\n" + + " http_enabled: false\n" + + " unknown_property: false\n" + + " authorization_backend:\n" + + " type: \"xxx\"\n" + + " config: {}\n" + + " description: \"Migrated from v6\"", + CType.CONFIG, + DEFAULT_CONFIG_VERSION, + 0, + 0 + ); + } catch (JsonMappingException e) { + verifyNoInteractions(logger); + } + } } diff --git a/src/test/resources/restapi/invalid_config.json b/src/test/resources/restapi/invalid_config.json index 7bbbf2201f..1d43e1edab 100644 --- a/src/test/resources/restapi/invalid_config.json +++ b/src/test/resources/restapi/invalid_config.json @@ -23,7 +23,6 @@ "authc":{ "authentication_domain_kerb":{ "http_enabled":false, - "transport_enabled":false, "order":3, "http_authenticator":{ "challenge":true, @@ -42,7 +41,6 @@ }, "authentication_domain_clientcert":{ "http_enabled":false, - "transport_enabled":false, "order":1, "http_authenticator":{ "challenge":true, @@ -61,7 +59,6 @@ }, "authentication_domain_proxy":{ "http_enabled":false, - "transport_enabled":false, "order":2, "http_authenticator":{ "challenge":true, @@ -81,7 +78,6 @@ }, "authentication_domain_basic_internal":{ "http_enabled":true, - "transport_enabled":true, "order":0, "http_authenticator":{ "challenge":true, @@ -102,7 +98,6 @@ "authz":{ "roles_from_xxx":{ "http_enabled":false, - "transport_enabled":false, "authorization_backend":{ "type":"xxx", "config":{ @@ -113,7 +108,6 @@ }, "roles_from_myldap":{ "http_enabled":false, - "transport_enabled":false, "authorization_backend":{ "type":"ldap", "config":{ diff --git a/src/test/resources/restapi/security_config.json b/src/test/resources/restapi/security_config.json index e5c09050cc..30b8611e5a 100644 --- a/src/test/resources/restapi/security_config.json +++ b/src/test/resources/restapi/security_config.json @@ -23,7 +23,6 @@ "authc":{ "authentication_domain_kerb":{ "http_enabled":false, - "transport_enabled":false, "order":3, "http_authenticator":{ "challenge":true, @@ -42,7 +41,6 @@ }, "authentication_domain_clientcert":{ "http_enabled":false, - "transport_enabled":false, "order":1, "http_authenticator":{ "challenge":true, @@ -61,7 +59,6 @@ }, "authentication_domain_proxy":{ "http_enabled":false, - "transport_enabled":false, "order":2, "http_authenticator":{ "challenge":true, @@ -81,7 +78,6 @@ }, "authentication_domain_basic_internal":{ "http_enabled":true, - "transport_enabled":true, "order":0, "http_authenticator":{ "challenge":true, @@ -102,7 +98,6 @@ "authz":{ "roles_from_xxx":{ "http_enabled":false, - "transport_enabled":false, "authorization_backend":{ "type":"xxx", "config":{ @@ -113,7 +108,6 @@ }, "roles_from_myldap":{ "http_enabled":false, - "transport_enabled":false, "authorization_backend":{ "type":"ldap", "config":{ diff --git a/src/test/resources/restapi/securityconfig.json b/src/test/resources/restapi/securityconfig.json index 4e4b1bba63..a577cb2a30 100644 --- a/src/test/resources/restapi/securityconfig.json +++ b/src/test/resources/restapi/securityconfig.json @@ -23,7 +23,6 @@ "authc":{ "authentication_domain_saml": { "http_enabled" : true, - "transport_enabled" : false, "order" : 5, "http_authenticator" : { "challenge" : true, @@ -44,7 +43,6 @@ }, "authentication_domain_kerb":{ "http_enabled":false, - "transport_enabled":false, "order":3, "http_authenticator":{ "challenge":true, @@ -63,7 +61,6 @@ }, "authentication_domain_clientcert":{ "http_enabled":false, - "transport_enabled":false, "order":1, "http_authenticator":{ "challenge":true, @@ -82,7 +79,6 @@ }, "authentication_domain_proxy":{ "http_enabled":false, - "transport_enabled":false, "order":2, "http_authenticator":{ "challenge":true, @@ -102,7 +98,6 @@ }, "authentication_domain_basic_internal":{ "http_enabled":true, - "transport_enabled":true, "order":0, "http_authenticator":{ "challenge":true, @@ -123,7 +118,6 @@ "authz":{ "roles_from_xxx":{ "http_enabled":false, - "transport_enabled":false, "authorization_backend":{ "type":"xxx", "config":{ @@ -134,7 +128,6 @@ }, "roles_from_myldap":{ "http_enabled":false, - "transport_enabled":false, "authorization_backend":{ "type":"ldap", "config":{ diff --git a/src/test/resources/restapi/securityconfig_nondefault.json b/src/test/resources/restapi/securityconfig_nondefault.json index a5660c6496..2482e99674 100644 --- a/src/test/resources/restapi/securityconfig_nondefault.json +++ b/src/test/resources/restapi/securityconfig_nondefault.json @@ -22,7 +22,6 @@ "authc" : { "jwt_auth_domain" : { "http_enabled" : true, - "transport_enabled" : true, "order" : 0, "http_authenticator" : { "challenge" : false, @@ -40,7 +39,6 @@ }, "ldap" : { "http_enabled" : false, - "transport_enabled" : false, "order" : 5, "http_authenticator" : { "challenge" : false, @@ -65,7 +63,6 @@ }, "basic_internal_auth_domain" : { "http_enabled" : true, - "transport_enabled" : true, "order" : 4, "http_authenticator" : { "challenge" : true, @@ -80,7 +77,6 @@ }, "proxy_auth_domain" : { "http_enabled" : false, - "transport_enabled" : false, "order" : 3, "http_authenticator" : { "challenge" : false, @@ -98,7 +94,6 @@ }, "clientcert_auth_domain" : { "http_enabled" : false, - "transport_enabled" : false, "order" : 2, "http_authenticator" : { "challenge" : false, @@ -115,7 +110,6 @@ }, "kerberos_auth_domain" : { "http_enabled" : false, - "transport_enabled" : false, "order" : 6, "http_authenticator" : { "challenge" : true, @@ -134,7 +128,6 @@ "authz" : { "roles_from_another_ldap" : { "http_enabled" : false, - "transport_enabled" : false, "authorization_backend" : { "type" : "ldap", "config" : { } @@ -143,7 +136,6 @@ }, "roles_from_myldap" : { "http_enabled" : false, - "transport_enabled" : false, "authorization_backend" : { "type" : "ldap", "config" : { From 31242741c0daff2857fc0ee0b82f08bd515ce196 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jan 2024 08:20:35 -0500 Subject: [PATCH 092/204] Bump com.fasterxml.woodstox:woodstox-core from 6.5.1 to 6.6.0 (#3967) Bumps [com.fasterxml.woodstox:woodstox-core](https://github.com/FasterXML/woodstox) from 6.5.1 to 6.6.0.
Commits
  • dd590bb [maven-release-plugin] prepare release woodstox-core-6.6.0
  • c3ffe88 ...
  • 57d0c4b Update branch to 6.6.1-SNAPSHOT
  • 6ba21f8 Re-sync version to branch 6.6
  • 8daa669 Update Maven version for Maven wrapper
  • acad270 [maven-release-plugin] prepare for next development iteration
  • 36f48bf [maven-release-plugin] prepare release woodstox-core-6.6.0
  • 8bad94c Prepare for 6.6.0 release
  • 172371f Add support to optionally allow surrogate pair entities (#165) (#174)
  • 37232fc Test renaming (newer naming convention)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.fasterxml.woodstox:woodstox-core&package-manager=gradle&previous-version=6.5.1&new-version=6.6.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index ea810c1e17..6c14a25084 100644 --- a/build.gradle +++ b/build.gradle @@ -649,7 +649,7 @@ dependencies { runtimeOnly 'org.xerial.snappy:snappy-java:1.1.10.5' runtimeOnly 'org.codehaus.woodstox:stax2-api:4.2.2' runtimeOnly "org.glassfish.jaxb:txw2:${jaxb_version}" - runtimeOnly 'com.fasterxml.woodstox:woodstox-core:6.5.1' + runtimeOnly 'com.fasterxml.woodstox:woodstox-core:6.6.0' runtimeOnly 'org.apache.ws.xmlschema:xmlschema-core:2.3.1' runtimeOnly 'org.apache.santuario:xmlsec:2.3.4' runtimeOnly "com.github.luben:zstd-jni:${versions.zstd}" From 843071be10a9e41b86d76636f5fc948f3ffa5a4b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jan 2024 08:20:59 -0500 Subject: [PATCH 093/204] Bump io.dropwizard.metrics:metrics-core from 4.2.23 to 4.2.24 (#3968) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.dropwizard.metrics:metrics-core](https://github.com/dropwizard/metrics) from 4.2.23 to 4.2.24.
Release notes

Sourced from io.dropwizard.metrics:metrics-core's releases.

v4.2.24

What's Changed

Full Changelog: https://github.com/dropwizard/metrics/compare/v4.2.23...v4.2.24

Commits
  • eab9d99 [maven-release-plugin] prepare release v4.2.24
  • 7a7f124 Fix IndexOutOfBoundsException in Jetty 9, 10, 11, 12 InstrumentedHandler (#3912)
  • a964779 Update github/codeql-action digest to 0b21cf2 (#3907)
  • 3d2d650 Update actions/cache action to v4 (#3908)
  • 5819379 Update dependency org.cyclonedx:cyclonedx-maven-plugin to v2.7.11
  • 4ad325e Update dependency org.mockito:mockito-core to v5.9.0 (#3902)
  • 8d85504 Update actions/cache action to v3.3.3
  • b47611a Update dependency org.apache.maven.plugins:maven-surefire-plugin to v3.2.5
  • c651019 Update slf4j.version to v2.0.11
  • eec8f6c Update github/codeql-action digest to e5f05b8 (#3890)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.dropwizard.metrics:metrics-core&package-manager=gradle&previous-version=4.2.23&new-version=4.2.24)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 6c14a25084..67b9b60a9c 100644 --- a/build.gradle +++ b/build.gradle @@ -643,7 +643,7 @@ dependencies { runtimeOnly 'com.google.j2objc:j2objc-annotations:2.8' compileOnly 'com.google.code.findbugs:jsr305:3.0.2' runtimeOnly 'org.lz4:lz4-java:1.8.0' - runtimeOnly 'io.dropwizard.metrics:metrics-core:4.2.23' + runtimeOnly 'io.dropwizard.metrics:metrics-core:4.2.24' runtimeOnly 'org.slf4j:slf4j-api:1.7.36' runtimeOnly "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}" runtimeOnly 'org.xerial.snappy:snappy-java:1.1.10.5' From 35229fbe5c570918f6a6f92f25621b88a611277e Mon Sep 17 00:00:00 2001 From: Xun Zhang Date: Tue, 23 Jan 2024 07:09:39 -0800 Subject: [PATCH 094/204] Adds new ml-commons system indices to the list (#3973) ### Description Adds renamed ml-commons systems indices to the list of existing system indices. This change is required so that security plugin correctly recognizes the new indices as system indices. * Category - Maintenance Signed-off-by: Xun Zhang --- .../security/tools/democonfig/SecuritySettingsConfigurer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java index 9c51fbe1d4..a68c93f03f 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java @@ -55,6 +55,8 @@ public class SecuritySettingsConfigurer { ".plugins-ml-task", ".plugins-ml-conversation-meta", ".plugins-ml-conversation-interactions", + ".plugins-ml-memory-meta", + ".plugins-ml-memory-message", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", From dadd03fcd7a856df895bb2299761fc7783dfe62b Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 23 Jan 2024 12:00:30 -0500 Subject: [PATCH 095/204] Fix: remove unnecessary trailing slashes in APIs. (#3976) ### Description Coming from https://github.com/opensearch-project/opensearch-api-specification/pull/179 which flags a couple of false positives because of mismatched trailing slash. ### Check List - [x] New functionality includes testing - [x] New functionality has been documented - [x] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). Signed-off-by: dblock --- .../security/dlic/rest/api/ActionGroupsApiAction.java | 6 +++--- .../opensearch/security/dlic/rest/api/AuditApiAction.java | 4 ++-- .../security/dlic/rest/api/InternalUsersApiAction.java | 6 +++--- .../opensearch/security/dlic/rest/api/NodesDnApiAction.java | 4 ++-- .../opensearch/security/dlic/rest/api/RolesApiAction.java | 4 ++-- .../security/dlic/rest/api/RolesMappingApiAction.java | 4 ++-- .../security/dlic/rest/api/SecuritySSLCertsApiAction.java | 2 +- .../opensearch/security/dlic/rest/api/TenantsApiAction.java | 4 ++-- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/ActionGroupsApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/ActionGroupsApiAction.java index 172d4a537b..3032054e64 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/ActionGroupsApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/ActionGroupsApiAction.java @@ -46,16 +46,16 @@ public class ActionGroupsApiAction extends AbstractApiAction { // legacy mapping for backwards compatibility // TODO: remove in next version new Route(Method.GET, "/actiongroup/{name}"), - new Route(Method.GET, "/actiongroup/"), + new Route(Method.GET, "/actiongroup"), new Route(Method.DELETE, "/actiongroup/{name}"), new Route(Method.PUT, "/actiongroup/{name}"), // corrected mapping, introduced in OpenSearch Security new Route(Method.GET, "/actiongroups/{name}"), - new Route(Method.GET, "/actiongroups/"), + new Route(Method.GET, "/actiongroups"), new Route(Method.DELETE, "/actiongroups/{name}"), new Route(Method.PUT, "/actiongroups/{name}"), - new Route(Method.PATCH, "/actiongroups/"), + new Route(Method.PATCH, "/actiongroups"), new Route(Method.PATCH, "/actiongroups/{name}") ) diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/AuditApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/AuditApiAction.java index 47bc1f184e..997bd85bdd 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/AuditApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/AuditApiAction.java @@ -123,9 +123,9 @@ public class AuditApiAction extends AbstractApiAction { private static final List routes = addRoutesPrefix( ImmutableList.of( - new Route(RestRequest.Method.GET, "/audit/"), + new Route(RestRequest.Method.GET, "/audit"), new Route(RestRequest.Method.PUT, "/audit/config"), - new Route(RestRequest.Method.PATCH, "/audit/") + new Route(RestRequest.Method.PATCH, "/audit") ) ); diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/InternalUsersApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/InternalUsersApiAction.java index 70994504bf..3cbcc18bd9 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/InternalUsersApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/InternalUsersApiAction.java @@ -64,18 +64,18 @@ protected void consumeParameters(final RestRequest request) { private static final List routes = addRoutesPrefix( ImmutableList.of( new Route(Method.GET, "/user/{name}"), - new Route(Method.GET, "/user/"), + new Route(Method.GET, "/user"), new Route(Method.POST, "/user/{name}/authtoken"), new Route(Method.DELETE, "/user/{name}"), new Route(Method.PUT, "/user/{name}"), // corrected mapping, introduced in OpenSearch Security new Route(Method.GET, "/internalusers/{name}"), - new Route(Method.GET, "/internalusers/"), + new Route(Method.GET, "/internalusers"), new Route(Method.POST, "/internalusers/{name}/authtoken"), new Route(Method.DELETE, "/internalusers/{name}"), new Route(Method.PUT, "/internalusers/{name}"), - new Route(Method.PATCH, "/internalusers/"), + new Route(Method.PATCH, "/internalusers"), new Route(Method.PATCH, "/internalusers/{name}") ) ); diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/NodesDnApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/NodesDnApiAction.java index 05c533b1d9..ff44867bd2 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/NodesDnApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/NodesDnApiAction.java @@ -62,10 +62,10 @@ public class NodesDnApiAction extends AbstractApiAction { private static final List routes = addRoutesPrefix( ImmutableList.of( new Route(Method.GET, "/nodesdn/{name}"), - new Route(Method.GET, "/nodesdn/"), + new Route(Method.GET, "/nodesdn"), new Route(Method.DELETE, "/nodesdn/{name}"), new Route(Method.PUT, "/nodesdn/{name}"), - new Route(Method.PATCH, "/nodesdn/"), + new Route(Method.PATCH, "/nodesdn"), new Route(Method.PATCH, "/nodesdn/{name}") ) ); diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/RolesApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/RolesApiAction.java index 9af04d17ec..50fac9b80c 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/RolesApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/RolesApiAction.java @@ -45,11 +45,11 @@ public class RolesApiAction extends AbstractApiAction { private static final List routes = addRoutesPrefix( ImmutableList.of( - new Route(Method.GET, "/roles/"), + new Route(Method.GET, "/roles"), new Route(Method.GET, "/roles/{name}"), new Route(Method.DELETE, "/roles/{name}"), new Route(Method.PUT, "/roles/{name}"), - new Route(Method.PATCH, "/roles/"), + new Route(Method.PATCH, "/roles"), new Route(Method.PATCH, "/roles/{name}") ) ); diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/RolesMappingApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/RolesMappingApiAction.java index 230ce0e1a1..b980a1e4ba 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/RolesMappingApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/RolesMappingApiAction.java @@ -38,11 +38,11 @@ public class RolesMappingApiAction extends AbstractApiAction { private static final List routes = addRoutesPrefix( ImmutableList.of( - new Route(Method.GET, "/rolesmapping/"), + new Route(Method.GET, "/rolesmapping"), new Route(Method.GET, "/rolesmapping/{name}"), new Route(Method.DELETE, "/rolesmapping/{name}"), new Route(Method.PUT, "/rolesmapping/{name}"), - new Route(Method.PATCH, "/rolesmapping/"), + new Route(Method.PATCH, "/rolesmapping"), new Route(Method.PATCH, "/rolesmapping/{name}") ) ); diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/SecuritySSLCertsApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/SecuritySSLCertsApiAction.java index 48e1c9b704..e60070288e 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/SecuritySSLCertsApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/SecuritySSLCertsApiAction.java @@ -50,7 +50,7 @@ */ public class SecuritySSLCertsApiAction extends AbstractApiAction { private static final List ROUTES = addRoutesPrefix( - ImmutableList.of(new Route(Method.GET, "/ssl/certs"), new Route(Method.PUT, "/ssl/{certType}/reloadcerts/")) + ImmutableList.of(new Route(Method.GET, "/ssl/certs"), new Route(Method.PUT, "/ssl/{certType}/reloadcerts")) ); private final SecurityKeyStore securityKeyStore; diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/TenantsApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/TenantsApiAction.java index 28fd6dcdcb..e16d31ba6f 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/TenantsApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/TenantsApiAction.java @@ -50,10 +50,10 @@ public class TenantsApiAction extends AbstractApiAction { private static final List routes = addRoutesPrefix( ImmutableList.of( new Route(Method.GET, "/tenants/{name}"), - new Route(Method.GET, "/tenants/"), + new Route(Method.GET, "/tenants"), new Route(Method.DELETE, "/tenants/{name}"), new Route(Method.PUT, "/tenants/{name}"), - new Route(Method.PATCH, "/tenants/"), + new Route(Method.PATCH, "/tenants"), new Route(Method.PATCH, "/tenants/{name}") ) ); From d44113808659ddc8e57f8fd3f67ba343cc4f462f Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Fri, 26 Jan 2024 11:43:35 -0500 Subject: [PATCH 096/204] Bump spotless (6.24.0 -> 6.25.0) to bump eclipse resources (3.18 -> 3.19) (#3992) ### Description This PR bumps spotless to bump the transient dependency on org.eclipse.platform:org.eclipse.core.resources@3.18.100 -> org.eclipse.platform:org.eclipse.core.resources@3.19.100. In turn this should stop scanners from reporting the project as vulnerable to: https://nvd.nist.gov/vuln/detail/CVE-2023-4218. I was not able to easily move just the Eclipse dependency because it seems that the package causing the flagging org.eclipse.platform:org.eclipse.core.resources@3.18.100 does not have a straight path forward to the recommended versions listed on the CVE. However, https://security.snyk.io/package/maven/org.eclipse.platform:org.eclipse.core.resources/3.19.100 reports that this version should remove the issue while https://security.snyk.io/package/maven/org.eclipse.platform:org.eclipse.core.resources/3.18.100 will cause the flag. One note: We should not actually be concerned about this issue as it is related to Eclipse IDE behavior and nothing to do with the type of dependency on the Eclipse packages like we have. ### Check List - [ ] ~New functionality includes testing~ - [ ] ~New functionality has been documented~ - [x] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). Signed-off-by: Stephen Crawford --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 67b9b60a9c..db7af24910 100644 --- a/build.gradle +++ b/build.gradle @@ -62,7 +62,7 @@ plugins { id 'idea' id 'jacoco' id 'maven-publish' - id 'com.diffplug.spotless' version '6.24.0' + id 'com.diffplug.spotless' version '6.25.0' id 'checkstyle' id 'com.netflix.nebula.ospackage' version "11.6.0" id "org.gradle.test-retry" version "1.5.8" From af7c7e4c6fe64b9719dad09acf2eec907afb31cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jan 2024 14:32:56 +0100 Subject: [PATCH 097/204] Bump io.dropwizard.metrics:metrics-core from 4.2.24 to 4.2.25 (#3996) Bumps [io.dropwizard.metrics:metrics-core](https://github.com/dropwizard/metrics) from 4.2.24 to 4.2.25.
Commits
  • 7c2ffc5 [maven-release-plugin] prepare release v4.2.25
  • b116e89 Jakarta HealthCheckServlet object mapper and status indicator (#3924)
  • 5255717 Update dependency org.assertj:assertj-core to v3.25.2
  • 0aada3f Update dependency org.apache.httpcomponents.client5:httpclient5 to v5.3.1
  • 29ce821 [maven-release-plugin] prepare for next development iteration
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.dropwizard.metrics:metrics-core&package-manager=gradle&previous-version=4.2.24&new-version=4.2.25)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index db7af24910..6659bc8470 100644 --- a/build.gradle +++ b/build.gradle @@ -643,7 +643,7 @@ dependencies { runtimeOnly 'com.google.j2objc:j2objc-annotations:2.8' compileOnly 'com.google.code.findbugs:jsr305:3.0.2' runtimeOnly 'org.lz4:lz4-java:1.8.0' - runtimeOnly 'io.dropwizard.metrics:metrics-core:4.2.24' + runtimeOnly 'io.dropwizard.metrics:metrics-core:4.2.25' runtimeOnly 'org.slf4j:slf4j-api:1.7.36' runtimeOnly "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}" runtimeOnly 'org.xerial.snappy:snappy-java:1.1.10.5' From 4525c958972cbee29e047a8623263c1413c31654 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jan 2024 14:33:34 +0100 Subject: [PATCH 098/204] Bump jjwt_version from 0.12.3 to 0.12.4 (#3995) Bumps `jjwt_version` from 0.12.3 to 0.12.4. Updates `io.jsonwebtoken:jjwt-api` from 0.12.3 to 0.12.4
Release notes

Sourced from io.jsonwebtoken:jjwt-api's releases.

0.12.4

This is patch release completes 10 issues, with two especially noteworthy changes, and a number of other smaller bug fixes and enhancements.

  1. The default Jackson deserializer will now reject duplicate JSON members by default in an attempt to be a little more strict at rejecting potentially malicious or malformed JSON. This is a default and can be overridden with a custom ObjectMapper if desired.
  2. Password-based JWE encryption key algorithms (PBES2_HS256_A128KW, PBES2_HS384_A192KW and PBES2_HS512_A256KW) now enforce an upper bound (maximum) number of iterations allowed during decryption to mitigate against potential DoS attacks. Many thanks to Jingcheng Yang and Jianjun Chen from Sichuan University and Zhongguancun Lab for their work on this!

A number of other issues fixed: thread-safe ServiceLoader usage for dynamic JSON processor lookup, Android enhancements for JSON Reader APIs, fixed Elliptic Curve field element padding, and more. Please read the 0.12.4 CHANGELOG for full details of all of these changes, and as always, project documentation is in the 0.12.4 README.

Please allow 30 minutes from the time this announcement is published for the release to be available in Maven Central.

Changelog

Sourced from io.jsonwebtoken:jjwt-api's changelog.

0.12.4

This patch release includes various changes listed below.

Jackson Default Parsing Behavior

This release makes two behavioral changes to JJWT's default Jackson ObjectMapper parsing settings:

  1. In the interest of having stronger standards to reject potentially malformed/malicious/accidental JSON that could have undesirable effects on an application, JJWT's default ObjectMapper is now configured to explicitly reject/fail parsing JSON (JWT headers and/or Claims) if/when that JSON contains duplicate JSON member names.

    For example, now the following JSON, if parsed, would fail (be rejected) by default:

    {
  "hello": "world",
  "thisWillFail": 42,
  "thisWillFail": "test"
}

    Technically, the JWT RFCs do allow duplicate named fields as long as the last parsed member is the one used (see JWS RFC 7515, Section 4), so this is allowed. However, because JWTs often reflect security concepts, it's usually better to be defensive and reject these unexpected scenarios by default. The RFC later supports this position/preference in Section 10.12:

    Ambiguous and potentially exploitable situations
could arise if the JSON parser used does not enforce the uniqueness
of member names or returns an unpredictable value for duplicate
member names.

    Finally, this is just a default, and the RFC does indeed allow duplicate member names if the last value is used, so applications that require duplicates to be allowed can simply configure their own ObjectMapper and use that with JJWT instead of assuming this (new) JJWT default. See [Issue #877](jwtk/jjwt#877) for more.

  2. If using JJWT's support to use Jackson to parse Custom Claim Types (for example, a Claim that should be unmarshalled into a POJO), and the JSON for that POJO contained a member that is not represented in the specified class, Jackson would fail parsing by default. Because POJOs and JSON data models can sometimes be out of sync due to different class versions, the default behavior has been changed to ignore these unknown JSON members instead of failing (i.e. the ObjectMapper's DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES is now set to false) by default.

    Again, if you prefer the stricter behavior of rejecting JSON with extra or unknown properties, you can configure true on your own ObjectMapper instance and use that instance with the Jwts.parser() builder.

Additional Changes

This release also:

... (truncated)

Commits
  • bf4168c [maven-release-plugin] prepare release 0.12.4
  • 5c6dec0 - Adding 0.12.4 release version references
  • dd10b12 Added JWK Set documentation to README.mdJwkset doc (#912)
  • 6335381 PBES2 decryption maximum iterations (#911)
  • 2884eb7 - Updating to GitHub latest actions/checkout and actions/setup-java script ve...
  • 628bd6f Secret JWK k values larger than HMAC-SHA minimums (#909)
  • b12dabf Fix small typos (#908)
  • 26f5dc3 Updating changelog with more information/clarity for the 0.12.4 release (#907)
  • f61cfa8 Test case change to reflect accurate assertion for Elliptic Curve 'd' values ...
  • fd619e0 disable FAIL_ON_UNKNOWN_PROPERTIES deserialization feature of Jackson by defa...
  • Additional commits viewable in compare view

Updates `io.jsonwebtoken:jjwt-impl` from 0.12.3 to 0.12.4
Release notes

Sourced from io.jsonwebtoken:jjwt-impl's releases.

0.12.4

This is patch release completes 10 issues, with two especially noteworthy changes, and a number of other smaller bug fixes and enhancements.

  1. The default Jackson deserializer will now reject duplicate JSON members by default in an attempt to be a little more strict at rejecting potentially malicious or malformed JSON. This is a default and can be overridden with a custom ObjectMapper if desired.
  2. Password-based JWE encryption key algorithms (PBES2_HS256_A128KW, PBES2_HS384_A192KW and PBES2_HS512_A256KW) now enforce an upper bound (maximum) number of iterations allowed during decryption to mitigate against potential DoS attacks. Many thanks to Jingcheng Yang and Jianjun Chen from Sichuan University and Zhongguancun Lab for their work on this!

A number of other issues fixed: thread-safe ServiceLoader usage for dynamic JSON processor lookup, Android enhancements for JSON Reader APIs, fixed Elliptic Curve field element padding, and more. Please read the 0.12.4 CHANGELOG for full details of all of these changes, and as always, project documentation is in the 0.12.4 README.

Please allow 30 minutes from the time this announcement is published for the release to be available in Maven Central.

Changelog

Sourced from io.jsonwebtoken:jjwt-impl's changelog.

0.12.4

This patch release includes various changes listed below.

Jackson Default Parsing Behavior

This release makes two behavioral changes to JJWT's default Jackson ObjectMapper parsing settings:

  1. In the interest of having stronger standards to reject potentially malformed/malicious/accidental JSON that could have undesirable effects on an application, JJWT's default ObjectMapper is now configured to explicitly reject/fail parsing JSON (JWT headers and/or Claims) if/when that JSON contains duplicate JSON member names.

    For example, now the following JSON, if parsed, would fail (be rejected) by default:

    {
  "hello": "world",
  "thisWillFail": 42,
  "thisWillFail": "test"
}

    Technically, the JWT RFCs do allow duplicate named fields as long as the last parsed member is the one used (see JWS RFC 7515, Section 4), so this is allowed. However, because JWTs often reflect security concepts, it's usually better to be defensive and reject these unexpected scenarios by default. The RFC later supports this position/preference in Section 10.12:

    Ambiguous and potentially exploitable situations
could arise if the JSON parser used does not enforce the uniqueness
of member names or returns an unpredictable value for duplicate
member names.

    Finally, this is just a default, and the RFC does indeed allow duplicate member names if the last value is used, so applications that require duplicates to be allowed can simply configure their own ObjectMapper and use that with JJWT instead of assuming this (new) JJWT default. See [Issue #877](jwtk/jjwt#877) for more.

  2. If using JJWT's support to use Jackson to parse Custom Claim Types (for example, a Claim that should be unmarshalled into a POJO), and the JSON for that POJO contained a member that is not represented in the specified class, Jackson would fail parsing by default. Because POJOs and JSON data models can sometimes be out of sync due to different class versions, the default behavior has been changed to ignore these unknown JSON members instead of failing (i.e. the ObjectMapper's DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES is now set to false) by default.

    Again, if you prefer the stricter behavior of rejecting JSON with extra or unknown properties, you can configure true on your own ObjectMapper instance and use that instance with the Jwts.parser() builder.

Additional Changes

This release also:

... (truncated)

Commits
  • bf4168c [maven-release-plugin] prepare release 0.12.4
  • 5c6dec0 - Adding 0.12.4 release version references
  • dd10b12 Added JWK Set documentation to README.mdJwkset doc (#912)
  • 6335381 PBES2 decryption maximum iterations (#911)
  • 2884eb7 - Updating to GitHub latest actions/checkout and actions/setup-java script ve...
  • 628bd6f Secret JWK k values larger than HMAC-SHA minimums (#909)
  • b12dabf Fix small typos (#908)
  • 26f5dc3 Updating changelog with more information/clarity for the 0.12.4 release (#907)
  • f61cfa8 Test case change to reflect accurate assertion for Elliptic Curve 'd' values ...
  • fd619e0 disable FAIL_ON_UNKNOWN_PROPERTIES deserialization feature of Jackson by defa...
  • Additional commits viewable in compare view

Updates `io.jsonwebtoken:jjwt-jackson` from 0.12.3 to 0.12.4 Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 6659bc8470..e0ae931d5a 100644 --- a/build.gradle +++ b/build.gradle @@ -29,7 +29,7 @@ buildscript { apache_cxf_version = '4.0.3' open_saml_version = '4.3.0' one_login_java_saml = '2.9.0' - jjwt_version = '0.12.3' + jjwt_version = '0.12.4' guava_version = '32.1.3-jre' jaxb_version = '2.3.9' spring_version = '5.3.31' From 157d137bd85696d55c6c78ba72124656d4cb5d5c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jan 2024 09:04:22 -0500 Subject: [PATCH 099/204] Bump gradle/gradle-build-action from 2 to 3 (#3994) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [gradle/gradle-build-action](https://github.com/gradle/gradle-build-action) from 2 to 3.
Release notes

Sourced from gradle/gradle-build-action's releases.

v3.0.0-rc.1

First release candidate of gradle/gradle-build-action@v3.0.0. This release candidate will the first release available under the v3 version tag.

[!IMPORTANT] As of v3 this action has been superceded by gradle/actions/setup-gradle. Any workflow that uses gradle/gradle-build-action@v3 will transparently delegate to gradle/actions/setup-gradle@v3.

Users are encouraged to update their workflows, replacing:

uses: gradle/gradle-build-action@v3

with

uses: gradle/actions/setup-gradle@v3

See the setup-gradle documentation for up-to-date documentation for gradle/actons/setup-gradle.

Changes from gradle-build-action@v2

This release brings some useful and much requested features, including:

  • save and restore the Gradle configuration-cache data
  • add the Job summary content as a PR comment
  • easily publish Build Scans® to the free Gradle Build Scan service
  • compatibility with Node 20

The only major breaking change from gradle-build-action@v2.12.0 is the update to require a Node 20 runtime environment. Aside from that change, this release should generally serve as a drop-in replacement for gradle-build-action@v2.

Changelog

  • [NEW] - Run with NodeJs 20.x (gradle/gradle-build-action#946)
  • [NEW] - Support for save & restore of configuration-cache data (gradle/gradle-build-action#966)
  • [NEW] - Support for automatic adding PR comment with Job Summary content (gradle/gradle-build-action#1020)
  • [NEW] - Make it easy to publish a Build Scan® to https://scans.gradle.com (gradle/gradle-build-action#1044)
  • [NEW] - Added dependency-graph-continue-on-failure input, which can be set to false to force the Job to fail when dependency graph submission fails (gradle/gradle-build-action#1036). Failure modes include:
  • [NEW] - Add dependency-graph: clear option to clear any dependency-graph previously submitted by the job
  • [FIX] Allow cache entries to be reused by jobs with the same ID in different workflows (gradle/gradle-build-action#1017)
    • Workflow name remains part of the cache key, but cache entries generated by the same job id in a different workflow may be restored
  • [FIX] Register pre-installed JDKs in Maven toolchains.xml file (gradle/gradle-build-action#1024)
    • This allows pre-installed JDKs to be auto-detected by Gradle Toolchain support on Windows
  • [FIX] - Update the Gradle Enterprise injection configuration for product rename to Develocity (gradle/gradle-build-action#995)
  • [FIX] - Avoid submitting an empty dependency graph when state is loaded from configuration-cache
  • [DEPRECATION] - Deprecation of the arguments parameter (gradle/gradle-build-action#996)
  • [BREAKING CHANGE] - Remove the gradle-executable input parameter. Use a separate workflow Step to execute a Gradle from a custom location.

... (truncated)

Commits
  • 4a8703f Delegate to 'setup-gradle@v3.0.0-rc.1'
  • 4a39eed Mention setup-gradle in README
  • 272883a Remove all action sources: these have been migrated to 'gradle/actions'
  • 2a8bfcf Delegate action implementation to gradle/actions/setup-gradle
  • e1ada08 Bump the github-actions group with 1 update (#1047)
  • a8e3e5e Apply dependency version updates
  • 2be01ca Build outputs
  • a00827e Bump the npm-dependencies group with 7 updates
  • ad80850 Bump the github-actions group with 2 updates
  • bd6d0a7 Configure explicit java version for config-cache test
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=gradle/gradle-build-action&package-manager=github_actions&previous-version=2&new-version=3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 8 ++++---- .github/workflows/code-hygiene.yml | 6 +++--- .github/workflows/plugin_install.yml | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f64a0408b..acd9b9de27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,7 @@ jobs: uses: actions/checkout@v4 - name: Build and Test - uses: gradle/gradle-build-action@v2 + uses: gradle/gradle-build-action@v3 with: cache-disabled: true arguments: | @@ -112,7 +112,7 @@ jobs: uses: actions/checkout@v4 - name: Build and Test - uses: gradle/gradle-build-action@v2 + uses: gradle/gradle-build-action@v3 with: cache-disabled: true arguments: | @@ -147,7 +147,7 @@ jobs: uses: actions/checkout@v4 - name: Build and Test - uses: gradle/gradle-build-action@v2 + uses: gradle/gradle-build-action@v3 with: cache-disabled: true arguments: | @@ -165,7 +165,7 @@ jobs: uses: actions/checkout@v4 - name: Build BWC tests - uses: gradle/gradle-build-action@v2 + uses: gradle/gradle-build-action@v3 with: cache-disabled: true arguments: | diff --git a/.github/workflows/code-hygiene.yml b/.github/workflows/code-hygiene.yml index e6afd8fede..2f8820709a 100644 --- a/.github/workflows/code-hygiene.yml +++ b/.github/workflows/code-hygiene.yml @@ -24,7 +24,7 @@ jobs: distribution: temurin # Temurin is a distribution of adoptium java-version: 17 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/gradle-build-action@v3 with: cache-disabled: true arguments: spotlessCheck @@ -40,7 +40,7 @@ jobs: distribution: temurin # Temurin is a distribution of adoptium java-version: 11 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/gradle-build-action@v3 with: cache-disabled: true arguments: checkstyleMain checkstyleTest checkstyleIntegrationTest @@ -56,7 +56,7 @@ jobs: distribution: temurin # Temurin is a distribution of adoptium java-version: 11 - - uses: gradle/gradle-build-action@v2 + - uses: gradle/gradle-build-action@v3 with: cache-disabled: true arguments: spotbugsMain diff --git a/.github/workflows/plugin_install.yml b/.github/workflows/plugin_install.yml index c051e2f6a3..b88cfb166f 100644 --- a/.github/workflows/plugin_install.yml +++ b/.github/workflows/plugin_install.yml @@ -29,7 +29,7 @@ jobs: uses: actions/checkout@v4 - name: Assemble target plugin - uses: gradle/gradle-build-action@v2 + uses: gradle/gradle-build-action@v3 with: cache-disabled: true arguments: assemble @@ -63,7 +63,7 @@ jobs: admin-password: ${{ steps.random-password.outputs.generated_name }} - name: Run sanity tests - uses: gradle/gradle-build-action@v2 + uses: gradle/gradle-build-action@v3 with: cache-disabled: true arguments: integTestRemote -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="opensearch" -Dhttps=true -Duser=admin -Dpassword=${{ steps.random-password.outputs.generated_name }} -i From c06365ca94c9a1d15e85b578a1ae48168bf0bca7 Mon Sep 17 00:00:00 2001 From: Prabhas Kurapati <66924475+prabhask5@users.noreply.github.com> Date: Mon, 29 Jan 2024 06:54:56 -0800 Subject: [PATCH 100/204] [BUG-2556] Add new DLS filtering test (#3908) Signed-off-by: Prabhas Kurapati --- .../security/DlsIntegrationTests.java | 205 +++++++++++++++++- 1 file changed, 204 insertions(+), 1 deletion(-) diff --git a/src/integrationTest/java/org/opensearch/security/DlsIntegrationTests.java b/src/integrationTest/java/org/opensearch/security/DlsIntegrationTests.java index aa7202cddf..3e3ac61502 100644 --- a/src/integrationTest/java/org/opensearch/security/DlsIntegrationTests.java +++ b/src/integrationTest/java/org/opensearch/security/DlsIntegrationTests.java @@ -10,12 +10,15 @@ package org.opensearch.security; import java.io.IOException; +import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.function.BiFunction; +import java.util.stream.Collectors; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.apache.commons.lang3.tuple.Pair; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; @@ -36,6 +39,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; import static org.opensearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions.Type.ADD; import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.opensearch.client.RequestOptions.DEFAULT; @@ -57,6 +61,7 @@ import static org.opensearch.test.framework.matcher.SearchResponseMatchers.isSuccessfulSearchResponse; import static org.opensearch.test.framework.matcher.SearchResponseMatchers.numberOfTotalHitsIsEqualTo; import static org.opensearch.test.framework.matcher.SearchResponseMatchers.searchHitContainsFieldWithValue; +import static org.opensearch.test.framework.matcher.SearchResponseMatchers.searchHitsContainDocumentsInAnyOrder; @RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) @ThreadLeakScope(ThreadLeakScope.Scope.NONE) @@ -82,6 +87,7 @@ public class DlsIntegrationTests { static final String FIRST_INDEX_ALIAS_FILTERED_BY_TWINS_ARTIST = FIRST_INDEX_NAME.concat("-filtered-by-twins-artist"); static final String FIRST_INDEX_ALIAS_FILTERED_BY_FIRST_ARTIST = FIRST_INDEX_NAME.concat("-filtered-by-first-artist"); static final String ALL_INDICES_ALIAS = "_all"; + static final String UNION_TEST_INDEX_NAME = "my_index1"; static final TestSecurityConfig.User ADMIN_USER = new TestSecurityConfig.User("admin").roles(ALL_ACCESS); @@ -158,6 +164,62 @@ public class DlsIntegrationTests { .on("*") ); + /** + * Test role 1 for DLS filtering with two (non)overlapping roles. This role imposes a filter where the user can only access documents where the sensitive field is false. This role is applied at a higher level for all index patterns. + */ + static final TestSecurityConfig.Role ROLE_NON_SENSITIVE_ONLY = new TestSecurityConfig.Role("test_role_1").clusterPermissions( + "cluster_composite_ops_ro" + ).indexPermissions("read").dls("{\"match\":{\"sensitive\":false}}").on("*"); + + /** + * Test role 2 for DLS filtering with two overlapping roles. This role does not impose any filter, and combined with TEST_ROLE_ONE should yield a union that does not impose any filter. This role is applied at a lower level for index patterns my_index*. + */ + static final TestSecurityConfig.Role ROLE_ALLOW_ALL = new TestSecurityConfig.Role("test_role_2").clusterPermissions( + "cluster_composite_ops_ro" + ).indexPermissions("read").dls("{\"match_all\": {}}").on("my_index*"); + + /** + * Test role 3 for DLS filtering with two nonoverlapping roles. This role imposes a filter where the user can only access documents where the genre field is History, and combined with TEST_ROLE_ONE should yield a union that allows the user to access every document except the one with genre Science and sensitive true. This role is applied at a lower level for index patterns my_index*. + */ + static final TestSecurityConfig.Role ROLE_MATCH_HISTORY_GENRE_ONLY = new TestSecurityConfig.Role("test_role_3").clusterPermissions( + "cluster_composite_ops_ro" + ).indexPermissions("read").dls("{\"match\":{\"genre\":\"History\"}}").on("my_index*"); + + /** + * User with DLS permission to only be able to access documents with false sensitive property. + */ + static final TestSecurityConfig.User USER_NON_SENSITIVE_ONLY = new TestSecurityConfig.User("test_role_1_user").roles( + ROLE_NON_SENSITIVE_ONLY + ); + + /** + * User with DLS permission to access all documents. + */ + static final TestSecurityConfig.User USER_ALLOW_ALL = new TestSecurityConfig.User("test_role_2_user").roles(ROLE_ALLOW_ALL); + + /** + * User with DLS permission to access documents with genre property matching History. + */ + static final TestSecurityConfig.User USER_MATCH_HISTORY_GENRE_ONLY = new TestSecurityConfig.User("test_role_3_user").roles( + ROLE_MATCH_HISTORY_GENRE_ONLY + ); + + /** + * User with overlapping DLS permissions to access documents with false sensitive property and access all documents- should yield accessing all documents. + */ + static final TestSecurityConfig.User USER_UNION_OF_OVERLAPPING_ROLES_NON_SENSITIVE_ONLY_AND_ALLOW_ALL = new TestSecurityConfig.User( + "test_union_of_overlapping_roles_user" + ).roles(ROLE_NON_SENSITIVE_ONLY, ROLE_ALLOW_ALL); + + /** + * User with non-overlapping DLS permissions to access documents with false sensitive property and genre property matching History. + */ + static final TestSecurityConfig.User USER_UNION_OF_NONOVERLAPPING_ROLES_NON_SENSITIVE_ONLY_AND_HISTORY_GENRE_ONLY = + new TestSecurityConfig.User("test_union_of_non_overlapping_roles_user").roles( + ROLE_NON_SENSITIVE_ONLY, + ROLE_MATCH_HISTORY_GENRE_ONLY + ); + @ClassRule public static final LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS) .anonymousAuth(false) @@ -172,7 +234,12 @@ public class DlsIntegrationTests { READ_WHERE_FIELD_ARTIST_MATCHES_ARTIST_STRING, READ_WHERE_STARS_LESS_THAN_THREE, READ_WHERE_FIELD_ARTIST_MATCHES_ARTIST_TWINS_OR_FIELD_STARS_GREATER_THAN_FIVE, - READ_WHERE_FIELD_ARTIST_MATCHES_ARTIST_TWINS_OR_MATCHES_ARTIST_FIRST + READ_WHERE_FIELD_ARTIST_MATCHES_ARTIST_TWINS_OR_MATCHES_ARTIST_FIRST, + USER_NON_SENSITIVE_ONLY, + USER_ALLOW_ALL, + USER_MATCH_HISTORY_GENRE_ONLY, + USER_UNION_OF_OVERLAPPING_ROLES_NON_SENSITIVE_ONLY_AND_ALLOW_ALL, + USER_UNION_OF_NONOVERLAPPING_ROLES_NON_SENSITIVE_ONLY_AND_HISTORY_GENRE_ONLY ) .build(); @@ -218,6 +285,21 @@ public class DlsIntegrationTests { } }; + static final TreeMap> UNION_ROLE_TEST_DATA = new TreeMap<>() { + { + put("1", Map.of("genre", "History", "date", "01-01-2020", "sensitive", true)); + put("2", Map.of("genre", "History", "date", "01-01-2020", "sensitive", true)); + put("3", Map.of("genre", "History", "date", "01-01-2020", "sensitive", true)); + put("4", Map.of("genre", "History", "date", "01-01-2020", "sensitive", true)); + put("5", Map.of("genre", "History", "date", "01-01-2020", "sensitive", true)); + put("6", Map.of("genre", "Math", "date", "01-01-2020", "sensitive", false)); + put("7", Map.of("genre", "Math", "date", "01-01-2020", "sensitive", false)); + put("8", Map.of("genre", "Math", "date", "01-01-2020", "sensitive", false)); + put("9", Map.of("genre", "Math", "date", "01-01-2020", "sensitive", false)); + put("10", Map.of("genre", "Science", "date", "01-01-2020", "sensitive", true)); + } + }; + @BeforeClass public static void createTestData() { try (Client client = cluster.getInternalNodeClient()) { @@ -275,6 +357,10 @@ public static void createTestData() { ) ) .actionGet(); + + UNION_ROLE_TEST_DATA.forEach((index, document) -> { + client.prepareIndex(UNION_TEST_INDEX_NAME).setId(index).setRefreshPolicy(IMMEDIATE).setSource(document).get(); + }); } } @@ -517,4 +603,121 @@ public void testAggregateAndComputeStarRatings() throws IOException { assertThat(((ParsedAvg) actualAggregation).getValue(), is(1.5)); } } + + @Test + public void testOverlappingRoleUnionSearchFiltering() throws Exception { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(USER_NON_SENSITIVE_ONLY)) { + SearchRequest searchRequest = new SearchRequest(UNION_TEST_INDEX_NAME); + SearchResponse searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); + + assertSearchResponseHitsEqualTo(searchResponse, 4); + + assertThat( + searchResponse, + searchHitsContainDocumentsInAnyOrder( + UNION_ROLE_TEST_DATA.entrySet() + .stream() + .filter(e -> e.getValue().get("sensitive").equals(false)) + .map(e -> Pair.of(UNION_TEST_INDEX_NAME, e.getKey())) + .collect(Collectors.toList()) + ) + ); + } + + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(USER_ALLOW_ALL)) { + SearchRequest searchRequest = new SearchRequest(UNION_TEST_INDEX_NAME); + SearchResponse searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); + + assertSearchResponseHitsEqualTo(searchResponse, 10); + } + + try ( + RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient( + USER_UNION_OF_OVERLAPPING_ROLES_NON_SENSITIVE_ONLY_AND_ALLOW_ALL + ) + ) { + SearchRequest searchRequest = new SearchRequest(UNION_TEST_INDEX_NAME); + SearchResponse searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); + + assertSearchResponseHitsEqualTo(searchResponse, 10); + + // shows that roles are additive and the overlapping role with less filtering is used + assertThat( + searchResponse, + searchHitsContainDocumentsInAnyOrder( + UNION_ROLE_TEST_DATA.keySet().stream().map(id -> Pair.of(UNION_TEST_INDEX_NAME, id)).collect(Collectors.toList()) + ) + ); + } + } + + @Test + @SuppressWarnings("unchecked") + public void testNonOverlappingRoleUnionSearchFiltering() throws Exception { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(USER_NON_SENSITIVE_ONLY)) { + SearchRequest searchRequest = new SearchRequest(UNION_TEST_INDEX_NAME); + SearchResponse searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); + + assertSearchResponseHitsEqualTo(searchResponse, 4); + + assertThat( + searchResponse, + searchHitsContainDocumentsInAnyOrder( + UNION_ROLE_TEST_DATA.entrySet() + .stream() + .filter(e -> e.getValue().get("sensitive").equals(false)) + .map(e -> Pair.of(UNION_TEST_INDEX_NAME, e.getKey())) + .collect(Collectors.toList()) + ) + ); + } + + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(USER_MATCH_HISTORY_GENRE_ONLY)) { + SearchRequest searchRequest = new SearchRequest(UNION_TEST_INDEX_NAME); + SearchResponse searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); + + assertSearchResponseHitsEqualTo(searchResponse, 5); + + assertThat( + searchResponse, + searchHitsContainDocumentsInAnyOrder( + UNION_ROLE_TEST_DATA.entrySet() + .stream() + .filter(e -> e.getValue().get("genre").equals("History")) + .map(e -> Pair.of(UNION_TEST_INDEX_NAME, e.getKey())) + .collect(Collectors.toList()) + ) + ); + } + + try ( + RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient( + USER_UNION_OF_NONOVERLAPPING_ROLES_NON_SENSITIVE_ONLY_AND_HISTORY_GENRE_ONLY + ) + ) { + SearchRequest searchRequest = new SearchRequest(UNION_TEST_INDEX_NAME); + SearchResponse searchResponse = restHighLevelClient.search(searchRequest, DEFAULT); + + assertSearchResponseHitsEqualTo(searchResponse, 9); + + assertThat( + searchResponse, + searchHitsContainDocumentsInAnyOrder( + UNION_ROLE_TEST_DATA.keySet() + .stream() + .filter(id -> !id.equals("10")) + .map(id -> Pair.of(UNION_TEST_INDEX_NAME, id)) + .collect(Collectors.toList()) + ) + ); + + // shows that the roles are additive, but excludes one document since the DLS filters for both roles do not account for this + assertThat(searchResponse, not(searchHitsContainDocumentsInAnyOrder(Pair.of(UNION_TEST_INDEX_NAME, "10")))); + } + } + + private void assertSearchResponseHitsEqualTo(SearchResponse searchResponse, int hits) throws Exception { + assertThat(searchResponse, isSuccessfulSearchResponse()); + assertThat(searchResponse, numberOfTotalHitsIsEqualTo(hits)); + } } From 9187da18b034ff37a454924d5e2b9223dae4dac8 Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Thu, 1 Feb 2024 01:14:00 -0500 Subject: [PATCH 101/204] Add additional sendRequestDecorate cases (#3920) Signed-off-by: Stephen Crawford --- .../security/OpenSearchSecurityPlugin.java | 4 +- .../transport/SecurityInterceptor.java | 11 +- .../transport/SecurityInterceptorTests.java | 270 ++++++++++++++---- 3 files changed, 231 insertions(+), 54 deletions(-) diff --git a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java index 569380582b..53493e7f6c 100644 --- a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java +++ b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java @@ -251,6 +251,7 @@ public final class OpenSearchSecurityPlugin extends OpenSearchSecuritySSLPlugin private volatile OpensearchDynamicSetting transportPassiveAuthSetting; public static boolean isActionTraceEnabled() { + return actionTrace.isTraceEnabled(); } @@ -1108,7 +1109,8 @@ public Collection createComponents( cs, Objects.requireNonNull(sslExceptionHandler), Objects.requireNonNull(cih), - SSLConfig + SSLConfig, + OpenSearchSecurityPlugin::isActionTraceEnabled ); components.add(principalExtractor); diff --git a/src/main/java/org/opensearch/security/transport/SecurityInterceptor.java b/src/main/java/org/opensearch/security/transport/SecurityInterceptor.java index fe1094c411..f791cd013a 100644 --- a/src/main/java/org/opensearch/security/transport/SecurityInterceptor.java +++ b/src/main/java/org/opensearch/security/transport/SecurityInterceptor.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.function.Supplier; import java.util.stream.Collectors; import com.google.common.collect.Maps; @@ -71,8 +72,6 @@ import org.opensearch.transport.TransportRequestOptions; import org.opensearch.transport.TransportResponseHandler; -import static org.opensearch.security.OpenSearchSecurityPlugin.isActionTraceEnabled; - public class SecurityInterceptor { protected final Logger log = LogManager.getLogger(getClass()); @@ -86,6 +85,7 @@ public class SecurityInterceptor { private final SslExceptionHandler sslExceptionHandler; private final ClusterInfoHolder clusterInfoHolder; private final SSLConfig SSLConfig; + private final Supplier actionTraceEnabled; public SecurityInterceptor( final Settings settings, @@ -97,7 +97,8 @@ public SecurityInterceptor( final ClusterService cs, final SslExceptionHandler sslExceptionHandler, final ClusterInfoHolder clusterInfoHolder, - final SSLConfig SSLConfig + final SSLConfig SSLConfig, + final Supplier actionTraceSupplier ) { this.backendRegistry = backendRegistry; this.auditLog = auditLog; @@ -109,6 +110,7 @@ public SecurityInterceptor( this.sslExceptionHandler = sslExceptionHandler; this.clusterInfoHolder = clusterInfoHolder; this.SSLConfig = SSLConfig; + this.actionTraceEnabled = actionTraceSupplier; } public SecurityRequestHandler getHandler(String action, TransportRequestHandler actualHandler) { @@ -247,7 +249,7 @@ && getThreadContext().getHeader(ConfigConstants.OPENDISTRO_SECURITY_INJECTED_ROL useJDKSerialization ); - if (isActionTraceEnabled()) { + if (actionTraceEnabled.get()) { getThreadContext().putHeader( "_opendistro_security_trace" + System.currentTimeMillis() + "#" + UUID.randomUUID().toString(), Thread.currentThread().getName() @@ -407,5 +409,4 @@ public String executor() { return innerHandler.executor(); } } - } diff --git a/src/test/java/org/opensearch/security/transport/SecurityInterceptorTests.java b/src/test/java/org/opensearch/security/transport/SecurityInterceptorTests.java index 903ad89eac..4b3636a000 100644 --- a/src/test/java/org/opensearch/security/transport/SecurityInterceptorTests.java +++ b/src/test/java/org/opensearch/security/transport/SecurityInterceptorTests.java @@ -10,7 +10,10 @@ // CS-SUPPRESS-SINGLE: RegexpSingleline Extensions manager used for creating a mock import java.net.InetAddress; +import java.net.InetSocketAddress; import java.net.UnknownHostException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import org.junit.Before; import org.junit.Test; @@ -51,6 +54,9 @@ import static java.util.Collections.emptySet; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -82,12 +88,44 @@ public class SecurityInterceptorTests { @Mock private SSLConfig sslConfig; - private Settings settings; + @Mock + private TransportRequest request; + + @Mock + private TransportRequestOptions options; + + @SuppressWarnings("unchecked") + private TransportResponseHandler handler = mock(TransportResponseHandler.class); + private Settings settings; private ThreadPool threadPool; + private ClusterName clusterName = ClusterName.DEFAULT; + private MockTransport transport; + private TransportService transportService; + private OpenSearchSecurityPlugin.GuiceHolder guiceHolder; + private User user; + private String action = "testAction"; + private Version remoteNodeVersion = Version.V_2_0_0; + + private InetAddress localAddress; + private InetAddress remoteAddress; + private DiscoveryNode localNode; + private Connection connection1; + private DiscoveryNode otherNode; + private Connection connection2; + private DiscoveryNode remoteNode; + private Connection connection3; + private DiscoveryNode otherRemoteNode; + private Connection connection4; + + private AsyncSender sender; + private AsyncSender serializedSender; + private AsyncSender nullSender; @Before public void setup() { + + // Build mocked objects MockitoAnnotations.openMocks(this); settings = Settings.builder() .put("node.name", SecurityInterceptorTests.class.getSimpleName()) @@ -104,17 +142,15 @@ public void setup() { clusterService, sslExceptionHandler, clusterInfoHolder, - sslConfig + sslConfig, + () -> true ); - } - private void testSendRequestDecorate(Version remoteNodeVersion) { - boolean useJDKSerialization = remoteNodeVersion.before(ConfigConstants.FIRST_CUSTOM_SERIALIZATION_SUPPORTED_OS_VERSION); - ClusterName clusterName = ClusterName.DEFAULT; + clusterName = ClusterName.DEFAULT; when(clusterService.getClusterName()).thenReturn(clusterName); - MockTransport transport = new MockTransport(); - TransportService transportService = transport.createTransportService( + transport = new MockTransport(); + transportService = transport.createTransportService( Settings.EMPTY, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, @@ -125,7 +161,7 @@ private void testSendRequestDecorate(Version remoteNodeVersion) { ); // CS-SUPPRESS-SINGLE: RegexpSingleline Extensions manager used for creating a mock - OpenSearchSecurityPlugin.GuiceHolder guiceHolder = new OpenSearchSecurityPlugin.GuiceHolder( + guiceHolder = new OpenSearchSecurityPlugin.GuiceHolder( mock(RepositoriesService.class), transportService, mock(IndicesService.class), @@ -134,30 +170,34 @@ private void testSendRequestDecorate(Version remoteNodeVersion) { ); // CS-ENFORCE-SINGLE - User user = new User("John Doe"); - threadPool.getThreadContext().putTransient(ConfigConstants.OPENDISTRO_SECURITY_USER, user); + // Instantiate objects for tests + user = new User("John Doe"); - String action = "testAction"; - TransportRequest request = mock(TransportRequest.class); - TransportRequestOptions options = mock(TransportRequestOptions.class); - @SuppressWarnings("unchecked") - TransportResponseHandler handler = mock(TransportResponseHandler.class); + request = mock(TransportRequest.class); + options = mock(TransportRequestOptions.class); - InetAddress localAddress = null; + localAddress = null; + remoteAddress = null; try { localAddress = InetAddress.getByName("0.0.0.0"); + remoteAddress = InetAddress.getByName("1.1.1.1"); } catch (final UnknownHostException uhe) { throw new RuntimeException(uhe); } - DiscoveryNode localNode = new DiscoveryNode("local-node", new TransportAddress(localAddress, 1234), Version.CURRENT); - Connection connection1 = transportService.getConnection(localNode); + localNode = new DiscoveryNode("local-node1", new TransportAddress(localAddress, 1234), Version.CURRENT); + connection1 = transportService.getConnection(localNode); - DiscoveryNode otherNode = new DiscoveryNode("remote-node", new TransportAddress(localAddress, 4321), remoteNodeVersion); - Connection connection2 = transportService.getConnection(otherNode); + otherNode = new DiscoveryNode("local-node2", new TransportAddress(localAddress, 4321), Version.CURRENT); + connection2 = transportService.getConnection(otherNode); - // from thread context inside sendRequestDecorate - AsyncSender sender = new AsyncSender() { + remoteNode = new DiscoveryNode("remote-node", new TransportAddress(localAddress, 6789), remoteNodeVersion); + connection3 = transportService.getConnection(remoteNode); + + otherRemoteNode = new DiscoveryNode("remote-node2", new TransportAddress(remoteAddress, 9876), remoteNodeVersion); + connection4 = transportService.getConnection(otherRemoteNode); + + serializedSender = new AsyncSender() { @Override public void sendRequest( Connection connection, @@ -166,19 +206,11 @@ public void sendRequest( TransportRequestOptions options, TransportResponseHandler handler ) { - User transientUser = threadPool.getThreadContext().getTransient(ConfigConstants.OPENDISTRO_SECURITY_USER); - assertEquals(transientUser, user); + String serializedUserHeader = threadPool.getThreadContext().getHeader(ConfigConstants.OPENDISTRO_SECURITY_USER_HEADER); + assertEquals(serializedUserHeader, Base64Helper.serializeObject(user, true)); } }; - // isSameNodeRequest = true - securityInterceptor.sendRequestDecorate(sender, connection1, action, request, options, handler, localNode); - - // from original context - User transientUser = threadPool.getThreadContext().getTransient(ConfigConstants.OPENDISTRO_SECURITY_USER); - assertEquals(transientUser, user); - assertEquals(threadPool.getThreadContext().getHeader(ConfigConstants.OPENDISTRO_SECURITY_USER_HEADER), null); - // checking thread context inside sendRequestDecorate sender = new AsyncSender() { @Override public void sendRequest( @@ -188,30 +220,172 @@ public void sendRequest( TransportRequestOptions options, TransportResponseHandler handler ) { - String serializedUserHeader = threadPool.getThreadContext().getHeader(ConfigConstants.OPENDISTRO_SECURITY_USER_HEADER); - assertEquals(serializedUserHeader, Base64Helper.serializeObject(user, useJDKSerialization)); + User transientUser = threadPool.getThreadContext().getTransient(ConfigConstants.OPENDISTRO_SECURITY_USER); + assertEquals(transientUser, user); } }; - // isSameNodeRequest = false - securityInterceptor.sendRequestDecorate(sender, connection2, action, request, options, handler, localNode); - // from original context - User transientUser2 = threadPool.getThreadContext().getTransient(ConfigConstants.OPENDISTRO_SECURITY_USER); - assertEquals(transientUser2, user); - assertEquals(threadPool.getThreadContext().getHeader(ConfigConstants.OPENDISTRO_SECURITY_USER_HEADER), null); + threadPool.getThreadContext().putTransient(ConfigConstants.OPENDISTRO_SECURITY_USER, user); + } + + /** + * A method to confirm the original thread context is maintained + * @param user The expected user to be in the transient header + */ + final void verifyOriginalContext(User user) { + + User transientUser = threadPool.getThreadContext().getTransient(ConfigConstants.OPENDISTRO_SECURITY_USER); + assertEquals(transientUser, user); + assertNull(threadPool.getThreadContext().getHeader(ConfigConstants.OPENDISTRO_SECURITY_USER_HEADER)); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + final void completableRequestDecorate( + AsyncSender sender, + Connection connection, + String action, + TransportRequest request, + TransportRequestOptions options, + TransportResponseHandler handler, + DiscoveryNode localNode + ) { + + ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); + + singleThreadExecutor.execute(() -> { + try { + securityInterceptor.sendRequestDecorate(sender, connection, action, request, options, handler, localNode); + verifyOriginalContext(user); + } finally { + singleThreadExecutor.shutdown(); + } + }); } @Test - public void testSendRequestDecorate() { - testSendRequestDecorate(Version.CURRENT); + public void testSendRequestDecorateLocalConnection() { + + // local node request + completableRequestDecorate(sender, connection1, action, request, options, handler, localNode); + // this is also a local request + completableRequestDecorate(sender, connection2, action, request, options, handler, otherNode); } - /** - * Tests the scenario when remote node does not implement custom serialization protocol and uses JDK serialization - */ @Test - public void testSendRequestDecorateWhenRemoteNodeUsesJDKSerde() { - testSendRequestDecorate(Version.V_2_0_0); + public void testSendRequestDecorateRemoteConnection() { + + // this is a remote request + completableRequestDecorate(serializedSender, connection3, action, request, options, handler, localNode); + // this is a remote request where the transport address is different + completableRequestDecorate(serializedSender, connection4, action, request, options, handler, localNode); + } + + @Test + public void testSendNoOriginNodeCausesSerialization() { + + // this is a request where the local node is null; have to use the remote connection since the serialization will fail + completableRequestDecorate(serializedSender, connection3, action, request, options, handler, null); + } + + @Test + public void testSendNoConnectionShouldThrowNPE() { + + // The completable version swallows the NPE so have to call actual method + assertThrows( + java.lang.NullPointerException.class, + () -> securityInterceptor.sendRequestDecorate(serializedSender, null, action, request, options, handler, localNode) + ); } + @Test + public void testNullOriginHeaderCausesNoSerialization() { + + // Make the origin null should cause the ensureCorrectHeaders method to populate with Origin.LOCAL.toString() + threadPool.getThreadContext().putHeader(ConfigConstants.OPENDISTRO_SECURITY_ORIGIN, null); + // This is a different way to get the same result which exercises the origin0 = null logic of ensureCorrectHeaders + securityInterceptor.sendRequestDecorate(sender, connection1, action, request, options, handler, localNode); + verifyOriginalContext(user); + } + + @Test + public void testNullRemoteAddressCausesNoSerialization() { + + // Make the remote address null should cause the ensureCorrectHeaders to keep the TransportAddress as null ultimately causing local + // logic to occur + threadPool.getThreadContext().putHeader(ConfigConstants.OPENDISTRO_SECURITY_REMOTE_ADDRESS, null); + // This is a different way to get the same result which exercises the origin0 = null logic of ensureCorrectHeaders + completableRequestDecorate(sender, connection1, action, request, options, handler, localNode); + } + + @Test + public void testCustomRemoteAddressCausesSerialization() { + + threadPool.getThreadContext() + .putHeader( + ConfigConstants.OPENDISTRO_SECURITY_REMOTE_ADDRESS, + String.valueOf(new TransportAddress(new InetSocketAddress("8.8.8.8", 80))) + ); + completableRequestDecorate(serializedSender, connection3, action, request, options, handler, localNode); + } + + @Test + public void testTraceHeaderIsRemoved() { + + threadPool.getThreadContext().putTransient("_opendistro_security_trace", "fake trace value"); + // this case is just for action trace logic validation + // local node request + completableRequestDecorate(sender, connection1, action, request, options, handler, localNode); + // even though we add the trace the restoring handler should remove it from the thread context + assertFalse( + threadPool.getThreadContext().getHeaders().keySet().stream().anyMatch(header -> header.startsWith("_opendistro_security_trace")) + ); + } + + @Test + public void testFakeHeaderIsIgnored() { + + threadPool.getThreadContext().putHeader("FAKE_HEADER", "fake_value"); + // this is a local request + completableRequestDecorate(sender, connection1, action, request, options, handler, localNode); + // this is a remote request + completableRequestDecorate(serializedSender, connection3, action, request, options, handler, localNode); + } + + @Test + public void testNullHeaderIsIgnored() { + + // Add a null header + threadPool.getThreadContext().putHeader(null, null); + threadPool.getThreadContext().putHeader(null, "null"); + // this is a local request + completableRequestDecorate(sender, connection1, action, request, options, handler, localNode); + // this is a remote request + completableRequestDecorate(serializedSender, connection3, action, request, options, handler, localNode); + } + + @Test + public void testFakeHeadersAreIgnored() { + + threadPool.getThreadContext() + .putHeader(ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER, "fake security config request header"); + threadPool.getThreadContext().putHeader(ConfigConstants.OPENDISTRO_SECURITY_ORIGIN_HEADER, "fake security origin header"); + threadPool.getThreadContext() + .putHeader(ConfigConstants.OPENDISTRO_SECURITY_REMOTE_ADDRESS_HEADER, "fake security remote address header"); + threadPool.getThreadContext().putHeader(ConfigConstants.OPENDISTRO_SECURITY_DLS_QUERY_HEADER, "fake dls query header"); + threadPool.getThreadContext().putHeader(ConfigConstants.OPENDISTRO_SECURITY_FLS_FIELDS_HEADER, "fake fls fields header"); + threadPool.getThreadContext().putHeader(ConfigConstants.OPENDISTRO_SECURITY_MASKED_FIELD_HEADER, "fake masked field header"); + threadPool.getThreadContext().putHeader(ConfigConstants.OPENDISTRO_SECURITY_DOC_ALLOWLIST_HEADER, "fake doc allowlist header"); + threadPool.getThreadContext().putHeader(ConfigConstants.OPENDISTRO_SECURITY_FILTER_LEVEL_DLS_DONE, "fake filter level dls header"); + threadPool.getThreadContext().putHeader(ConfigConstants.OPENDISTRO_SECURITY_DLS_MODE_HEADER, "fake dls mode header"); + threadPool.getThreadContext() + .putHeader(ConfigConstants.OPENDISTRO_SECURITY_DLS_FILTER_LEVEL_QUERY_HEADER, "fake dls filter header"); + threadPool.getThreadContext() + .putHeader(ConfigConstants.OPENDISTRO_SECURITY_INITIAL_ACTION_CLASS_HEADER, "fake initial action header"); + threadPool.getThreadContext().putHeader("_opendistro_security_source_field_context", "fake source field context value"); + threadPool.getThreadContext() + .putHeader(ConfigConstants.OPENDISTRO_SECURITY_INJECTED_ROLES_VALIDATION, "fake injected roles validation string"); + + // this is a local request + completableRequestDecorate(sender, connection1, action, request, options, handler, localNode); + } } From ff3b77ce66388f81cee2564d54de6db8b5a7bf59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 07:43:54 -0500 Subject: [PATCH 102/204] Bump jjwt_version from 0.12.4 to 0.12.5 (#4011) Bumps `jjwt_version` from 0.12.4 to 0.12.5. Updates `io.jsonwebtoken:jjwt-api` from 0.12.4 to 0.12.5
Release notes

Sourced from io.jsonwebtoken:jjwt-api's releases.

0.12.5

This release fixes issue #916 and ensures that builders' NestedCollection changes are applied to the collection immediately as mutation methods are called, no longer requiring application developers to call .and() to 'commit' or apply a change. For example, prior to this release, the following code did not apply changes:

JwtBuilder builder = Jwts.builder();
builder.audience().add("an-audience"); // no .and() call
builder.compact(); // would not keep 'an-audience'

Now this code works as expected and all other NestedCollection instances like it apply changes immediately (e.g. when calling .add(value)).

However, standard fluent builder chains are still recommended for readability when feasible, e.g.

Jwts.builder()
.audience().add("an-audience").and() // allows fluent chaining
    .subject("Joe")
    // etc...
    .compact()

These same notes are repeated in the CHANGELOG, and as always, project documentation is in the README.

Please allow 30 minutes from the time this announcement is published for the release to be available in Maven Central.

Changelog

Sourced from io.jsonwebtoken:jjwt-api's changelog.

0.12.5

This patch release:

  • Ensures that builders' NestedCollection changes are applied to the collection immediately as mutation methods are called, no longer requiring application developers to call .and() to 'commit' or apply a change. For example, prior to this release, the following code did not apply changes:

    JwtBuilder builder = Jwts.builder();
builder.audience().add("an-audience"); // no .and() call
builder.compact(); // would not keep 'an-audience'

    Now this code works as expected and all other NestedCollection instances like it apply changes immediately (e.g. when calling .add(value)).

    However, standard fluent builder chains are still recommended for readability when feasible, e.g.

    Jwts.builder()
.audience().add("an-audience").and() // allows fluent chaining
    .subject("Joe")
    // etc...
    .compact()

    See Issue 916.

Commits

Updates `io.jsonwebtoken:jjwt-impl` from 0.12.4 to 0.12.5
Release notes

Sourced from io.jsonwebtoken:jjwt-impl's releases.

0.12.5

This release fixes issue #916 and ensures that builders' NestedCollection changes are applied to the collection immediately as mutation methods are called, no longer requiring application developers to call .and() to 'commit' or apply a change. For example, prior to this release, the following code did not apply changes:

JwtBuilder builder = Jwts.builder();
builder.audience().add("an-audience"); // no .and() call
builder.compact(); // would not keep 'an-audience'

Now this code works as expected and all other NestedCollection instances like it apply changes immediately (e.g. when calling .add(value)).

However, standard fluent builder chains are still recommended for readability when feasible, e.g.

Jwts.builder()
.audience().add("an-audience").and() // allows fluent chaining
    .subject("Joe")
    // etc...
    .compact()

These same notes are repeated in the CHANGELOG, and as always, project documentation is in the README.

Please allow 30 minutes from the time this announcement is published for the release to be available in Maven Central.

Changelog

Sourced from io.jsonwebtoken:jjwt-impl's changelog.

0.12.5

This patch release:

  • Ensures that builders' NestedCollection changes are applied to the collection immediately as mutation methods are called, no longer requiring application developers to call .and() to 'commit' or apply a change. For example, prior to this release, the following code did not apply changes:

    JwtBuilder builder = Jwts.builder();
builder.audience().add("an-audience"); // no .and() call
builder.compact(); // would not keep 'an-audience'

    Now this code works as expected and all other NestedCollection instances like it apply changes immediately (e.g. when calling .add(value)).

    However, standard fluent builder chains are still recommended for readability when feasible, e.g.

    Jwts.builder()
.audience().add("an-audience").and() // allows fluent chaining
    .subject("Joe")
    // etc...
    .compact()

    See Issue 916.

Commits

Updates `io.jsonwebtoken:jjwt-jackson` from 0.12.4 to 0.12.5 Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index e0ae931d5a..edfebfe04e 100644 --- a/build.gradle +++ b/build.gradle @@ -29,7 +29,7 @@ buildscript { apache_cxf_version = '4.0.3' open_saml_version = '4.3.0' one_login_java_saml = '2.9.0' - jjwt_version = '0.12.4' + jjwt_version = '0.12.5' guava_version = '32.1.3-jre' jaxb_version = '2.3.9' spring_version = '5.3.31' From 698af8800754b121e531d985fd91b9e48297a145 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 07:44:16 -0500 Subject: [PATCH 103/204] Bump org.apache.camel:camel-xmlsecurity from 3.22.0 to 3.22.1 (#4012) Bumps org.apache.camel:camel-xmlsecurity from 3.22.0 to 3.22.1. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.camel:camel-xmlsecurity&package-manager=gradle&previous-version=3.22.0&new-version=3.22.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index edfebfe04e..4b8b1c94be 100644 --- a/build.gradle +++ b/build.gradle @@ -614,7 +614,7 @@ dependencies { runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.1' runtimeOnly 'org.ow2.asm:asm:9.6' - testImplementation 'org.apache.camel:camel-xmlsecurity:3.22.0' + testImplementation 'org.apache.camel:camel-xmlsecurity:3.22.1' //OpenSAML implementation 'net.shibboleth.utilities:java-support:8.4.0' From 90e25ac756ccfa4c573005cbf8376431bcf0485d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 07:44:40 -0500 Subject: [PATCH 104/204] Bump com.netflix.nebula.ospackage from 11.6.0 to 11.7.0 (#4013) Bumps com.netflix.nebula.ospackage from 11.6.0 to 11.7.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.netflix.nebula.ospackage&package-manager=gradle&previous-version=11.6.0&new-version=11.7.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 4b8b1c94be..c27689b54a 100644 --- a/build.gradle +++ b/build.gradle @@ -64,7 +64,7 @@ plugins { id 'maven-publish' id 'com.diffplug.spotless' version '6.25.0' id 'checkstyle' - id 'com.netflix.nebula.ospackage' version "11.6.0" + id 'com.netflix.nebula.ospackage' version "11.7.0" id "org.gradle.test-retry" version "1.5.8" id 'eclipse' id "com.github.spotbugs" version "5.2.5" From 5bd0dac648098040eb14b4e06b86e89fbae3dac1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 07:45:03 -0500 Subject: [PATCH 105/204] Bump org.junit.jupiter:junit-jupiter from 5.10.1 to 5.10.2 (#4014) Bumps [org.junit.jupiter:junit-jupiter](https://github.com/junit-team/junit5) from 5.10.1 to 5.10.2.
Release notes

Sourced from org.junit.jupiter:junit-jupiter's releases.

JUnit 5.10.2 = Platform 1.10.2 + Jupiter 5.10.2 + Vintage 5.10.2

See Release Notes.

Full Changelog: https://github.com/junit-team/junit5/compare/r5.10.1...r5.10.2

Commits
  • 4c0ddda Release 5.10.2
  • 463a147 Finalize release notes for 5.10.2
  • 43c105a Revert "Apply method predicate before searching type hierarchy"
  • 63d464d Revert "Harmonize application of method and field filters in search algorithms"
  • 85ec2fc Revert "Apply field predicate before searching type hierarchy"
  • 6209006 Update release notes
  • 5ee499f Fix CI build
  • d919ba7 Namespace user-specific build parameters
  • e26cd83 Prepare release notes for 5.10.2
  • ec8d428 Include LauncherInterceptor in launcher module declaration
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.junit.jupiter:junit-jupiter&package-manager=gradle&previous-version=5.10.1&new-version=5.10.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index c27689b54a..d84475c794 100644 --- a/build.gradle +++ b/build.gradle @@ -685,8 +685,8 @@ dependencies { testImplementation 'commons-validator:commons-validator:1.8.0' testImplementation 'org.springframework.kafka:spring-kafka-test:2.9.13' testImplementation "org.springframework:spring-beans:${spring_version}" - testImplementation 'org.junit.jupiter:junit-jupiter:5.10.1' - testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.1' + testImplementation 'org.junit.jupiter:junit-jupiter:5.10.2' + testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.2' testImplementation('org.awaitility:awaitility:4.2.0') { exclude(group: 'org.hamcrest', module: 'hamcrest') } From f75d0c9c426ce12e067228f302527b377aec7627 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 07:50:28 -0500 Subject: [PATCH 106/204] Bump release-drafter/release-drafter from 5 to 6 (#4016) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [release-drafter/release-drafter](https://github.com/release-drafter/release-drafter) from 5 to 6.
Release notes

Sourced from release-drafter/release-drafter's releases.

v6.0.0

What's Changed

Full Changelog: https://github.com/release-drafter/release-drafter/compare/v5.25.0...v6.0.0

v6.0.0-beta.1

Prerelease of v6, first release of the CLI, feel free to provide feedback in the pull request: release-drafter/release-drafter#1204

v5.25.0

What's Changed

New

Full Changelog: https://github.com/release-drafter/release-drafter/compare/v5.24.0...v5.25.0

v5.24.0

What's Changed

New

Bug Fixes

Full Changelog: https://github.com/release-drafter/release-drafter/compare/v5.23.0...v5.24.0

v5.23.0

What's Changed

New

Full Changelog: https://github.com/release-drafter/release-drafter/compare/v5.22.0...v5.23.0

v5.22.0

What's Changed

New

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=release-drafter/release-drafter&package-manager=github_actions&previous-version=5&new-version=6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release-drafter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index bf6e5b0674..10e870d04f 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: # Drafts the next Release notes as Pull Requests are merged into "main" - - uses: release-drafter/release-drafter@v5 + - uses: release-drafter/release-drafter@v6 with: config-name: release-notes-drafter-config.yml env: From 6b9ded21b2b7af18f8bacbcf1b242ffe12960da1 Mon Sep 17 00:00:00 2001 From: Chenyang Ji Date: Mon, 5 Feb 2024 07:44:18 -0800 Subject: [PATCH 107/204] Admin role for Query insights plugin (#4006) Signed-off-by: Chenyang Ji --- config/roles.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/roles.yml b/config/roles.yml index ccd1759355..8d1b1f88f6 100644 --- a/config/roles.yml +++ b/config/roles.yml @@ -399,3 +399,9 @@ flow_framework_read_access: - 'cluster:admin/opensearch/flow_framework/workflow_state/get' - 'cluster:admin/opensearch/flow_framework/workflow_state/search' - 'cluster:admin/opensearch/flow_framework/workflow_step/get' + +# Allows users to use all query insights APIs +query_insights_full_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opensearch/insights/top_queries/*' From 321604c59d97fc1275dc111fd59c1809ccf6158b Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Mon, 5 Feb 2024 12:21:24 -0500 Subject: [PATCH 108/204] Update TRIAGING.md with new time for triage meetings (#4023) Signed-off-by: Craig Perkins --- TRIAGING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TRIAGING.md b/TRIAGING.md index 22e59afc2d..7f70501dae 100644 --- a/TRIAGING.md +++ b/TRIAGING.md @@ -12,7 +12,7 @@ Each meeting we seek to address all new issues. However, should we run out of t ### How do I join the Backlog & Triage meeting? -Meetings are hosted regularly at 3 PM Eastern Time (Noon Pacific Time) and can be joined via the links posted on the [Upcoming Events](https://opensearch.org/events) webpage. +Meetings are hosted regularly at 11 AM Eastern Time (8AM Pacific Time) and can be joined via the links posted on the [OpenSearch Meetup Group](https://www.meetup.com/opensearch/events/) list of events. The event will be titled `Development Backlog & Triage Meeting - Security`. After joining the Zoom meeting, you can enable your video / voice to join the discussion. If you do not have a webcam or microphone available, you can still join in via the text chat. From a41b3f7f7ccbc4a78856e97c8b7254040b2f288e Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 8 Feb 2024 13:17:43 -0500 Subject: [PATCH 109/204] Redact sensitive configuration values when retrieving security configuration (#4024) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Description Redacts the field `password` when calling `GET /_plugins/_security/api/securityconfig`. Redacting the field through the API will also ensure that it doesn't get displayed on OSD. **Note**: This value would only be displayed to users with access to the security pages. Screenshot 2024-02-05 at 3 27 34 PM * Category (Enhancement, New feature, Bug fix, Test fix, Refactoring, Maintenance, Documentation) Enhancement ### Issues Resolved - https://github.com/opensearch-project/security/issues/4004 ### Check List - [ ] New functionality includes testing - [ ] New functionality has been documented - [ ] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: Craig Perkins --- .../security/http/LdapAuthenticationTest.java | 25 +++++++++- .../security/DefaultObjectMapper.java | 48 +++++++++++++++++++ .../dlic/rest/api/AbstractApiAction.java | 15 +++++- .../impl/SecurityDynamicConfiguration.java | 9 ++++ 4 files changed, 95 insertions(+), 2 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/http/LdapAuthenticationTest.java b/src/integrationTest/java/org/opensearch/security/http/LdapAuthenticationTest.java index b4a3717287..7339808d8c 100644 --- a/src/integrationTest/java/org/opensearch/security/http/LdapAuthenticationTest.java +++ b/src/integrationTest/java/org/opensearch/security/http/LdapAuthenticationTest.java @@ -41,6 +41,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; import static org.opensearch.security.http.CertificateAuthenticationTest.POINTER_BACKEND_ROLES; import static org.opensearch.security.http.DirectoryInformationTrees.CN_GROUP_ADMIN; import static org.opensearch.security.http.DirectoryInformationTrees.DN_CAPTAIN_SPOCK_PEOPLE_TEST_ORG; @@ -55,6 +56,8 @@ import static org.opensearch.security.http.DirectoryInformationTrees.USER_KIRK; import static org.opensearch.security.http.DirectoryInformationTrees.USER_SEARCH; import static org.opensearch.security.http.DirectoryInformationTrees.USER_SPOCK; +import static org.opensearch.security.support.ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST; +import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_ROLES_ENABLED; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.BASIC_AUTH_DOMAIN_ORDER; import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS; @@ -83,7 +86,16 @@ public class LdapAuthenticationTest { public static LocalCluster cluster = new LocalCluster.Builder().testCertificates(TEST_CERTIFICATES) .clusterManager(ClusterManager.SINGLENODE) .anonymousAuth(false) - .nodeSettings(Map.of(ConfigConstants.SECURITY_AUTHCZ_REST_IMPERSONATION_USERS + "." + ADMIN_USER.getName(), List.of(USER_KIRK))) + .nodeSettings( + Map.of( + ConfigConstants.SECURITY_AUTHCZ_REST_IMPERSONATION_USERS + "." + ADMIN_USER.getName(), + List.of(USER_KIRK), + SECURITY_RESTAPI_ROLES_ENABLED, + List.of("user_" + ADMIN_USER.getName() + "__" + ALL_ACCESS.getName()), + SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, + false + ) + ) .authc( new AuthcDomain("ldap", BASIC_AUTH_DOMAIN_ORDER + 1, true).httpAuthenticator(new HttpAuthenticator("basic").challenge(false)) .backend( @@ -190,4 +202,15 @@ public void testShouldCreateScrollWithLdapUserAndImpersonateWithAdmin() { scrollResponse.assertStatusCode(200); } } + + @Test + public void testShouldRedactPasswordWhenGettingSecurityConfig() { + try (TestRestClient client = cluster.getRestClient(ADMIN_USER)) { + TestRestClient.HttpResponse response = client.get("_plugins/_security/api/securityconfig"); + + response.assertStatusCode(200); + String redactedPassword = response.getTextFromJsonBody("/config/dynamic/authc/ldap/authentication_backend/config/password"); + assertThat("******", equalTo(redactedPassword)); + } + } } diff --git a/src/main/java/org/opensearch/security/DefaultObjectMapper.java b/src/main/java/org/opensearch/security/DefaultObjectMapper.java index 48aa09541a..2d18667c54 100644 --- a/src/main/java/org/opensearch/security/DefaultObjectMapper.java +++ b/src/main/java/org/opensearch/security/DefaultObjectMapper.java @@ -35,6 +35,7 @@ import com.google.common.collect.ImmutableSet; import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; @@ -42,14 +43,40 @@ import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.exc.InvalidFormatException; import com.fasterxml.jackson.databind.exc.MismatchedInputException; import com.fasterxml.jackson.databind.introspect.BeanPropertyDefinition; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; import com.fasterxml.jackson.databind.type.TypeFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.opensearch.SpecialPermission; +class ConfigMapSerializer extends StdSerializer> { + private static final Set SENSITIVE_CONFIG_KEYS = Set.of("password"); + + @SuppressWarnings("unchecked") + public ConfigMapSerializer() { + // Pass Map.class to the superclass + super((Class>) (Class) Map.class); + } + + @Override + public void serialize(Map value, JsonGenerator gen, SerializerProvider serializers) throws IOException { + gen.writeStartObject(); + for (Map.Entry entry : value.entrySet()) { + if (SENSITIVE_CONFIG_KEYS.contains(entry.getKey())) { + gen.writeStringField(entry.getKey(), "******"); // Redact + } else { + gen.writeObjectField(entry.getKey(), entry.getValue()); + } + } + gen.writeEndObject(); + } +} + public class DefaultObjectMapper { public static final ObjectMapper objectMapper = new ObjectMapper(); public final static ObjectMapper YAML_MAPPER = new ObjectMapper(new YAMLFactory()); @@ -180,6 +207,27 @@ public static String writeValueAsString(Object value, boolean omitDefaults) thro } + @SuppressWarnings("removal") + public static String writeValueAsStringAndRedactSensitive(Object value) throws JsonProcessingException { + final SecurityManager sm = System.getSecurityManager(); + + if (sm != null) { + sm.checkPermission(new SpecialPermission()); + } + + SimpleModule module = new SimpleModule(); + module.addSerializer(new ConfigMapSerializer()); + ObjectMapper mapper = new ObjectMapper(); + mapper.registerModule(module); + + try { + return AccessController.doPrivileged((PrivilegedExceptionAction) () -> mapper.writeValueAsString(value)); + } catch (final PrivilegedActionException e) { + throw (JsonProcessingException) e.getCause(); + } + + } + @SuppressWarnings("removal") public static T readValue(String string, TypeReference tr) throws IOException { diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/AbstractApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/AbstractApiAction.java index 04148e8b99..ef8a00d700 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/AbstractApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/AbstractApiAction.java @@ -372,7 +372,12 @@ protected final ValidationResult> loadConfigurat boolean omitSensitiveData, final boolean logComplianceEvent ) { - final var configuration = load(cType, logComplianceEvent); + SecurityDynamicConfiguration configuration; + if (omitSensitiveData) { + configuration = loadAndRedact(cType, logComplianceEvent); + } else { + configuration = load(cType, logComplianceEvent); + } if (configuration.getSeqNo() < 0) { return ValidationResult.error( @@ -448,6 +453,14 @@ protected final SecurityDynamicConfiguration load(final CType config, boolean return DynamicConfigFactory.addStatics(loaded); } + protected final SecurityDynamicConfiguration loadAndRedact(final CType config, boolean logComplianceEvent) { + SecurityDynamicConfiguration loaded = securityApiDependencies.configurationRepository() + .getConfigurationsFromIndex(List.of(config), logComplianceEvent) + .get(config) + .deepCloneWithRedaction(); + return DynamicConfigFactory.addStatics(loaded); + } + protected boolean ensureIndexExists() { return clusterService.state().metadata().hasConcreteIndex(securityApiDependencies.securityIndexName()); } diff --git a/src/main/java/org/opensearch/security/securityconf/impl/SecurityDynamicConfiguration.java b/src/main/java/org/opensearch/security/securityconf/impl/SecurityDynamicConfiguration.java index bba44b5e28..90508840e7 100644 --- a/src/main/java/org/opensearch/security/securityconf/impl/SecurityDynamicConfiguration.java +++ b/src/main/java/org/opensearch/security/securityconf/impl/SecurityDynamicConfiguration.java @@ -303,6 +303,15 @@ public SecurityDynamicConfiguration deepClone() { } } + @JsonIgnore + public SecurityDynamicConfiguration deepCloneWithRedaction() { + try { + return fromJson(DefaultObjectMapper.writeValueAsStringAndRedactSensitive(this), ctype, version, seqNo, primaryTerm); + } catch (Exception e) { + throw ExceptionsHelper.convertToOpenSearchException(e); + } + } + @JsonIgnore public void remove(String key) { synchronized (modificationLock) { From f5170e64142d66292bd9ab5f1ce588236296e40f Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 8 Feb 2024 23:47:43 -0500 Subject: [PATCH 110/204] Add release notes for 2.12.0.0 (#4030) ### Description Add release notes for 2.12.0.0 release * Category (Enhancement, New feature, Bug fix, Test fix, Refactoring, Maintenance, Documentation) Documentation ### Issues Resolved - https://github.com/opensearch-project/security/issues/3513 ### Check List - [ ] New functionality includes testing - [ ] New functionality has been documented - [ ] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: Craig Perkins --- ...nsearch-security.release-notes-2.12.0.0.md | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 release-notes/opensearch-security.release-notes-2.12.0.0.md diff --git a/release-notes/opensearch-security.release-notes-2.12.0.0.md b/release-notes/opensearch-security.release-notes-2.12.0.0.md new file mode 100644 index 0000000000..be6ddbc125 --- /dev/null +++ b/release-notes/opensearch-security.release-notes-2.12.0.0.md @@ -0,0 +1,61 @@ +## 2024-02-20 Version 2.12.0.0 + +Compatible with OpenSearch 2.12.0 + +### Enhancements +* Add additional sendRequestDecorate cases ([#4007](https://github.com/opensearch-project/security/pull/4007)) +* [BUG-2556] Add new DLS filtering test ([#4001](https://github.com/opensearch-project/security/pull/4001)) +* [Enhancement-3191] `transport_enabled` setting on an auth domain and authorizer may be unnecessary after transport client removal ([#3966](https://github.com/opensearch-project/security/pull/3966)) +* Update roles.yml with new API for experimental alerting plugin feature [#4027](https://github.com/opensearch-project/security/pull/4027) ([#4029](https://github.com/opensearch-project/security/pull/4029)) +* Admin role for Query insights plugin ([#4022](https://github.com/opensearch-project/security/pull/4022)) +* Validate 409s occur when multiple config updates happen simultaneously ([#3962](https://github.com/opensearch-project/security/pull/3962)) +* Protect config object from concurrent modification issues ([#3956](https://github.com/opensearch-project/security/pull/3956)) +* Add test coverage for ComplianceConfig ([#3957](https://github.com/opensearch-project/security/pull/3957)) +* Update security analytics roles to include custom log type cluster permissions ([#3954](https://github.com/opensearch-project/security/pull/3954)) +* Add logging for test LdapServer actions ([#3942](https://github.com/opensearch-project/security/pull/3942)) +* HeapBasedRateTracker uses time provider to allow simluating of time in unit tests ([#3941](https://github.com/opensearch-project/security/pull/3941)) +* Add additional logging around `testShouldSearchAll` tests ([#3943](https://github.com/opensearch-project/security/pull/3943)) +* Add permission for get workflow step ([#3940](https://github.com/opensearch-project/security/pull/3940)) +* Add additional ignore_headers audit configuration setting ([#3926](https://github.com/opensearch-project/security/pull/3926)) +* Update to Gradle 8.5 ([#3919](https://github.com/opensearch-project/security/pull/3919)) ([#3923](https://github.com/opensearch-project/security/pull/3923)) +* Refactor SSL handler retrieval to use HttpChannel / TranportChannel APIs instead of typecasting ([#3917](https://github.com/opensearch-project/security/pull/3917)) ([#3922](https://github.com/opensearch-project/security/pull/3922)) +* Improve messaging on how to set initial admin password ([#3918](https://github.com/opensearch-project/security/pull/3918)) +* Re-enable disabled PIT integration tests ([#3914](https://github.com/opensearch-project/security/pull/3914)) +* Switched to more reliable OpenSearch Lucene snapshot location ([#3913](https://github.com/opensearch-project/security/pull/3913)) +* Add deprecation check for `jwt_header` setting ([#3896](https://github.com/opensearch-project/security/pull/3896)) +* Add render search template as a cluster permission ([#3689](https://github.com/opensearch-project/security/pull/3689)) ([#3872](https://github.com/opensearch-project/security/pull/3872)) +* Add flow framework system indices and roles ([#3851](https://github.com/opensearch-project/security/pull/3851)) ([#3880](https://github.com/opensearch-project/security/pull/3880)) +* Search operation test flakiness fix ([#3862](https://github.com/opensearch-project/security/pull/3862)) +* Extracts demo configuration setup into a java tool, adds support for Bundled JDK for this tool and updates DEVELOPER_GUIDE.md ([#3845](https://github.com/opensearch-project/security/pull/3845)) +* SAML permissions changes in DynamicConfigModelV7 ([#3853](https://github.com/opensearch-project/security/pull/3853)) +* Add do not fail on forbidden test cases around the stats API ([#3825](https://github.com/opensearch-project/security/pull/3825)) ([#3828](https://github.com/opensearch-project/security/pull/3828)) + +### Bug Fixes +* Fix Bug with Install demo configuration running in cluster mode with -y ([#3936](https://github.com/opensearch-project/security/pull/3936)) +* Allow TransportConfigUpdateAction when security config initialization has completed ([#3810](https://github.com/opensearch-project/security/pull/3810)) ([#3927](https://github.com/opensearch-project/security/pull/3927)) +* Fix the CI / report-coverage check by switching to corresponding actions/upload-artifact@v4 ([#3893](https://github.com/opensearch-project/security/pull/3893)) ([#3895](https://github.com/opensearch-project/security/pull/3895)) + +### Maintenance +* Bump org.apache.camel:camel-xmlsecurity from 3.22.0 to 3.22.1 ([#4018](https://github.com/opensearch-project/security/pull/4018)) +* Bump release-drafter/release-drafter from 5 to 6 ([#4021](https://github.com/opensearch-project/security/pull/4021)) +* Bump com.netflix.nebula.ospackage from 11.6.0 to 11.7.0 ([#4019](https://github.com/opensearch-project/security/pull/4019)) +* Bump org.junit.jupiter:junit-jupiter from 5.10.1 to 5.10.2 ([#4020](https://github.com/opensearch-project/security/pull/4020)) +* Bump jjwt_version from 0.12.4 to 0.12.5 ([#4017](https://github.com/opensearch-project/security/pull/4017)) +* Bump io.dropwizard.metrics:metrics-core from 4.2.24 to 4.2.25 ([#3998](https://github.com/opensearch-project/security/pull/3998)) +* Bump gradle/gradle-build-action from 2 to 3 ([#4000](https://github.com/opensearch-project/security/pull/4000)) +* Bump jjwt_version from 0.12.3 to 0.12.4 ([#3999](https://github.com/opensearch-project/security/pull/3999)) +* Bump spotless (6.24.0 -> 6.25.0) to bump eclipse resources (3.18 -> 3.19) ([#3993](https://github.com/opensearch-project/security/pull/3993)) +* Fix: remove unnecessary trailing slashes in APIs. ([#3978](https://github.com/opensearch-project/security/pull/3978)) +* Adds new ml-commons system indices to the list ([#3974](https://github.com/opensearch-project/security/pull/3974)) +* Bump io.dropwizard.metrics:metrics-core from 4.2.23 to 4.2.24 ([#3970](https://github.com/opensearch-project/security/pull/3970)) +* Bump com.fasterxml.woodstox:woodstox-core from 6.5.1 to 6.6.0 ([#3969](https://github.com/opensearch-project/security/pull/3969)) +* Bump com.diffplug.spotless from 6.23.3 to 6.24.0 ([#3947](https://github.com/opensearch-project/security/pull/3947)) +* Bump org.apache.camel:camel-xmlsecurity from 3.21.3 to 3.22.0 ([#3906](https://github.com/opensearch-project/security/pull/3906)) +* Bump com.google.errorprone:error_prone_annotations from 2.23.0 to 2.24.0 ([#3897](https://github.com/opensearch-project/security/pull/3897)) ([#3902](https://github.com/opensearch-project/security/pull/3902)) +* Bump io.dropwizard.metrics:metrics-core from 4.2.22 to 4.2.23 ([#3900](https://github.com/opensearch-project/security/pull/3900)) +* Bump com.google.googlejavaformat:google-java-format from 1.18.1 to 1.19.1 ([#3901](https://github.com/opensearch-project/security/pull/3901)) +* Bump github/codeql-action from 2 to 3 ([#3859](https://github.com/opensearch-project/security/pull/3859)) ([#3867](https://github.com/opensearch-project/security/pull/3867)) +* Bump org.apache.camel:camel-xmlsecurity from 3.21.2 to 3.21.3 ([#3864](https://github.com/opensearch-project/security/pull/3864)) +* Bump org.checkerframework:checker-qual from 3.40.0 to 3.42.0 ([#3857](https://github.com/opensearch-project/security/pull/3857)) ([#3866](https://github.com/opensearch-project/security/pull/3866)) +* Bump com.flipkart.zjsonpatch:zjsonpatch from 0.4.14 to 0.4.16 ([#3865](https://github.com/opensearch-project/security/pull/3865)) +* Bump com.netflix.nebula.ospackage from 11.5.0 to 11.6.0 ([#3863](https://github.com/opensearch-project/security/pull/3863)) From ca4eedf245c85683b6622fa6a11fad2bfcac88a3 Mon Sep 17 00:00:00 2001 From: AWSHurneyt Date: Thu, 8 Feb 2024 21:07:40 -0800 Subject: [PATCH 111/204] v2.12 update roles.yml with new API for experimental alerting plugin feature (#4027) Signed-off-by: AWSHurneyt --- config/roles.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/roles.yml b/config/roles.yml index 8d1b1f88f6..efa83ed02e 100644 --- a/config/roles.yml +++ b/config/roles.yml @@ -33,6 +33,7 @@ alerting_read_access: - 'cluster:admin/opendistro/alerting/monitor/get' - 'cluster:admin/opendistro/alerting/monitor/search' - 'cluster:admin/opensearch/alerting/findings/get' + - 'cluster:admin/opensearch/alerting/remote/indexes/get' - 'cluster:admin/opensearch/alerting/workflow/get' - 'cluster:admin/opensearch/alerting/workflow_alerts/get' From f5c260ec9c8eb41a7cffaa2ee57458b27b161cfc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Feb 2024 06:49:07 -0600 Subject: [PATCH 112/204] Bump com.netflix.nebula.ospackage from 11.7.0 to 11.8.0 (#4040) Bumps com.netflix.nebula.ospackage from 11.7.0 to 11.8.0. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index d84475c794..8a31c3b874 100644 --- a/build.gradle +++ b/build.gradle @@ -64,7 +64,7 @@ plugins { id 'maven-publish' id 'com.diffplug.spotless' version '6.25.0' id 'checkstyle' - id 'com.netflix.nebula.ospackage' version "11.7.0" + id 'com.netflix.nebula.ospackage' version "11.8.0" id "org.gradle.test-retry" version "1.5.8" id 'eclipse' id "com.github.spotbugs" version "5.2.5" From 3d68394d1e424ce33f3c02e182252bb7a2800a4f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Feb 2024 07:53:45 -0500 Subject: [PATCH 113/204] Bump Wandalen/wretry.action from 1.3.0 to 1.4.4 (#4038) Bumps [Wandalen/wretry.action](https://github.com/wandalen/wretry.action) from 1.3.0 to 1.4.4.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Wandalen/wretry.action&package-manager=github_actions&previous-version=1.3.0&new-version=1.4.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acd9b9de27..fbb4d6c266 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,7 +81,7 @@ jobs: working-directory: downloaded-artifacts - name: Upload Coverage with retry - uses: Wandalen/wretry.action@v1.3.0 + uses: Wandalen/wretry.action@v1.4.4 with: attempt_limit: 5 attempt_delay: 2000 From 69bf786ec55aadeb39dc9396f5cd104147e2ffcf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Feb 2024 12:55:27 +0000 Subject: [PATCH 114/204] Bump commons-codec:commons-codec from 1.16.0 to 1.16.1 (#4039) Bumps [commons-codec:commons-codec](https://github.com/apache/commons-codec) from 1.16.0 to 1.16.1.
Changelog

Sourced from commons-codec:commons-codec's changelog.

Apache Commons Codec 1.16.1 RELEASE NOTES

The Apache Commons Codec component contains encoder and decoders for various formats such as Base16, Base32, Base64, digest, and Hexadecimal. In addition to these widely used encoders and decoders, the codec package also maintains a collection of phonetic encoding utilities.

Feature and fix release. Requires a minimum of Java 8.

Changes in this version include:

New features: o Add Maven property project.build.outputTimestamp for build reproducibility. Thanks to Gary Gregory.

Fixed Bugs: o CODEC-295: Test clean ups. Thanks to Gary Gregory. o [StepSecurity] ci: Harden GitHub Actions #187. Thanks to step-security-bot, Gary Gregory. o CODEC-295: Correct error in Base64 Javadoc #188. Thanks to Evan Saulpaugh. o CODEC-295: Add minimum Java version in changes.xml #186. Thanks to Olivier Jaquemet, Gary Gregory. o CODEC-310: Documentation update for the org.apache.commons.codec.digest.* package #208. Thanks to Yakov Shafranovich. o Precompile regular expression in UnixCrypt.crypt(byte[], String). Thanks to Gary Gregory. o CODEC-315: Fix possible IndexOutOfBoundException in PhoneticEngine.encode method #223. Thanks to Arthur Chan, Gary Gregory. o CODEC-313: Fix possible ArrayIndexOutOfBoundsException in QuotedPrintableCodec.encodeQuotedPrintable() method #221. Thanks to Arthur Chan, Gary Gregory. o CODEC-312: Fix possible StringIndexOutOfBoundException in MatchRatingApproachEncoder.encode() method #220. Thanks to Arthur Chan, Gary Gregory. o CODEC-311: Fix possible ArrayIndexOutOfBoundException in RefinedSoundex.getMappingCode() #219. Thanks to Arthur Chan, Gary Gregory. o CODEC-314: Fix possible IndexOutOfBoundsException in PercentCodec.insertAlwaysEncodeChars() method #222. Thanks to Arthur Chan, Gary Gregory. o Deprecate UnixCrypt 0-argument constructor. Thanks to Gary Gregory. o Deprecate Md5Crypt 0-argument constructor. Thanks to Gary Gregory. o Deprecate Crypt 0-argument constructor. Thanks to Gary Gregory. o Deprecate StringUtils 0-argument constructor. Thanks to Gary Gregory. o Deprecate Resources 0-argument constructor. Thanks to Gary Gregory. o Deprecate Charsets 0-argument constructor. Thanks to Gary Gregory. o Deprecate CharEncoding 0-argument constructor. Thanks to Gary Gregory. o Add missing version for animal-sniffer-maven-plugin. Thanks to Gary Gregory.

Changes: o Bump commons-parent from 58 to 66. Thanks to Dependabot, Gary Gregory. o Bump commons-lang3 from 3.12.0 to 3.14.0. Thanks to Gary Gregory. o Bump commons-io from 2.13.0 to 2.15.1. Thanks to Gary Gregory.

For complete information on Apache Commons Codec, including instructions on how to submit bug reports, patches, or suggestions for improvement, see the Apache Commons Codec website:

https://commons.apache.org/proper/commons-codec/

Download page: https://commons.apache.org/proper/commons-codec/download_codec.cgi

... (truncated)

Commits
  • e59fc76 Prepare release candidate
  • 90c8023 Prepare for the next release candidate
  • 05714ad Prepare release candidate
  • 060be1a Add missing version for animal-sniffer-maven-plugin
  • 0fd7b59 Remove variable assignment just before returning it
  • 19649cd Add Maven property project.build.outputTimestamp for build
  • 6d92b6a Bump org.apache.commons:commons-parent from 65 to 66 #239
  • a76c362 Bump org.apache.commons:commons-parent from 65 to 66 (#239)
  • 0aee0c8 Add property project.build.outputTimestamp for build reproducibility
  • d322ef0 Bump codecov/codecov-action from 3.1.5 to 4.0.1 (#238)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=commons-codec:commons-codec&package-manager=gradle&previous-version=1.16.0&new-version=1.16.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 8a31c3b874..d5d1dbae3d 100644 --- a/build.gradle +++ b/build.gradle @@ -474,7 +474,7 @@ bundlePlugin { configurations { all { resolutionStrategy { - force 'commons-codec:commons-codec:1.16.0' + force 'commons-codec:commons-codec:1.16.1' force 'org.slf4j:slf4j-api:1.7.36' force 'org.scala-lang:scala-library:2.13.12' force "com.fasterxml.jackson:jackson-bom:${versions.jackson}" @@ -607,7 +607,7 @@ dependencies { runtimeOnly 'com.sun.activation:jakarta.activation:1.2.2' runtimeOnly 'com.eclipsesource.minimal-json:minimal-json:0.9.5' - runtimeOnly 'commons-codec:commons-codec:1.16.0' + runtimeOnly 'commons-codec:commons-codec:1.16.1' runtimeOnly 'org.cryptacular:cryptacular:1.2.6' compileOnly 'com.google.errorprone:error_prone_annotations:2.24.1' runtimeOnly 'com.sun.istack:istack-commons-runtime:4.2.0' From 0bb31ca365050f201d2a261e66670c3dbc8d8f7c Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Mon, 12 Feb 2024 11:55:36 -0500 Subject: [PATCH 115/204] Add `exclude_roles` configuration parameter to LDAP authorization backend (#4025) Signed-off-by: Maciej Mierzwa Signed-off-by: Craig Perkins Co-authored-by: Maciej Mierzwa --- .../backend/LDAPAuthorizationBackend.java | 18 ++-- .../dlic/auth/ldap/util/ConfigConstants.java | 1 + .../dlic/auth/ldap/util/LdapHelper.java | 1 - .../auth/ldap2/LDAPAuthorizationBackend2.java | 24 ++++-- .../security/support/WildcardMatcher.java | 8 +- .../dlic/auth/ldap/LdapBackendTest.java | 86 +++++++++++++++++++ .../ldap2/LdapBackendTestNewStyleConfig2.java | 12 +-- .../org/opensearch/security/UtilTests.java | 2 + 8 files changed, 130 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/amazon/dlic/auth/ldap/backend/LDAPAuthorizationBackend.java b/src/main/java/com/amazon/dlic/auth/ldap/backend/LDAPAuthorizationBackend.java index d8b33b2a7e..0ad0da54c6 100755 --- a/src/main/java/com/amazon/dlic/auth/ldap/backend/LDAPAuthorizationBackend.java +++ b/src/main/java/com/amazon/dlic/auth/ldap/backend/LDAPAuthorizationBackend.java @@ -101,6 +101,7 @@ public class LDAPAuthorizationBackend implements AuthorizationBackend { protected static final Logger log = LogManager.getLogger(LDAPAuthorizationBackend.class); private final Settings settings; private final WildcardMatcher skipUsersMatcher; + private final WildcardMatcher excludeRolesMatcher; private final WildcardMatcher nestedRoleMatcher; private final Path configPath; private final List> roleBaseSettings; @@ -112,6 +113,7 @@ public class LDAPAuthorizationBackend implements AuthorizationBackend { public LDAPAuthorizationBackend(final Settings settings, final Path configPath) { this.settings = settings; this.skipUsersMatcher = WildcardMatcher.from(settings.getAsList(ConfigConstants.LDAP_AUTHZ_SKIP_USERS)); + this.excludeRolesMatcher = WildcardMatcher.from(settings.getAsList(ConfigConstants.LDAP_AUTHZ_EXCLUDE_ROLES)); this.nestedRoleMatcher = settings.getAsBoolean(ConfigConstants.LDAP_AUTHZ_RESOLVE_NESTED_ROLES, false) ? WildcardMatcher.from(settings.getAsList(ConfigConstants.LDAP_AUTHZ_NESTEDROLEFILTER)) : null; @@ -962,10 +964,12 @@ public void fillRoles(final User user, final AuthCredentials optionalAuthCreds) for (final LdapName roleLdapName : nestedReturn) { final String role = getRoleFromEntry(connection, roleLdapName, roleName); - if (!Strings.isNullOrEmpty(role)) { - user.addRole(role); + if (excludeRolesMatcher.test(role)) { + if (isDebugEnabled) { + log.debug("Role was excluded or empty, attribute: '{}' for entry: {}", roleName, roleLdapName); + } } else { - log.warn("No or empty attribute '{}' for entry {}", roleName, roleLdapName); + user.addRole(role); } } @@ -974,10 +978,12 @@ public void fillRoles(final User user, final AuthCredentials optionalAuthCreds) for (final LdapName roleLdapName : ldapRoles) { final String role = getRoleFromEntry(connection, roleLdapName, roleName); - if (!Strings.isNullOrEmpty(role)) { - user.addRole(role); + if (excludeRolesMatcher.test(role)) { + if (isDebugEnabled) { + log.debug("Role was excluded or empty, attribute: '{}' for entry: {}", roleName, roleLdapName); + } } else { - log.warn("No or empty attribute '{}' for entry {}", roleName, roleLdapName); + user.addRole(role); } } diff --git a/src/main/java/com/amazon/dlic/auth/ldap/util/ConfigConstants.java b/src/main/java/com/amazon/dlic/auth/ldap/util/ConfigConstants.java index 4854f80332..d3c0b798da 100755 --- a/src/main/java/com/amazon/dlic/auth/ldap/util/ConfigConstants.java +++ b/src/main/java/com/amazon/dlic/auth/ldap/util/ConfigConstants.java @@ -29,6 +29,7 @@ public final class ConfigConstants { public static final String LDAP_AUTHZ_USERROLEATTRIBUTE = "userroleattribute";// multi-value public static final String LDAP_AUTHZ_USERROLENAME = "userrolename";// multi-value public static final String LDAP_AUTHZ_SKIP_USERS = "skip_users"; + public static final String LDAP_AUTHZ_EXCLUDE_ROLES = "exclude_roles"; public static final String LDAP_AUTHZ_ROLESEARCH_ENABLED = "rolesearch_enabled"; public static final String LDAP_AUTHZ_NESTEDROLEFILTER = "nested_role_filter"; public static final String LDAP_AUTHZ_MAX_NESTED_DEPTH = "max_nested_depth"; diff --git a/src/main/java/com/amazon/dlic/auth/ldap/util/LdapHelper.java b/src/main/java/com/amazon/dlic/auth/ldap/util/LdapHelper.java index f2dffa62fd..bdb2e00754 100644 --- a/src/main/java/com/amazon/dlic/auth/ldap/util/LdapHelper.java +++ b/src/main/java/com/amazon/dlic/auth/ldap/util/LdapHelper.java @@ -122,7 +122,6 @@ private static Object escapeForwardSlash(Object input) { } else { return input; } - } } diff --git a/src/main/java/com/amazon/dlic/auth/ldap2/LDAPAuthorizationBackend2.java b/src/main/java/com/amazon/dlic/auth/ldap2/LDAPAuthorizationBackend2.java index e05b2e1e64..8c1569bfb6 100755 --- a/src/main/java/com/amazon/dlic/auth/ldap2/LDAPAuthorizationBackend2.java +++ b/src/main/java/com/amazon/dlic/auth/ldap2/LDAPAuthorizationBackend2.java @@ -69,6 +69,7 @@ public class LDAPAuthorizationBackend2 implements AuthorizationBackend, Destroya protected static final Logger log = LogManager.getLogger(LDAPAuthorizationBackend2.class); private final Settings settings; private final WildcardMatcher skipUsersMatcher; + private final WildcardMatcher excludeRolesMatcher; private final WildcardMatcher nestedRoleMatcher; private final List> roleBaseSettings; private ConnectionPool connectionPool; @@ -80,6 +81,7 @@ public class LDAPAuthorizationBackend2 implements AuthorizationBackend, Destroya public LDAPAuthorizationBackend2(final Settings settings, final Path configPath) throws SSLConfigException { this.settings = settings; this.skipUsersMatcher = WildcardMatcher.from(settings.getAsList(ConfigConstants.LDAP_AUTHZ_SKIP_USERS)); + this.excludeRolesMatcher = WildcardMatcher.from(settings.getAsList(ConfigConstants.LDAP_AUTHZ_EXCLUDE_ROLES)); this.nestedRoleMatcher = settings.getAsBoolean(ConfigConstants.LDAP_AUTHZ_RESOLVE_NESTED_ROLES, false) ? WildcardMatcher.from(settings.getAsList(ConfigConstants.LDAP_AUTHZ_NESTEDROLEFILTER)) : null; @@ -329,8 +331,10 @@ private void fillRoles0(final User user, final AuthCredentials optionalAuthCreds for (final Iterator iterator = rolesResult.iterator(); iterator.hasNext();) { LdapEntry searchResultEntry = iterator.next(); LdapName ldapName = new LdapName(searchResultEntry.getDn()); - ldapRoles.add(ldapName); - resultRoleSearchBaseKeys.put(ldapName, roleSearchSettingsEntry); + if (!excludeRolesMatcher.test(searchResultEntry.getDn())) { + ldapRoles.add(ldapName); + resultRoleSearchBaseKeys.put(ldapName, roleSearchSettingsEntry); + } } } } @@ -376,10 +380,12 @@ private void fillRoles0(final User user, final AuthCredentials optionalAuthCreds for (final LdapName roleLdapName : nestedReturn) { final String role = getRoleFromEntry(connection, roleLdapName, roleName); - if (!Strings.isNullOrEmpty(role)) { - user.addRole(role); + if (excludeRolesMatcher.test(role)) { + if (isDebugEnabled) { + log.debug("Role was excluded or empty attribute '{}' for entry {}", roleName, roleLdapName); + } } else { - log.warn("No or empty attribute '{}' for entry {}", roleName, roleLdapName); + user.addRole(role); } } @@ -388,10 +394,12 @@ private void fillRoles0(final User user, final AuthCredentials optionalAuthCreds for (final LdapName roleLdapName : ldapRoles) { final String role = getRoleFromEntry(connection, roleLdapName, roleName); - if (!Strings.isNullOrEmpty(role)) { - user.addRole(role); + if (excludeRolesMatcher.test(role)) { + if (isDebugEnabled) { + log.debug("Role was excluded or empty attribute '{}' for entry {}", roleName, roleLdapName); + } } else { - log.warn("No or empty attribute '{}' for entry {}", roleName, roleLdapName); + user.addRole(role); } } diff --git a/src/main/java/org/opensearch/security/support/WildcardMatcher.java b/src/main/java/org/opensearch/security/support/WildcardMatcher.java index 99c34b53ba..d811a73730 100644 --- a/src/main/java/org/opensearch/security/support/WildcardMatcher.java +++ b/src/main/java/org/opensearch/security/support/WildcardMatcher.java @@ -150,7 +150,9 @@ public String toString() { }; public static WildcardMatcher from(String pattern, boolean caseSensitive) { - if (pattern.equals("*")) { + if (pattern == null) { + return NONE; + } else if (pattern.equals("*")) { return ANY; } else if (pattern.startsWith("/") && pattern.endsWith("/")) { return new RegexMatcher(pattern, caseSensitive); @@ -168,7 +170,9 @@ public static WildcardMatcher from(String pattern) { // This may in future use more optimized techniques to combine multiple WildcardMatchers in a single automaton public static WildcardMatcher from(Stream stream, boolean caseSensitive) { Collection matchers = stream.map(t -> { - if (t instanceof String) { + if (t == null) { + return NONE; + } else if (t instanceof String) { return WildcardMatcher.from(((String) t), caseSensitive); } else if (t instanceof WildcardMatcher) { return ((WildcardMatcher) t); diff --git a/src/test/java/com/amazon/dlic/auth/ldap/LdapBackendTest.java b/src/test/java/com/amazon/dlic/auth/ldap/LdapBackendTest.java index 4fe7ad0514..8e5e2541b8 100755 --- a/src/test/java/com/amazon/dlic/auth/ldap/LdapBackendTest.java +++ b/src/test/java/com/amazon/dlic/auth/ldap/LdapBackendTest.java @@ -14,6 +14,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.util.Arrays; +import java.util.List; import org.hamcrest.MatcherAssert; import org.junit.AfterClass; @@ -39,6 +40,7 @@ import org.ldaptive.ReturnAttributes; import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.not; public class LdapBackendTest { @@ -579,6 +581,90 @@ public void testLdapAuthorizationNested() throws Exception { MatcherAssert.assertThat(user.getRoles(), hasItem("nested1")); } + @Test + public void testLdapNestedRoleFiltering() { + + final Settings settings = Settings.builder() + .putList(ConfigConstants.LDAP_HOSTS, "localhost:" + ldapPort) + .put(ConfigConstants.LDAP_AUTHC_USERSEARCH, "(uid={0})") + .put(ConfigConstants.LDAP_AUTHC_USERBASE, "ou=people,o=TEST") + .put(ConfigConstants.LDAP_AUTHZ_ROLEBASE, "ou=groups,o=TEST") + .put(ConfigConstants.LDAP_AUTHZ_ROLENAME, "cn") + .put(ConfigConstants.LDAP_AUTHZ_RESOLVE_NESTED_ROLES, true) + .put(ConfigConstants.LDAP_AUTHZ_ROLESEARCH, "(uniqueMember={0})") + .putList(ConfigConstants.LDAP_AUTHZ_EXCLUDE_ROLES, List.of("nested1", "nested2")) + .build(); + + final User user = new User("spock"); + + new LDAPAuthorizationBackend(settings, null).fillRoles(user, null); + + Assert.assertNotNull(user); + Assert.assertEquals("spock", user.getName()); + Assert.assertEquals(2, user.getRoles().size()); + // filtered out + MatcherAssert.assertThat(user.getRoles(), not(hasItem("nested1"))); + MatcherAssert.assertThat(user.getRoles(), not(hasItem("nested2"))); + MatcherAssert.assertThat(user.getRoles(), hasItem("role2")); + MatcherAssert.assertThat(user.getRoles(), hasItem("ceo")); + } + + @Test + public void testLdapNestedRoleFilteringWithExcludedRolesWildcard() { + + final Settings settings = Settings.builder() + .putList(ConfigConstants.LDAP_HOSTS, "localhost:" + ldapPort) + .put(ConfigConstants.LDAP_AUTHC_USERSEARCH, "(uid={0})") + .put(ConfigConstants.LDAP_AUTHC_USERBASE, "ou=people,o=TEST") + .put(ConfigConstants.LDAP_AUTHZ_ROLEBASE, "ou=groups,o=TEST") + .put(ConfigConstants.LDAP_AUTHZ_ROLENAME, "cn") + .put(ConfigConstants.LDAP_AUTHZ_RESOLVE_NESTED_ROLES, true) + .put(ConfigConstants.LDAP_AUTHZ_ROLESEARCH, "(uniqueMember={0})") + .putList(ConfigConstants.LDAP_AUTHZ_EXCLUDE_ROLES, List.of("nested*")) + .build(); + + final User user = new User("spock"); + + new LDAPAuthorizationBackend(settings, null).fillRoles(user, null); + + Assert.assertNotNull(user); + Assert.assertEquals("spock", user.getName()); + Assert.assertEquals(2, user.getRoles().size()); + // filtered out + MatcherAssert.assertThat(user.getRoles(), not(hasItem("nested1"))); + MatcherAssert.assertThat(user.getRoles(), not(hasItem("nested2"))); + MatcherAssert.assertThat(user.getRoles(), hasItem("role2")); + MatcherAssert.assertThat(user.getRoles(), hasItem("ceo")); + } + + @Test + public void testLdapdRoleFiltering() { + + final Settings settings = Settings.builder() + .putList(ConfigConstants.LDAP_HOSTS, "localhost:" + ldapPort) + .put(ConfigConstants.LDAP_AUTHC_USERSEARCH, "(uid={0})") + .put(ConfigConstants.LDAP_AUTHC_USERBASE, "ou=people,o=TEST") + .put(ConfigConstants.LDAP_AUTHZ_ROLEBASE, "ou=groups,o=TEST") + .put(ConfigConstants.LDAP_AUTHZ_ROLENAME, "cn") + .put(ConfigConstants.LDAP_AUTHZ_RESOLVE_NESTED_ROLES, true) + .put(ConfigConstants.LDAP_AUTHZ_ROLESEARCH, "(uniqueMember={0})") + .putList(ConfigConstants.LDAP_AUTHZ_EXCLUDE_ROLES, List.of("ceo", "role1", "role2")) + .build(); + + final User user = new User("spock"); + + new LDAPAuthorizationBackend(settings, null).fillRoles(user, null); + + Assert.assertNotNull(user); + Assert.assertEquals("spock", user.getName()); + Assert.assertEquals(2, user.getRoles().size()); + MatcherAssert.assertThat(user.getRoles(), hasItem("nested1")); + MatcherAssert.assertThat(user.getRoles(), hasItem("nested2")); + // filtered out + MatcherAssert.assertThat(user.getRoles(), not(hasItem("role2"))); + MatcherAssert.assertThat(user.getRoles(), not(hasItem("ceo"))); + } + @Test public void testLdapAuthorizationNestedFilter() throws Exception { diff --git a/src/test/java/com/amazon/dlic/auth/ldap2/LdapBackendTestNewStyleConfig2.java b/src/test/java/com/amazon/dlic/auth/ldap2/LdapBackendTestNewStyleConfig2.java index 634584c167..6f23e4ab44 100644 --- a/src/test/java/com/amazon/dlic/auth/ldap2/LdapBackendTestNewStyleConfig2.java +++ b/src/test/java/com/amazon/dlic/auth/ldap2/LdapBackendTestNewStyleConfig2.java @@ -15,6 +15,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.TreeSet; import org.apache.commons.lang3.exception.ExceptionUtils; @@ -564,16 +565,18 @@ public void testLdapAuthorizationNested() throws Exception { .put(ConfigConstants.LDAP_AUTHZ_ROLENAME, "cn") .put(ConfigConstants.LDAP_AUTHZ_RESOLVE_NESTED_ROLES, true) .put("roles.g1.search", "(uniqueMember={0})") + .putList(ConfigConstants.LDAP_AUTHZ_EXCLUDE_ROLES, List.of("nested2")) .build(); final User user = new User("spock"); - new LDAPAuthorizationBackend(settings, null).fillRoles(user, null); + new LDAPAuthorizationBackend2(settings, null).fillRoles(user, null); Assert.assertNotNull(user); Assert.assertEquals("spock", user.getName()); - Assert.assertEquals(4, user.getRoles().size()); - Assert.assertEquals("nested1", new ArrayList<>(new TreeSet<>(user.getRoles())).get(1)); + Assert.assertEquals(3, user.getRoles().size()); + Assert.assertTrue(user.getRoles().contains("nested1")); + Assert.assertFalse(user.getRoles().contains("nested2")); } @Test @@ -759,7 +762,7 @@ public void testLdapAuthorizationNestedAttrFilter() throws Exception { } @Test - public void testLdapAuthorizationNestedAttrFilterAll() throws Exception { + public void testLdapAuthorizationNestedAttrFilterAll() { final Settings settings = createBaseSettings().putList(ConfigConstants.LDAP_HOSTS, "localhost:" + ldapPort) .put("users.u1.search", "(uid={0})") @@ -780,7 +783,6 @@ public void testLdapAuthorizationNestedAttrFilterAll() throws Exception { Assert.assertNotNull(user); Assert.assertEquals("spock", user.getName()); Assert.assertEquals(4, user.getRoles().size()); - } @Test diff --git a/src/test/java/org/opensearch/security/UtilTests.java b/src/test/java/org/opensearch/security/UtilTests.java index 3b6ed2edc9..402d5dc92f 100644 --- a/src/test/java/org/opensearch/security/UtilTests.java +++ b/src/test/java/org/opensearch/security/UtilTests.java @@ -70,6 +70,8 @@ public void testWildcardMatcherClasses() { assertTrue(wc("/\\S+/").test("abc")); assertTrue(wc("abc").test("abc")); assertFalse(wc("ABC").test("abc")); + assertFalse(wc(null).test("abc")); + assertTrue(WildcardMatcher.from(null, "abc").test("abc")); } @Test From babf0123383d8f8334c128071e75ec1334cc1725 Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Wed, 14 Feb 2024 14:05:55 -0500 Subject: [PATCH 116/204] Add exlusion for logback-core to resolve CVE-2023-6378 (#4049) ### Description [Describe what this change achieves] This change adds an exclusion for the transitive logback-core dependency that the Security plugin was still using as a test dependency. This should resolve the flagging of CVE-2023-6378 even though we should not have been directly impacted. ### Check List - [ ] ~New functionality includes testing~ - [ ] ~New functionality has been documented~ - [x] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). Signed-off-by: Stephen Crawford --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index d5d1dbae3d..15cbcb0426 100644 --- a/build.gradle +++ b/build.gradle @@ -707,6 +707,7 @@ dependencies { testRuntimeOnly 'com.typesafe.scala-logging:scala-logging_3:3.9.5' testRuntimeOnly('org.apache.zookeeper:zookeeper:3.9.1') { exclude(group:'ch.qos.logback', module: 'logback-classic' ) + exclude(group:'ch.qos.logback', module: 'logback-core' ) } testRuntimeOnly "org.apache.kafka:kafka-metadata:${kafka_version}" testRuntimeOnly "org.apache.kafka:kafka-storage:${kafka_version}" From b7b49b9de147126a7c7a3ed0f2f1a33d02eaee9f Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Wed, 14 Feb 2024 16:38:32 -0500 Subject: [PATCH 117/204] Update Bouncy Castle Version from *jdk15to18 to *jdk18on (#4052) ### Description [Describe what this change achieves] Following: https://github.com/opensearch-project/OpenSearch/pull/12317 in core, this PR increases the version used for bouncycastle in the Security plugin. This is an attempt to correct the intermittent failures described here: [#3299](https://github.com/opensearch-project/security/issues/3299) ### Check List - [ ] ~New functionality includes testing~ - [ ] ~New functionality has been documented~ - [x] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). Signed-off-by: Stephen Crawford --- build.gradle | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 15cbcb0426..b50cdcc7a7 100644 --- a/build.gradle +++ b/build.gradle @@ -582,7 +582,7 @@ dependencies { implementation "com.google.guava:guava:${guava_version}" implementation 'org.greenrobot:eventbus-java:3.3.1' implementation 'commons-cli:commons-cli:1.6.0' - implementation "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}" + implementation "org.bouncycastle:bcprov-jdk18on:${versions.bouncycastle}" implementation 'org.ldaptive:ldaptive:1.2.3' implementation 'com.nimbusds:nimbus-jose-jwt:9.37.3' @@ -654,7 +654,7 @@ dependencies { runtimeOnly 'org.apache.santuario:xmlsec:2.3.4' runtimeOnly "com.github.luben:zstd-jni:${versions.zstd}" runtimeOnly 'org.checkerframework:checker-qual:3.42.0' - runtimeOnly "org.bouncycastle:bcpkix-jdk15to18:${versions.bouncycastle}" + runtimeOnly "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}" runtimeOnly 'org.scala-lang.modules:scala-java8-compat_3:1.0.2' @@ -728,8 +728,8 @@ dependencies { integrationTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}" integrationTestImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}" integrationTestImplementation 'org.hamcrest:hamcrest:2.2' - integrationTestImplementation "org.bouncycastle:bcpkix-jdk15to18:${versions.bouncycastle}" - integrationTestImplementation "org.bouncycastle:bcutil-jdk15to18:${versions.bouncycastle}" + integrationTestImplementation "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}" + integrationTestImplementation "org.bouncycastle:bcutil-jdk18on:${versions.bouncycastle}" integrationTestImplementation('org.awaitility:awaitility:4.2.0') { exclude(group: 'org.hamcrest', module: 'hamcrest') } From 9fa07370036d32d9cd1e00ba86751dbb2c8ca5c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 08:42:29 -0600 Subject: [PATCH 118/204] Bump com.google.errorprone:error_prone_annotations from 2.24.1 to 2.25.0 (#4057) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index b50cdcc7a7..bfce9121b1 100644 --- a/build.gradle +++ b/build.gradle @@ -499,7 +499,7 @@ configurations { // For integrationTest force "org.apache.httpcomponents:httpclient:4.5.14" force "org.apache.httpcomponents:httpcore:4.4.16" - force "com.google.errorprone:error_prone_annotations:2.24.1" + force "com.google.errorprone:error_prone_annotations:2.25.0" force "org.checkerframework:checker-qual:3.42.0" force "ch.qos.logback:logback-classic:1.2.13" } @@ -609,7 +609,7 @@ dependencies { runtimeOnly 'com.eclipsesource.minimal-json:minimal-json:0.9.5' runtimeOnly 'commons-codec:commons-codec:1.16.1' runtimeOnly 'org.cryptacular:cryptacular:1.2.6' - compileOnly 'com.google.errorprone:error_prone_annotations:2.24.1' + compileOnly 'com.google.errorprone:error_prone_annotations:2.25.0' runtimeOnly 'com.sun.istack:istack-commons-runtime:4.2.0' runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.1' runtimeOnly 'org.ow2.asm:asm:9.6' From bd3415a63c6bf4ea513543a3c59b88b7be79563f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 08:42:40 -0600 Subject: [PATCH 119/204] Bump spring_version from 5.3.31 to 5.3.32 (#4056) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index bfce9121b1..76974f469b 100644 --- a/build.gradle +++ b/build.gradle @@ -32,7 +32,7 @@ buildscript { jjwt_version = '0.12.5' guava_version = '32.1.3-jre' jaxb_version = '2.3.9' - spring_version = '5.3.31' + spring_version = '5.3.32' if (buildVersionQualifier) { opensearch_build += "-${buildVersionQualifier}" From ccea7441545c807b7c4eb531706b1d919ea11988 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 21 Feb 2024 20:48:31 -0500 Subject: [PATCH 120/204] Fix unconsumed parameter exception when authenticating with jwtUrlParameter (#3975) Signed-off-by: Craig Perkins Signed-off-by: Peter Nied Signed-off-by: Peter Nied Co-authored-by: Peter Nied Co-authored-by: Peter Nied --- .../JwtAuthenticationWithUrlParamTests.java | 108 ++++++++++++++++++ .../test/framework/JwtConfigBuilder.java | 9 ++ .../framework/cluster/TestRestClient.java | 7 ++ .../security/filter/NettyRequest.java | 35 +++++- .../security/filter/OpenSearchRequest.java | 15 ++- .../security/filter/SecurityRequest.java | 4 + .../filter/SecurityRequestChannel.java | 4 +- .../security/filter/SecurityRestFilter.java | 8 ++ .../http/SecurityHttpServerTransport.java | 3 + .../Netty4HttpRequestHeaderVerifier.java | 3 + 10 files changed, 192 insertions(+), 4 deletions(-) create mode 100644 src/integrationTest/java/org/opensearch/security/http/JwtAuthenticationWithUrlParamTests.java diff --git a/src/integrationTest/java/org/opensearch/security/http/JwtAuthenticationWithUrlParamTests.java b/src/integrationTest/java/org/opensearch/security/http/JwtAuthenticationWithUrlParamTests.java new file mode 100644 index 0000000000..6dfb3c3bbc --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/http/JwtAuthenticationWithUrlParamTests.java @@ -0,0 +1,108 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.security.http; + +import java.security.KeyPair; +import java.util.Base64; +import java.util.List; +import java.util.Map; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.apache.hc.core5.http.Header; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.test.framework.JwtConfigBuilder; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; +import org.opensearch.test.framework.log.LogsRule; + +import io.jsonwebtoken.SignatureAlgorithm; +import io.jsonwebtoken.security.Keys; + +import static java.nio.charset.StandardCharsets.US_ASCII; +import static org.apache.http.HttpHeaders.AUTHORIZATION; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.BASIC_AUTH_DOMAIN_ORDER; +import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS; + +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class JwtAuthenticationWithUrlParamTests { + + public static final String CLAIM_USERNAME = "preferred-username"; + public static final String CLAIM_ROLES = "backend-user-roles"; + public static final String POINTER_USERNAME = "/user_name"; + + private static final KeyPair KEY_PAIR = Keys.keyPairFor(SignatureAlgorithm.RS256); + private static final String PUBLIC_KEY = new String(Base64.getEncoder().encode(KEY_PAIR.getPublic().getEncoded()), US_ASCII); + + static final TestSecurityConfig.User ADMIN_USER = new TestSecurityConfig.User("admin").roles(ALL_ACCESS); + + private static final String TOKEN_URL_PARAM = "token"; + + private static final JwtAuthorizationHeaderFactory tokenFactory = new JwtAuthorizationHeaderFactory( + KEY_PAIR.getPrivate(), + CLAIM_USERNAME, + CLAIM_ROLES, + AUTHORIZATION + ); + + public static final TestSecurityConfig.AuthcDomain JWT_AUTH_DOMAIN = new TestSecurityConfig.AuthcDomain( + "jwt", + BASIC_AUTH_DOMAIN_ORDER - 1 + ).jwtHttpAuthenticator( + new JwtConfigBuilder().jwtUrlParameter(TOKEN_URL_PARAM).signingKey(PUBLIC_KEY).subjectKey(CLAIM_USERNAME).rolesKey(CLAIM_ROLES) + ).backend("noop"); + + @ClassRule + public static final LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.SINGLENODE) + .anonymousAuth(false) + .nodeSettings( + Map.of("plugins.security.restapi.roles_enabled", List.of("user_" + ADMIN_USER.getName() + "__" + ALL_ACCESS.getName())) + ) + .authc(AUTHC_HTTPBASIC_INTERNAL) + .authc(JWT_AUTH_DOMAIN) + .users(ADMIN_USER) + .build(); + + @Rule + public LogsRule logsRule = new LogsRule("com.amazon.dlic.auth.http.jwt.HTTPJwtAuthenticator"); + + @Test + public void shouldAuthenticateWithJwtTokenInUrl_positive() { + Header jwtToken = tokenFactory.generateValidToken(ADMIN_USER.getName()); + String jwtTokenValue = jwtToken.getValue(); + try (TestRestClient client = cluster.getRestClient()) { + HttpResponse response = client.getAuthInfo(Map.of(TOKEN_URL_PARAM, jwtTokenValue)); + + response.assertStatusCode(200); + String username = response.getTextFromJsonBody(POINTER_USERNAME); + assertThat(username, equalTo(ADMIN_USER.getName())); + } + } + + @Test + public void testUnauthenticatedRequest() { + try (TestRestClient client = cluster.getRestClient()) { + HttpResponse response = client.getAuthInfo(); + + response.assertStatusCode(401); + logsRule.assertThatContainExactly(String.format("No JWT token found in '%s' url parameter header", TOKEN_URL_PARAM)); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/JwtConfigBuilder.java b/src/integrationTest/java/org/opensearch/test/framework/JwtConfigBuilder.java index 48dfa128e0..88297bacd2 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/JwtConfigBuilder.java +++ b/src/integrationTest/java/org/opensearch/test/framework/JwtConfigBuilder.java @@ -18,6 +18,7 @@ public class JwtConfigBuilder { private String jwtHeader; + private String jwtUrlParameter; private String signingKey; private String subjectKey; private String rolesKey; @@ -27,6 +28,11 @@ public JwtConfigBuilder jwtHeader(String jwtHeader) { return this; } + public JwtConfigBuilder jwtUrlParameter(String jwtUrlParameter) { + this.jwtUrlParameter = jwtUrlParameter; + return this; + } + public JwtConfigBuilder signingKey(String signingKey) { this.signingKey = signingKey; return this; @@ -51,6 +57,9 @@ public Map build() { if (isNoneBlank(jwtHeader)) { builder.put("jwt_header", jwtHeader); } + if (isNoneBlank(jwtUrlParameter)) { + builder.put("jwt_url_parameter", jwtUrlParameter); + } if (isNoneBlank(subjectKey)) { builder.put("subject_key", subjectKey); } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java index c2e01bb338..b7da92b270 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -36,6 +36,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -113,6 +114,12 @@ public HttpResponse getAuthInfo(Header... headers) { return executeRequest(new HttpGet(getHttpServerUri() + "/_opendistro/_security/authinfo?pretty"), headers); } + public HttpResponse getAuthInfo(Map urlParams, Header... headers) { + String urlParamsString = "?" + + urlParams.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&")); + return executeRequest(new HttpGet(getHttpServerUri() + "/_opendistro/_security/authinfo" + urlParamsString), headers); + } + public void confirmCorrectCredentials(String expectedUserName) { HttpResponse response = getAuthInfo(); assertThat(response, notNullValue()); diff --git a/src/main/java/org/opensearch/security/filter/NettyRequest.java b/src/main/java/org/opensearch/security/filter/NettyRequest.java index 7b65e4e0de..c827ddc779 100644 --- a/src/main/java/org/opensearch/security/filter/NettyRequest.java +++ b/src/main/java/org/opensearch/security/filter/NettyRequest.java @@ -14,12 +14,17 @@ import java.net.InetSocketAddress; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.TreeMap; import javax.net.ssl.SSLEngine; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; + import org.opensearch.http.netty4.Netty4HttpChannel; import org.opensearch.rest.RestRequest.Method; import org.opensearch.rest.RestUtils; @@ -34,6 +39,7 @@ public class NettyRequest implements SecurityRequest { protected final HttpRequest underlyingRequest; protected final Netty4HttpChannel underlyingChannel; + protected final Supplier parameters = Suppliers.memoize(() -> new CheckedAccessMap(params(uri()))); NettyRequest(final HttpRequest request, final Netty4HttpChannel channel) { this.underlyingRequest = request; @@ -82,7 +88,12 @@ public String uri() { @Override public Map params() { - return params(underlyingRequest.uri()); + return parameters.get(); + } + + @Override + public Set getUnconsumedParams() { + return parameters.get().accessedKeys(); } private static Map params(String uri) { @@ -100,4 +111,26 @@ private static Map params(String uri) { return params; } + + /** Records access of any keys if explicitly requested from this map */ + private static class CheckedAccessMap extends HashMap { + private final Set accessedKeys = new HashSet<>(); + + public CheckedAccessMap(final Map map) { + super(map); + } + + @Override + public String get(final Object key) { + // Never noticed this about java's map interface the getter is not generic + if (key instanceof String) { + accessedKeys.add((String) key); + } + return super.get(key); + } + + public Set accessedKeys() { + return accessedKeys; + } + } } diff --git a/src/main/java/org/opensearch/security/filter/OpenSearchRequest.java b/src/main/java/org/opensearch/security/filter/OpenSearchRequest.java index e86012f594..7cca8111c9 100644 --- a/src/main/java/org/opensearch/security/filter/OpenSearchRequest.java +++ b/src/main/java/org/opensearch/security/filter/OpenSearchRequest.java @@ -12,9 +12,11 @@ package org.opensearch.security.filter; import java.net.InetSocketAddress; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import javax.net.ssl.SSLEngine; import org.opensearch.rest.RestRequest; @@ -69,7 +71,18 @@ public String uri() { @Override public Map params() { - return underlyingRequest.params(); + return new HashMap<>(underlyingRequest.params()) { + @Override + public String get(Object key) { + return underlyingRequest.param((String) key); + } + }; + } + + @Override + public Set getUnconsumedParams() { + // params() Map consumes explict parameter access + return Set.of(); } /** Gets access to the underlying request object */ diff --git a/src/main/java/org/opensearch/security/filter/SecurityRequest.java b/src/main/java/org/opensearch/security/filter/SecurityRequest.java index 4c7ea27a87..d3741585ac 100644 --- a/src/main/java/org/opensearch/security/filter/SecurityRequest.java +++ b/src/main/java/org/opensearch/security/filter/SecurityRequest.java @@ -15,6 +15,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Stream; import javax.net.ssl.SSLEngine; @@ -49,4 +50,7 @@ default String header(final String headerName) { /** The parameters associated with this request */ Map params(); + + /** The list of parameters that have been accessed but not recorded as being consumed */ + Set getUnconsumedParams(); } diff --git a/src/main/java/org/opensearch/security/filter/SecurityRequestChannel.java b/src/main/java/org/opensearch/security/filter/SecurityRequestChannel.java index 66744d01dd..241c9009f4 100644 --- a/src/main/java/org/opensearch/security/filter/SecurityRequestChannel.java +++ b/src/main/java/org/opensearch/security/filter/SecurityRequestChannel.java @@ -14,13 +14,13 @@ import java.util.Optional; /** - * When a request is recieved by the security plugin this governs getting information about the request and complete with with a response + * When a request is received by the security plugin this governs getting information about the request and complete with a response */ public interface SecurityRequestChannel extends SecurityRequest { /** Associate a response with this channel */ void queueForSending(final SecurityResponse response); - /** Acess the queued response */ + /** Access the queued response */ Optional getQueuedResponse(); } diff --git a/src/main/java/org/opensearch/security/filter/SecurityRestFilter.java b/src/main/java/org/opensearch/security/filter/SecurityRestFilter.java index d52c5109fc..b649b8d71f 100644 --- a/src/main/java/org/opensearch/security/filter/SecurityRestFilter.java +++ b/src/main/java/org/opensearch/security/filter/SecurityRestFilter.java @@ -72,6 +72,7 @@ import static org.opensearch.security.http.SecurityHttpServerTransport.CONTEXT_TO_RESTORE; import static org.opensearch.security.http.SecurityHttpServerTransport.EARLY_RESPONSE; import static org.opensearch.security.http.SecurityHttpServerTransport.IS_AUTHENTICATED; +import static org.opensearch.security.http.SecurityHttpServerTransport.UNCONSUMED_PARAMS; public class SecurityRestFilter { @@ -144,6 +145,13 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c } }); + NettyAttribute.popFrom(request, UNCONSUMED_PARAMS).ifPresent(unconsumedParams -> { + for (String unconsumedParam : unconsumedParams) { + // Consume the parameter on the RestRequest + request.param(unconsumedParam); + } + }); + final SecurityRequestChannel requestChannel = SecurityRequestFactory.from(request, channel); // Authenticate request diff --git a/src/main/java/org/opensearch/security/http/SecurityHttpServerTransport.java b/src/main/java/org/opensearch/security/http/SecurityHttpServerTransport.java index c5fbbfbbc6..eb75f898f4 100644 --- a/src/main/java/org/opensearch/security/http/SecurityHttpServerTransport.java +++ b/src/main/java/org/opensearch/security/http/SecurityHttpServerTransport.java @@ -26,6 +26,8 @@ package org.opensearch.security.http; +import java.util.Set; + import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; @@ -47,6 +49,7 @@ public class SecurityHttpServerTransport extends SecuritySSLNettyHttpServerTransport { public static final AttributeKey EARLY_RESPONSE = AttributeKey.newInstance("opensearch-http-early-response"); + public static final AttributeKey> UNCONSUMED_PARAMS = AttributeKey.newInstance("opensearch-http-request-consumed-params"); public static final AttributeKey CONTEXT_TO_RESTORE = AttributeKey.newInstance( "opensearch-http-request-thread-context" ); diff --git a/src/main/java/org/opensearch/security/ssl/http/netty/Netty4HttpRequestHeaderVerifier.java b/src/main/java/org/opensearch/security/ssl/http/netty/Netty4HttpRequestHeaderVerifier.java index 9adca0f377..e6dec0c213 100644 --- a/src/main/java/org/opensearch/security/ssl/http/netty/Netty4HttpRequestHeaderVerifier.java +++ b/src/main/java/org/opensearch/security/ssl/http/netty/Netty4HttpRequestHeaderVerifier.java @@ -36,6 +36,7 @@ import static org.opensearch.security.http.SecurityHttpServerTransport.EARLY_RESPONSE; import static org.opensearch.security.http.SecurityHttpServerTransport.IS_AUTHENTICATED; import static org.opensearch.security.http.SecurityHttpServerTransport.SHOULD_DECOMPRESS; +import static org.opensearch.security.http.SecurityHttpServerTransport.UNCONSUMED_PARAMS; @Sharable public class Netty4HttpRequestHeaderVerifier extends SimpleChannelInboundHandler { @@ -84,6 +85,8 @@ public void channelRead0(ChannelHandlerContext ctx, DefaultHttpRequest msg) thro // If request channel is completed and a response is sent, then there was a failure during authentication restFilter.checkAndAuthenticateRequest(requestChannel); + ctx.channel().attr(UNCONSUMED_PARAMS).set(requestChannel.getUnconsumedParams()); + ThreadContext.StoredContext contextToRestore = threadPool.getThreadContext().newStoredContext(false); ctx.channel().attr(CONTEXT_TO_RESTORE).set(contextToRestore); From 9a6a018c44aeae20639dfb45ff5c74b2ec3822d8 Mon Sep 17 00:00:00 2001 From: Darshit Chanpura <35282393+DarshitChanpura@users.noreply.github.com> Date: Thu, 22 Feb 2024 11:13:53 -0500 Subject: [PATCH 121/204] Regenerates root-ca, kirk and esnode certificates to address already expired root ca certificate (#4061) ### Description During the last renewal of certs https://github.com/opensearch-project/security/pull/3268, the option `-days 3650` was missed for root-ca.pem cert causing it to set the default expiry of 30 days. This PR regenerates the public cert root-ca.pem, using the same private-key, and it also regenerate public certs `es-node.pem` and `kirk.pem` so that they can be verified with this new certificate. * Category : Bug fix * Why these changes are required? - To ensure the expiry is in 10 years from now * What is the old behavior before changes and new behavior after changes? - root-ca is currently expired, and this change will set expiry to 2034 ### Issues Resolved - Resolves https://github.com/opensearch-project/security/issues/4047 ### Testing - Automated testing + [Manual Testing](https://github.com/opensearch-project/security/pull/4061#issuecomment-1954817947) --------- Signed-off-by: Darshit Chanpura --- DEVELOPER_GUIDE.md | 8 ++- .../src/test/resources/security/esnode.pem | 16 ++--- bwc-test/src/test/resources/security/kirk.pem | 28 ++++---- .../src/test/resources/security/root-ca.pem | 18 ++--- .../security/OpenSearchSecurityPlugin.java | 5 ++ .../tools/democonfig/Certificates.java | 62 +++++++++--------- .../democonfig/CertificateGeneratorTests.java | 6 -- .../resources/sanity-tests/kirk-keystore.jks | Bin 4504 -> 3766 bytes src/test/resources/sanity-tests/root-ca.pem | 18 ++--- 9 files changed, 81 insertions(+), 80 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 472f3f06a9..4b5a53a8c0 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -86,7 +86,7 @@ rm -rf config/ ## ROOT openssl genrsa -out root-ca-key.pem 2048 -openssl req -new -x509 -sha256 -key root-ca-key.pem -subj "/DC=com/DC=example/O=Example Com Inc./OU=Example Com Inc. Root CA/CN=Example Com Inc. Root CA" -addext "basicConstraints = critical,CA:TRUE" -addext "keyUsage = critical, digitalSignature, keyCertSign, cRLSign" -addext "subjectKeyIdentifier = hash" -addext "authorityKeyIdentifier = keyid:always,issuer:always" -out root-ca.pem +openssl req -new -x509 -sha256 -days 3650 -key root-ca-key.pem -subj "/DC=com/DC=example/O=Example Com Inc./OU=Example Com Inc. Root CA/CN=Example Com Inc. Root CA" -addext "basicConstraints = critical,CA:TRUE" -addext "keyUsage = critical, digitalSignature, keyCertSign, cRLSign" -addext "subjectKeyIdentifier = hash" -addext "authorityKeyIdentifier = keyid:always,issuer:always" -out root-ca.pem ## NODE @@ -94,13 +94,15 @@ openssl req -new -x509 -sha256 -key root-ca-key.pem -subj "/DC=com/DC=example/O= openssl genrsa -out esnode-key-temp.pem 2048 openssl pkcs8 -inform PEM -outform PEM -in esnode-key-temp.pem -topk8 -nocrypt -v1 PBE-SHA1-3DES -out esnode-key.pem openssl req -new -key esnode-key.pem -subj "/C=de/L=test/O=node/OU=node/CN=node-0.example.com" -out esnode.csr -openssl x509 -req -in esnode.csr -out esnode.pem -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -days 3650 -extfile <(printf "subjectAltName = RID:1.2.3.4.5.5, DNS:node-0.example.com, DNS:localhost, IP:::1, IP:127.0.0.1\nkeyUsage = digitalSignature, nonRepudiation, keyEncipherment\nextendedKeyUsage = serverAuth, clientAuth\nbasicConstraints = critical,CA:FALSE") +printf "subjectAltName = RID:1.2.3.4.5.5, DNS:node-0.example.com, DNS:localhost, IP:::1, IP:127.0.0.1\nkeyUsage = digitalSignature, nonRepudiation, keyEncipherment\nextendedKeyUsage = serverAuth, clientAuth\nbasicConstraints = critical,CA:FALSE" > esnode_ext.conf +openssl x509 -req -in esnode.csr -out esnode.pem -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -days 3650 -extfile esnode_ext.conf ## ADMIN openssl req -new -newkey rsa:2048 -keyout kirk-key.pem -out kirk.csr -nodes -subj "/C=de/L=test/O=client/OU=client/CN=kirk" -openssl x509 -req -in kirk.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -out kirk.pem -days 3650 -extfile <(printf "basicConstraints = critical,CA:FALSE\nkeyUsage = critical,digitalSignature,nonRepudiation,keyEncipherment\nextendedKeyUsage = critical,clientAuth\nauthorityKeyIdentifier = keyid,issuer:always\nsubjectKeyIdentifier = hash") +printf "basicConstraints = critical,CA:FALSE\nkeyUsage = critical,digitalSignature,nonRepudiation,keyEncipherment\nextendedKeyUsage = critical,clientAuth\nauthorityKeyIdentifier=keyid,issuer:always\nsubjectKeyIdentifier = hash" > kirk_ext.conf +openssl x509 -req -in kirk.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -out kirk.pem -days 3650 -extfile kirk_ext.conf ## Remove root-ca-key.pem and other temp keys diff --git a/bwc-test/src/test/resources/security/esnode.pem b/bwc-test/src/test/resources/security/esnode.pem index 12801ce5e7..b690a603da 100644 --- a/bwc-test/src/test/resources/security/esnode.pem +++ b/bwc-test/src/test/resources/security/esnode.pem @@ -1,9 +1,9 @@ -----BEGIN CERTIFICATE----- -MIIEPDCCAySgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iMwDQYJKoZIhvcNAQEL +MIIEPDCCAySgAwIBAgIUaYSlET3nzsotWTrWueVPPh10yLYwDQYJKoZIhvcNAQEL BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v -dCBDQTAeFw0yMzA4MjkxOTQ0NDJaFw0zMzA4MjYxOTQ0NDJaMFcxCzAJBgNVBAYT +dCBDQTAeFw0yNDAyMjAxNzAzMjVaFw0zNDAyMTcxNzAzMjVaMFcxCzAJBgNVBAYT AmRlMQ0wCwYDVQQHDAR0ZXN0MQ0wCwYDVQQKDARub2RlMQ0wCwYDVQQLDARub2Rl MRswGQYDVQQDDBJub2RlLTAuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA A4IBDwAwggEKAoIBAQCm93kXteDQHMAvbUPNPW5pyRHKDD42XGWSgq0k1D29C/Ud @@ -16,10 +16,10 @@ BEAwPogFKgMEBQWCEm5vZGUtMC5leGFtcGxlLmNvbYIJbG9jYWxob3N0hxAAAAAA AAAAAAAAAAAAAAABhwR/AAABMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEF BQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU0/qDQaY10jIo wCjLUpz/HfQXyt8wHwYDVR0jBBgwFoAUF4ffoFrrZhKn1dD4uhJFPLcrAJwwDQYJ -KoZIhvcNAQELBQADggEBAHfKc6NUXJfsT5nL1CtaGy3dutV13UQkdUwEkB0BikkX -1PNaz2NeHyO2yQvp4G6WlovZB78tVqn5hbEZL7v8kUlAOTkjEJjsOu1Ib746eBdT -gmUBKpIeBrm3a+tsLR9OBOuDb8aQO6fnFehFs/70y0sbyRbVqSmxLaYgRkPhhqwl -3U7Ha1TpdJrckETk/iRcma0igvym1SvlUahgFXN4ZCLG3SycH+YRFtM749GVZBo5 -5E5gSfkWCj9jao3LjJn3ThtMsiL405uIPbFNm+5iXtflMk2aW666j1jlpeaZySuy -DWBtA+T5Y6HhDECSjHOV131UekvHLF+SbWrv0S+ptjA= +KoZIhvcNAQELBQADggEBAGbij5WyF0dKhQodQfTiFDb73ygU6IyeJkFSnxF67gDz +pQJZKFvXuVBa3cGP5e7Qp3TK50N+blXGH0xXeIV9lXeYUk4hVfBlp9LclZGX8tGi +7Xa2enMvIt5q/Yg3Hh755ZxnDYxCoGkNOXUmnMusKstE0YzvZ5Gv6fcRKFBUgZLh +hUBqIEAYly1EqH/y45APiRt3Nor1yF6zEI4TnL0yNrHw6LyQkUNCHIGMJLfnJQ9L +camMGIXOx60kXNMTigF9oXXwixWAnDM9y3QT8QXA7hej/4zkbO+vIeV/7lGUdkyg +PAi92EvyxmsliEMyMR0VINl8emyobvfwa7oMeWMR+hg= -----END CERTIFICATE----- diff --git a/bwc-test/src/test/resources/security/kirk.pem b/bwc-test/src/test/resources/security/kirk.pem index 716b4ec4d9..b89edfe18f 100644 --- a/bwc-test/src/test/resources/security/kirk.pem +++ b/bwc-test/src/test/resources/security/kirk.pem @@ -1,9 +1,9 @@ -----BEGIN CERTIFICATE----- -MIIEmDCCA4CgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iYwDQYJKoZIhvcNAQEL +MIIEmDCCA4CgAwIBAgIUaYSlET3nzsotWTrWueVPPh10yLcwDQYJKoZIhvcNAQEL BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v -dCBDQTAeFw0yMzA4MjkyMDA2MzdaFw0zMzA4MjYyMDA2MzdaME0xCzAJBgNVBAYT +dCBDQTAeFw0yNDAyMjAxNzA0MjRaFw0zNDAyMTcxNzA0MjRaME0xCzAJBgNVBAYT AmRlMQ0wCwYDVQQHDAR0ZXN0MQ8wDQYDVQQKDAZjbGllbnQxDzANBgNVBAsMBmNs aWVudDENMAsGA1UEAwwEa2lyazCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC ggEBAJVcOAQlCiuB9emCljROAXnlsPbG7PE3kNz2sN+BbGuw686Wgyl3uToVHvVs @@ -12,16 +12,16 @@ O0l/BG9E3mRGo/r0w+jtTQ3aR2p6eoxaOYbVyEMYtFI4QZTkcgGIPGxm05y8xonx vV5pbSW9L7qAVDzQC8EYGQMMI4ccu0NcHKWtmTYJA/wDPE2JwhngHwbcIbc4cDz6 cG0S3FmgiKGuuSqUy35v/k3y7zMHQSdx7DSR2tzhH/bBL/9qGvpT71KKrxPtaxS0 bAqPcEkKWDo7IMlGGW7LaAWfGg8CAwEAAaOCASswggEnMAwGA1UdEwEB/wQCMAAw -DgYDVR0PAQH/BAQDAgXgMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMCMIHPBgNVHSME -gccwgcSAFBeH36Ba62YSp9XQ+LoSRTy3KwCcoYGVpIGSMIGPMRMwEQYKCZImiZPy -LGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQRXhh -bXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENB -MSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0GCFHfkrz782p+T9k0G -xGeM4+BrehWKMB0GA1UdDgQWBBSjMS8tgguX/V7KSGLoGg7K6XMzIDANBgkqhkiG -9w0BAQsFAAOCAQEANMwD1JYlwAh82yG1gU3WSdh/tb6gqaSzZK7R6I0L7slaXN9m -y2ErUljpTyaHrdiBFmPhU/2Kj2r+fIUXtXdDXzizx/JdmueT0nG9hOixLqzfoC9p -fAhZxM62RgtyZoaczQN82k1/geMSwRpEndFe3OH7arkS/HSbIFxQhAIy229eWe5d -1bUzP59iu7f3r567I4ob8Vy7PP+Ov35p7Vv4oDHHwgsdRzX6pvL6mmwVrQ3BfVec -h9Dqprr+ukYmjho76g6k5cQuRaB6MxqldzUg+2E7IHQP8MCF+co51uZq2nl33mtp -RGr6JbdHXc96zsLTL3saJQ8AWEfu1gbTVrwyRA== +DgYDVR0PAQH/BAQDAgXgMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMCMB0GA1UdDgQW +BBSjMS8tgguX/V7KSGLoGg7K6XMzIDCBzwYDVR0jBIHHMIHEgBQXh9+gWutmEqfV +0Pi6EkU8tysAnKGBlaSBkjCBjzETMBEGCgmSJomT8ixkARkWA2NvbTEXMBUGCgmS +JomT8ixkARkWB2V4YW1wbGUxGTAXBgNVBAoMEEV4YW1wbGUgQ29tIEluYy4xITAf +BgNVBAsMGEV4YW1wbGUgQ29tIEluYy4gUm9vdCBDQTEhMB8GA1UEAwwYRXhhbXBs +ZSBDb20gSW5jLiBSb290IENBghQNZAmZZn3EFOxBR4630XlhI+mo4jANBgkqhkiG +9w0BAQsFAAOCAQEACEUPPE66/Ot3vZqRGpjDjPHAdtOq+ebaglQhvYcnDw8LOZm8 +Gbh9M88CiO6UxC8ipQLTPh2yyeWArkpJzJK/Pi1eoF1XLiAa0sQ/RaJfQWPm9dvl +1ZQeK5vfD4147b3iBobwEV+CR04SKow0YeEEzAJvzr8YdKI6jqr+2GjjVqzxvRBy +KRVHWCFiR7bZhHGLq3br8hSu0hwjb3oGa1ZI8dui6ujyZt6nm6BoEkau3G/6+zq9 +E6vX3+8Fj4HKCAL6i0SwfGmEpTNp5WUhqibK/fMhhmMT4Mx6MxkT+OFnIjdUU0S/ +e3kgnG8qjficUr38CyEli1U0M7koIXUZI7r+LQ== -----END CERTIFICATE----- diff --git a/bwc-test/src/test/resources/security/root-ca.pem b/bwc-test/src/test/resources/security/root-ca.pem index 5948a73b30..854323e6fe 100644 --- a/bwc-test/src/test/resources/security/root-ca.pem +++ b/bwc-test/src/test/resources/security/root-ca.pem @@ -1,9 +1,9 @@ -----BEGIN CERTIFICATE----- -MIIExjCCA66gAwIBAgIUd+SvPvzan5P2TQbEZ4zj4Gt6FYowDQYJKoZIhvcNAQEL +MIIExjCCA66gAwIBAgIUDWQJmWZ9xBTsQUeOt9F5YSPpqOIwDQYJKoZIhvcNAQEL BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v -dCBDQTAeFw0yMzA4MjkwNDIwMDNaFw0yMzA5MjgwNDIwMDNaMIGPMRMwEQYKCZIm +dCBDQTAeFw0yNDAyMjAxNzAwMzZaFw0zNDAyMTcxNzAwMzZaMIGPMRMwEQYKCZIm iZPyLGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQ RXhhbXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290 IENBMSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0EwggEiMA0GCSqG @@ -18,11 +18,11 @@ F4ffoFrrZhKn1dD4uhJFPLcrAJwwgc8GA1UdIwSBxzCBxIAUF4ffoFrrZhKn1dD4 uhJFPLcrAJyhgZWkgZIwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJ k/IsZAEZFgdleGFtcGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYD VQQLDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUg -Q29tIEluYy4gUm9vdCBDQYIUd+SvPvzan5P2TQbEZ4zj4Gt6FYowDQYJKoZIhvcN -AQELBQADggEBAIopqco/k9RSjouTeKP4z0EVUxdD4qnNh1GLSRqyAVe0aChyKF5f -qt1Bd1XCY8D16RgekkKGHDpJhGCpel+vtIoXPBxUaGQNYxmJCf5OzLMODlcrZk5i -jHIcv/FMeK02NBcz/WQ3mbWHVwXLhmwqa2zBsF4FmPCJAbFLchLhkAv1HJifHbnD -jQzlKyl5jxam/wtjWxSm0iyso0z2TgyzY+MESqjEqB1hZkCFzD1xtUOCxbXgtKae -dgfHVFuovr3fNLV3GvQk0s9okDwDUcqV7DSH61e5bUMfE84o3of8YA7+HUoPV5Du -8sTOKRf7ncGXdDRA8aofW268pTCuIu3+g/Y= +Q29tIEluYy4gUm9vdCBDQYIUDWQJmWZ9xBTsQUeOt9F5YSPpqOIwDQYJKoZIhvcN +AQELBQADggEBAL3Q3AHUhMiLUy6OlLSt8wX9I2oNGDKbBu0atpUNDztk/0s3YLQC +YuXgN4KrIcMXQIuAXCx407c+pIlT/T1FNn+VQXwi56PYzxQKtlpoKUL3oPQE1d0V +6EoiNk+6UodvyZqpdQu7fXVentRMk1QX7D9otmiiNuX+GSxJhJC2Lyzw65O9EUgG +1yVJon6RkUGtqBqKIuLksKwEr//ELnjmXit4LQKSnqKr0FTCB7seIrKJNyb35Qnq +qy9a/Unhokrmdda1tr6MbqU8l7HmxLuSd/Ky+L0eDNtYv6YfMewtjg0TtAnFyQov +rdXmeq1dy9HLo3Ds4AFz3Gx9076TxcRS/iI= -----END CERTIFICATE----- diff --git a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java index 53493e7f6c..bde984531e 100644 --- a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java +++ b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java @@ -339,6 +339,11 @@ public OpenSearchSecurityPlugin(final Settings settings, final Path configPath) demoCertHashes.add("ba9c5a61065f7f6115188128ffbdaa18fca34562b78b811f082439e2bef1d282"); // esnode-key demoCertHashes.add("9948688bc4c7a198f2a0db1d91f4f54499b8626902d03361b6d43e822d3691e4"); // root-ca + // updates certs with renewed root-ca (02-2024) + demoCertHashes.add("a3556d6bb61f7bd63cb19b1c8d0078d30c12739dedb0455c5792ac8627782042"); // kirk + demoCertHashes.add("a2ce3f577a5031398c1b4f58761444d837b031d0aff7614f8b9b5e4a9d59dbd1"); // esnode + demoCertHashes.add("cd708e8dc707ae065f7ad8582979764b497f062e273d478054ab2f49c5469c6"); // root-ca + final SecurityManager sm = System.getSecurityManager(); if (sm != null) { diff --git a/src/main/java/org/opensearch/security/tools/democonfig/Certificates.java b/src/main/java/org/opensearch/security/tools/democonfig/Certificates.java index 8e2af4dac7..baff8d7078 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/Certificates.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/Certificates.java @@ -23,11 +23,11 @@ public enum Certificates { () -> getCertContent( List.of( "-----BEGIN CERTIFICATE-----", - "MIIEmDCCA4CgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iYwDQYJKoZIhvcNAQEL", + "MIIEmDCCA4CgAwIBAgIUaYSlET3nzsotWTrWueVPPh10yLcwDQYJKoZIhvcNAQEL", "BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt", "cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl", "IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v", - "dCBDQTAeFw0yMzA4MjkyMDA2MzdaFw0zMzA4MjYyMDA2MzdaME0xCzAJBgNVBAYT", + "dCBDQTAeFw0yNDAyMjAxNzA0MjRaFw0zNDAyMTcxNzA0MjRaME0xCzAJBgNVBAYT", "AmRlMQ0wCwYDVQQHDAR0ZXN0MQ8wDQYDVQQKDAZjbGllbnQxDzANBgNVBAsMBmNs", "aWVudDENMAsGA1UEAwwEa2lyazCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC", "ggEBAJVcOAQlCiuB9emCljROAXnlsPbG7PE3kNz2sN+BbGuw686Wgyl3uToVHvVs", @@ -36,18 +36,18 @@ public enum Certificates { "vV5pbSW9L7qAVDzQC8EYGQMMI4ccu0NcHKWtmTYJA/wDPE2JwhngHwbcIbc4cDz6", "cG0S3FmgiKGuuSqUy35v/k3y7zMHQSdx7DSR2tzhH/bBL/9qGvpT71KKrxPtaxS0", "bAqPcEkKWDo7IMlGGW7LaAWfGg8CAwEAAaOCASswggEnMAwGA1UdEwEB/wQCMAAw", - "DgYDVR0PAQH/BAQDAgXgMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMCMIHPBgNVHSME", - "gccwgcSAFBeH36Ba62YSp9XQ+LoSRTy3KwCcoYGVpIGSMIGPMRMwEQYKCZImiZPy", - "LGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQRXhh", - "bXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENB", - "MSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0GCFHfkrz782p+T9k0G", - "xGeM4+BrehWKMB0GA1UdDgQWBBSjMS8tgguX/V7KSGLoGg7K6XMzIDANBgkqhkiG", - "9w0BAQsFAAOCAQEANMwD1JYlwAh82yG1gU3WSdh/tb6gqaSzZK7R6I0L7slaXN9m", - "y2ErUljpTyaHrdiBFmPhU/2Kj2r+fIUXtXdDXzizx/JdmueT0nG9hOixLqzfoC9p", - "fAhZxM62RgtyZoaczQN82k1/geMSwRpEndFe3OH7arkS/HSbIFxQhAIy229eWe5d", - "1bUzP59iu7f3r567I4ob8Vy7PP+Ov35p7Vv4oDHHwgsdRzX6pvL6mmwVrQ3BfVec", - "h9Dqprr+ukYmjho76g6k5cQuRaB6MxqldzUg+2E7IHQP8MCF+co51uZq2nl33mtp", - "RGr6JbdHXc96zsLTL3saJQ8AWEfu1gbTVrwyRA==", + "DgYDVR0PAQH/BAQDAgXgMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMCMB0GA1UdDgQW", + "BBSjMS8tgguX/V7KSGLoGg7K6XMzIDCBzwYDVR0jBIHHMIHEgBQXh9+gWutmEqfV", + "0Pi6EkU8tysAnKGBlaSBkjCBjzETMBEGCgmSJomT8ixkARkWA2NvbTEXMBUGCgmS", + "JomT8ixkARkWB2V4YW1wbGUxGTAXBgNVBAoMEEV4YW1wbGUgQ29tIEluYy4xITAf", + "BgNVBAsMGEV4YW1wbGUgQ29tIEluYy4gUm9vdCBDQTEhMB8GA1UEAwwYRXhhbXBs", + "ZSBDb20gSW5jLiBSb290IENBghQNZAmZZn3EFOxBR4630XlhI+mo4jANBgkqhkiG", + "9w0BAQsFAAOCAQEACEUPPE66/Ot3vZqRGpjDjPHAdtOq+ebaglQhvYcnDw8LOZm8", + "Gbh9M88CiO6UxC8ipQLTPh2yyeWArkpJzJK/Pi1eoF1XLiAa0sQ/RaJfQWPm9dvl", + "1ZQeK5vfD4147b3iBobwEV+CR04SKow0YeEEzAJvzr8YdKI6jqr+2GjjVqzxvRBy", + "KRVHWCFiR7bZhHGLq3br8hSu0hwjb3oGa1ZI8dui6ujyZt6nm6BoEkau3G/6+zq9", + "E6vX3+8Fj4HKCAL6i0SwfGmEpTNp5WUhqibK/fMhhmMT4Mx6MxkT+OFnIjdUU0S/", + "e3kgnG8qjficUr38CyEli1U0M7koIXUZI7r+LQ==", "-----END CERTIFICATE-----" ) ) @@ -92,11 +92,11 @@ public enum Certificates { () -> getCertContent( List.of( "-----BEGIN CERTIFICATE-----", - "MIIEPDCCAySgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iIwDQYJKoZIhvcNAQEL", + "MIIEPDCCAySgAwIBAgIUaYSlET3nzsotWTrWueVPPh10yLYwDQYJKoZIhvcNAQEL", "BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt", "cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl", "IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v", - "dCBDQTAeFw0yMzA4MjkwNDIzMTJaFw0zMzA4MjYwNDIzMTJaMFcxCzAJBgNVBAYT", + "dCBDQTAeFw0yNDAyMjAxNzAzMjVaFw0zNDAyMTcxNzAzMjVaMFcxCzAJBgNVBAYT", "AmRlMQ0wCwYDVQQHDAR0ZXN0MQ0wCwYDVQQKDARub2RlMQ0wCwYDVQQLDARub2Rl", "MRswGQYDVQQDDBJub2RlLTAuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA", "A4IBDwAwggEKAoIBAQCm93kXteDQHMAvbUPNPW5pyRHKDD42XGWSgq0k1D29C/Ud", @@ -109,12 +109,12 @@ public enum Certificates { "AAAAAAAAAAAAAAABhwR/AAABMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEF", "BQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU0/qDQaY10jIo", "wCjLUpz/HfQXyt8wHwYDVR0jBBgwFoAUF4ffoFrrZhKn1dD4uhJFPLcrAJwwDQYJ", - "KoZIhvcNAQELBQADggEBAD2hkndVih6TWxoe/oOW0i2Bq7ScNO/n7/yHWL04HJmR", - "MaHv/Xjc8zLFLgHuHaRvC02ikWIJyQf5xJt0Oqu2GVbqXH9PBGKuEP2kCsRRyU27", - "zTclAzfQhqmKBTYQ/3lJ3GhRQvXIdYTe+t4aq78TCawp1nSN+vdH/1geG6QjMn5N", - "1FU8tovDd4x8Ib/0dv8RJx+n9gytI8n/giIaDCEbfLLpe4EkV5e5UNpOnRgJjjuy", - "vtZutc81TQnzBtkS9XuulovDE0qI+jQrKkKu8xgGLhgH0zxnPkKtUg2I3Aq6zl1L", - "zYkEOUF8Y25J6WeY88Yfnc0iigI+Pnz5NK8R9GL7TYo=", + "KoZIhvcNAQELBQADggEBAGbij5WyF0dKhQodQfTiFDb73ygU6IyeJkFSnxF67gDz", + "pQJZKFvXuVBa3cGP5e7Qp3TK50N+blXGH0xXeIV9lXeYUk4hVfBlp9LclZGX8tGi", + "7Xa2enMvIt5q/Yg3Hh755ZxnDYxCoGkNOXUmnMusKstE0YzvZ5Gv6fcRKFBUgZLh", + "hUBqIEAYly1EqH/y45APiRt3Nor1yF6zEI4TnL0yNrHw6LyQkUNCHIGMJLfnJQ9L", + "camMGIXOx60kXNMTigF9oXXwixWAnDM9y3QT8QXA7hej/4zkbO+vIeV/7lGUdkyg", + "PAi92EvyxmsliEMyMR0VINl8emyobvfwa7oMeWMR+hg=", "-----END CERTIFICATE-----" ) ) @@ -159,11 +159,11 @@ public enum Certificates { () -> getCertContent( List.of( "-----BEGIN CERTIFICATE-----", - "MIIExjCCA66gAwIBAgIUd+SvPvzan5P2TQbEZ4zj4Gt6FYowDQYJKoZIhvcNAQEL", + "MIIExjCCA66gAwIBAgIUDWQJmWZ9xBTsQUeOt9F5YSPpqOIwDQYJKoZIhvcNAQEL", "BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt", "cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl", "IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v", - "dCBDQTAeFw0yMzA4MjkwNDIwMDNaFw0yMzA5MjgwNDIwMDNaMIGPMRMwEQYKCZIm", + "dCBDQTAeFw0yNDAyMjAxNzAwMzZaFw0zNDAyMTcxNzAwMzZaMIGPMRMwEQYKCZIm", "iZPyLGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQ", "RXhhbXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290", "IENBMSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0EwggEiMA0GCSqG", @@ -178,13 +178,13 @@ public enum Certificates { "uhJFPLcrAJyhgZWkgZIwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJ", "k/IsZAEZFgdleGFtcGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYD", "VQQLDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUg", - "Q29tIEluYy4gUm9vdCBDQYIUd+SvPvzan5P2TQbEZ4zj4Gt6FYowDQYJKoZIhvcN", - "AQELBQADggEBAIopqco/k9RSjouTeKP4z0EVUxdD4qnNh1GLSRqyAVe0aChyKF5f", - "qt1Bd1XCY8D16RgekkKGHDpJhGCpel+vtIoXPBxUaGQNYxmJCf5OzLMODlcrZk5i", - "jHIcv/FMeK02NBcz/WQ3mbWHVwXLhmwqa2zBsF4FmPCJAbFLchLhkAv1HJifHbnD", - "jQzlKyl5jxam/wtjWxSm0iyso0z2TgyzY+MESqjEqB1hZkCFzD1xtUOCxbXgtKae", - "dgfHVFuovr3fNLV3GvQk0s9okDwDUcqV7DSH61e5bUMfE84o3of8YA7+HUoPV5Du", - "8sTOKRf7ncGXdDRA8aofW268pTCuIu3+g/Y=", + "Q29tIEluYy4gUm9vdCBDQYIUDWQJmWZ9xBTsQUeOt9F5YSPpqOIwDQYJKoZIhvcN", + "AQELBQADggEBAL3Q3AHUhMiLUy6OlLSt8wX9I2oNGDKbBu0atpUNDztk/0s3YLQC", + "YuXgN4KrIcMXQIuAXCx407c+pIlT/T1FNn+VQXwi56PYzxQKtlpoKUL3oPQE1d0V", + "6EoiNk+6UodvyZqpdQu7fXVentRMk1QX7D9otmiiNuX+GSxJhJC2Lyzw65O9EUgG", + "1yVJon6RkUGtqBqKIuLksKwEr//ELnjmXit4LQKSnqKr0FTCB7seIrKJNyb35Qnq", + "qy9a/Unhokrmdda1tr6MbqU8l7HmxLuSd/Ky+L0eDNtYv6YfMewtjg0TtAnFyQov", + "rdXmeq1dy9HLo3Ds4AFz3Gx9076TxcRS/iI=", "-----END CERTIFICATE-----" ) ) diff --git a/src/test/java/org/opensearch/security/tools/democonfig/CertificateGeneratorTests.java b/src/test/java/org/opensearch/security/tools/democonfig/CertificateGeneratorTests.java index 58cf6d1368..3b43311679 100644 --- a/src/test/java/org/opensearch/security/tools/democonfig/CertificateGeneratorTests.java +++ b/src/test/java/org/opensearch/security/tools/democonfig/CertificateGeneratorTests.java @@ -116,12 +116,6 @@ private static void checkCertificateValidity(String certPath) throws Exception { Instant expiry = expiryDate.toInstant(); Period duration = getPeriodBetween(x509Certificate.getNotBefore().toInstant(), expiry); - if (certPath.endsWith("-ca.pem")) { - // root-ca.pem is already expired as the validity is only 30 days from generation - // so we just check interval to be of 30 days - assertThat(duration.getDays(), equalTo(30)); - return; - } // we check that cert is valid for total of ~10 yrs // we don't check days as leaps years may cause flaky-ness diff --git a/src/test/resources/sanity-tests/kirk-keystore.jks b/src/test/resources/sanity-tests/kirk-keystore.jks index 6dbc51e714784fa58a4209c75deab8b9ed1698ff..6c8c5ef77e20980f8c78295b159256b805da6a28 100644 GIT binary patch literal 3766 zcmd^=c{r47AIImJ%`(PV###wuU&o%k$xbMgr4m`Pk2Tv-j4?=zEwY?!X|aVw)I`=A zPAY52Rt6yODkPjhAQ%WsfbL*f;mp!-018Nf*#Q6sf)b!}Nv;s_8gzOC@mTmi+D9F}jyYkhL=#Xk3eYM2csmxKA&W!xAdE{tZ2mEGS z;L%QU`DHcrbdbw$3GsKUvmfQu0Z^?sH7B)!W)eLbG*fXB^G$&6CbCnj4~ z*J>Rkut6vL1EvT!JqAq#X=O~#!JHQ#QVSPuOGlnLrXXB~{{FsGRq?o?I;>^GFEhMB zw;z!v1sXap8nq3zz&+prKs-DRPm*XsS4BaP6Z{8tM~n@m|rxMA=p6*i(w=7 z*2&*Yg-uWU$5|W>>g5h)Fn{3B={`skAJ5_wXB5pDwyj{vG1_{{Y-`wB_i^B!5PA|= zrx=_>rprb&75BQ=J)SKPAJI;?(D#46)o+a?SsR^-&qJjXY2ER8S*1ZvU`t7~M6?NKULuzlAZ8C#X9>8j2;WDY z(TY-^!`&0%67`u|U_-Y(knWVcSlh-kwZQ6KG@S?L`W!iVl>Gyd(LnpMc@C!QeY{(E z)uAwF_CcqH#00}jer2dQk3}R|p^87XCxR8`n4c@g9rASTt9$8}SuGW!!+QQ&w&G!P zvv5Mft<&pzv^&XuuQAj&ieoa*3nI-hx}0`4kym=(cd>?v6yM3v43y@5@;yPeJ_N{@ z622W$@5Z4VqliMF3GAf_RcB;$HX^%cwTCgxg^4)5I0?*&oW|giBB@nUNBO+IX=iON zo~;L}HOwhyeqH4GHvAQ5i=|0c+_5*661aDyT_tr=I#+Zog%!9nRiuBb8m&SS4qp2fv7HJMG zwJFuqV*Hoq3`|Mayml;So|9W4Um6Lu8(k+(Hc2}p@&>?!7!7H~9*O%@BrKNAOa-~e z$e6#G)fJ+wNz5x9zU;#>&V}d z?!F1W_eNN;&LI9$!kWa0Zqa)0CVM4D=x(r>aXgW=XQ)PTRsJJ&MC?WjjoMwLRh`-I z8yD|^&(r#NU|pRpRF%wn&t%X`)8HQe%uxEKnXxIu9yui1s$eH0*YZ^Wvt25yOg6{5 zPefKstjqam-PRDz=&-BVb^xZe>{C{$cza!_sV&3M*l0ocMJVr!l~TlJi4JChDn9Nn zc&la1caY}0P&Ho=r;)l;mKBf$V<6A*R6XC}s98g%I7ZIAFI=e6SqQ4;oevw)nw0%^ zKq9#$;{3R0zJv}#mr7@}e+5-(`{C?^vEE#xb7uBY=X#_1v+@~@l?W@Zaq+Yo9bpu& zR<0us_T`(Q6qp1xYb)Rq;tJ|aTZ&y5xqx<_j-|>1$SEi@3!A|| z9YH<3ub_#ai=2WG_V9iQ!NU8mB|$4ZK3Gr>_s15;6W-XV-*##3TjwoMP&yb zq!L{!sQoUn<_ZWb)BbzloM2Zs1tb=+FBn*$!EQmp3Ml#oe;g0);^XP&_osni`NR1A z0SL>FG{F)8;h%d#4-g0eK+%&0UD-=ghUr~yDQ?!lNE5tKiJ_rjY{@`Q1vjbVAFU;|?Qs;w|1hFx_ z`*jR7rVAU>9*yRSpD1)#aOb!)@ak(5hk;guG$_9)=K8Ie^uOP<63|FjrX2UEcJw07 zD5c?bxHD${?)1+CMgPg@0|kH>4NzJZO*;#rl-xA_8*SHCS}ygKZP7*uHbRtmaTE%n zp7Vt7QIt|IIN?)fyS#8IxKHO$?TeY{DpQl5^kyAd$HH^Aa)SJC+I0!ULR znF7*z6R6~{CCW6M^qKuU!N`I`>YB3i6toA7f7#3%T&$5&wm0nY{&d9(g)LB$%g9dX zf>HfjVn9;)rG-^=)tiGDd<5M4wDHPl@yEGU_whSh78l$%S*WCqjvj^Xt?_VKp0T{pQGU!F;?_^4EMT$__$E zH0hMGQlo@W2p^_tPZsnirl@pGb<#0a^*g5ihYtSzKKx%Wg;i4h8B_c6Z+PPWM!I%g zOr-dLp|0@RV@@&InVrwRJfPT~ZY840gT$Jl4)HP^qcTUWE~1&}C2wS3Sv9pJWiRva zyK}a9ilnrYe7SB$bu~GF&GM`D1h@ukNsJY|Yt>|?q(4gzgSUuGwSIfsmlD)%J2V0@ zTU&-58&x%P)-#Oev2~&}bv^wwRbD$?Enu(jJiuwM3shGOZ{$juY+RGk#m^`!p7+vO zAjWFn1{dq`T?N^TggHmN3~VGf^5?a_)R-cj5yfk-?V<|S)%uKn{YGL)7(~eAhWA56 zj7ZS7amp#qQM;t>%6F)v{1S-Gq>88IPiL?2X9=q_r$vhc4{Pd3$WssBMbZaV2W zu&8||{U99-3!x+JudoA1KSAx^0qg$*YLr)FKtJ($lC@k)W?khPY!~B&3F~Xnxs_WH)b*(MC{~@>r={U4@A6+2p8il>0lojdT`r8~C>rA6;jw^lZK9gk<_y!v za(Rbclc{1;TFBtT`lr|YO0}|UXzh>FLsx6RQUq8=?V4{NR#=oxL2}kHb-ZAfuNRt32Rtcg+B4PQKLo)5nT`xBt(f8 zz4zYx{`1az=l47B(|aH0%$a-V&c}OZ28N+d1QLK?7-~f#Qh{)-@KbUEVuBnDwFn`G zTJSH-2g86X{uc$#Cd7a<{=zALBY_C=KPs|Y1i%~&Sotp~4}12H0!$9GfJy&blEDNC z=>%hA9@l)1y-8vD6#cH^U}=KBI0FdeqXH7J!^nt8{(B;j6byi|5|P@4YY{kr2nhrT zsl1TD93_M516EPM#9d4EG(rsFKtBW4^r*(5KwKbTLB){+^0E(}Q+A7HoW0lrA)@i+ zydGtY^95cAh7C?*2qIcESObb&7%#|($|(-eXIiQ#0>bYpj@=?*4?U=5@-ISTdSa4x zOtEjIWb0hr)D^1HVpX7-CjwnsDG8#WM@AVZvyufeW?}`^GtGW7WcGsVl)G*$?lP3S z^GYelg04B!ZBp4GnwCzq@uOLfB4xY#hE;StB61*Yd8?%(Nl9NW{s3+HODy#ik72s%Hj($a8 zhF0>hs}=106=eHlR<&9zT@LuHAUIZWLFWrKQ#$R3^=pv*&-7e6{O_Ji`|s`^^4v@-Hr>`?(V#!ktZ-$-0?Jt1G-G? zE9HvN@-0iPpKSDRsLacPB>#JY4d$KM!zs7xPBvUu4HQ}!Bz$qc)A`=Ver4EBC?!g7b zuW7GvE*puJA=;!bv2_S?8ZQx_n`M?F&kkb{-h zKwO=OA_@auvAUmAsQW~NjYK|}m{>`{*n^45MJ^ph*%K9}8GnxA%-;D^^-}ih8oWP* zXJ#vzJY3e4?&oSey+_=qv19lq zeLI>%Gjx=y!qVzf%Y&c7dgkjEw?^rl8^KxGs^%{Fd_(b51&l(wYCO&Rc~ZUl5^~y> zc}BJ!4+n2KaS|<{vd#M44my1W|M0Y-gfk9<&l%IBje@31-Sr1Mt!fvT(Pe+Gt$Bz? z_up@HJf$b!)YfI|4{%l^JDxgWvp75|nMzg7E)(qZ%=alvt zXMfZg7Z=_eanGP?tBXFKyvFRu$?uMAzg|k-(32orZccxnHGr$(gM%4Hgc&3blJCi; z6j@^Y3XVg*doBz7pms~Jn7 z9>1&oI7bPBOnn7vyV1x>YahPMDy_bySw!71ij);ebzBEUSZK&o1y43I-AuJKXJ~C3 z{ScF0neCZB8?5r>Px#3V%} zq$OY&i2FZH#6&q5i2Yy421o$-o6P@Z2>vgd4p$sB)+@I7CAQvk>m=OVG#EC`^#8Hx zXo}&oS5+Eg(sw4>QN4_Cy_0U!W9o!pxS@}|4s+L{ow)59*P>fYuDV~JqCwTL5s{)3(v zzbM`$E?)E;`zu*Kjpah> zgQl1ucOJOd1|%MDBk_Lsu64*-#r>9orWT19xT!DnCoNv_AnWczl?5a3@Sd4mtPrx@ z;QPqXK#%ve%3=_Sa$)(zJ)mvCYW0$Uim6bQ!S}#H@uPFY+qvmT_x`cr%&q*~6sufG zKKVZ8ebd?WhVYT)or=?jzV*~PLH&t?CH^KO=IX%=oHNr75%vVz=nN9ipHOrX*7{h! zNkaI3@a@JfTINcbD<@;DNwqa&=S5v4pM=tBEMN8HU3}euq?(dEFWfNC>H+2C+1dBA zFs|s&27315cK^vG`LRKX~{Ugw!|2K~TP_VAqXtzNY6)j={rQ zv73v$!psb1ph9o6`kKlGjC8GEdFX9+@{I}q{33}%?v>$a-cw6HGOOLVnv3ITN_D~k zo^QL%)6K#_{j)b&>8Qy@Eweq=Ne8rKsjJTe)mfDw?scqlc&US2dxU0@o5$(Zu(GB4 zujr5^yZdwlP>E{wrkq=NiW~PQZm5`fJz5m&9I}B^zPVNSSa9vWcXu^m%+bU|aOg5q zK%|a72J^vxGy)&3GlNod=Wt|FBG=mgP)o%{(2PCL$9s$dMvIcv^FdM?hbNYQrX%I| z{binoW_?J27M3L2H_Y4n0!3PGL#b*UxRbpd3l$RLC#I})-32((m#4}vP%kHB3Q7PGLpvuro4~7i2u6z$3ar+YSP2?_%+^%f* zR}5Rl@nUnDVdT&uE_ZP%NU-(Zn*^k2*4S;xubW_f3f-cK+=>uy-sK;&F{mRdpgwIgSHfJSw=22paH-mu>R=3Kf9cR*A_Sjg7q#MM< zqobyHu#q_oM3;REOf&nTGa=n6MK4QZ{pey;iGwX&bnAUCVq`=c0{gykLm{VZo%ulF z*n_LEk%}KbmVW1)L+Ab3sSZPR+Fe*5p$^HC|Oyb{_is> zsuD42;l;BT-a#X6fP(~C+`TP&(``5KD7dp9)GD&EVfNN4Bf@5N63j4c_IOZZ`^gF1 zphj9>;b1JVOWrk`HhO{mmk*Lp>wXpL*r|VQth!^2ajO2-Q$=;E0ZcMzj9V;D}3k7ej?g$MEOSvfr*p<&b z6B?7p3F^a78y9pEd$#q2Pm1b zU#?c^Op~TXSZ`3z2a{A=UzcS`zB%Z|XG2xth@1`h=wY$wyp|u2)s&QN#af+k>`vF! z&{oB;K{Wblwtcc`JH%E!TwV2q%vd}p>iZ9d@C(kwR>Dm)p? zV-i0tv8PP66)jD1#I*Qm*`@U`^o)}|58+bGD1y(EEM_dJh-O9xP^xdF-_Z#qZ&m{c zbC6W;iNU!24Cvnj14>>_V8a{IB$GXu&z39rEKNX_07*3xp*W3rJo!}pp2M0Hwe$#* zi#HgV_>>SSD;YT=uK8*Lu|$a+IIXPF$${!eaPU%X#jh@y96VcWEFGqB#<_hE8QPmQ zO_C$p_nXzGgQtqVrC1t-5`*juoj0Q%VLnw`@Yt&eCg!x)84Pq&N%`@t**O@LYz3OR(@+})Hu&$>gJ;6oxdO{ z&KR3!hDx52>YBb*JE@4B`8}j*yOg=37>&zbSN}#T@GA6n9+dFcA*9q_l2eI%Xh*7~ ziU87?k{%5!@e5oasj8xTY|ysPyOMR3W;w?vvG}prD%~$8wf$j!6&K4LI%aD1$6B&8 zG|Bq_{em<75I~pVeMNJ6Dv9e{<=x@Es?2r|L;d(lJhNv+5~$`ps7`1lAq>B{Ot5Ga z6qD6CeNHKADuYBeC(!$C>E5yJ7O5IFfdN*2lPV*LTj(fX$`T*h6!l7_BFQ%HhbJFp zKUVk@Dl`5ZH)LoQ^{7N6?HyY_;Jo?*Uu#dn_XW`49o!xdK!+JJN_3KD7k@2J((0h0 z?0!++a*3VkR_Y8-s+o<1M(>PCz=|sJMqa z0+r0sNH_$gvD_@AC}TCb8}m~2v}_leWOtWdheZwxJl0i{OGIRcO0iVJ-B>5CgP^O-M7OYVJ*8(0|euX~UGp`sq@@gaEw*bHD4*Dj8_ zPO4*=dce-k-f;9Xl`P>A2U6SzIPhFWQT>2(PjqTMlBf}zL3<&dS*!E0mM}&jbXhc- zAb9}5!V(`=H1zl4fM|8TdAE{XwAuTJ>dTw3o}wzSb&xhxCijhe4Q#{|l(FXGy+A)j zH>IZrWy4|#?wJ-1?zBm;cKLHK*H5ngXeiJE?k?6Lz1i+02rcMG7kNDQlDJ_??0D#; z(Bju>vbV@>IGl97vC?TD(|fa!E?NjDA;*m&#_ZiX>Vgi+wr`atYOngkRp_w%?M~sv zUVImV4>dX4Ih+MO4LU`Ui=K%20a~JOwq1$6)KUw@81y#uUGKMV4>O0ioDGDvtZ{Jl zmay)x!zLD>Hl1jqnzX9b_da}w9xr9S`kQwUZPAei4I5Ao#$N}f9I10=!}MXIF!F!C z6+i+ofRKI2Rvlk8erCmgYu2%A6S_nSX7!cGJQ6pQ{xw*Iw(KXQGft90Ft(YQ<7nw! ROz*Khv5A{`^It3We*oUlR=)rM diff --git a/src/test/resources/sanity-tests/root-ca.pem b/src/test/resources/sanity-tests/root-ca.pem index 5948a73b30..854323e6fe 100644 --- a/src/test/resources/sanity-tests/root-ca.pem +++ b/src/test/resources/sanity-tests/root-ca.pem @@ -1,9 +1,9 @@ -----BEGIN CERTIFICATE----- -MIIExjCCA66gAwIBAgIUd+SvPvzan5P2TQbEZ4zj4Gt6FYowDQYJKoZIhvcNAQEL +MIIExjCCA66gAwIBAgIUDWQJmWZ9xBTsQUeOt9F5YSPpqOIwDQYJKoZIhvcNAQEL BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v -dCBDQTAeFw0yMzA4MjkwNDIwMDNaFw0yMzA5MjgwNDIwMDNaMIGPMRMwEQYKCZIm +dCBDQTAeFw0yNDAyMjAxNzAwMzZaFw0zNDAyMTcxNzAwMzZaMIGPMRMwEQYKCZIm iZPyLGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQ RXhhbXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290 IENBMSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0EwggEiMA0GCSqG @@ -18,11 +18,11 @@ F4ffoFrrZhKn1dD4uhJFPLcrAJwwgc8GA1UdIwSBxzCBxIAUF4ffoFrrZhKn1dD4 uhJFPLcrAJyhgZWkgZIwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJ k/IsZAEZFgdleGFtcGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYD VQQLDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUg -Q29tIEluYy4gUm9vdCBDQYIUd+SvPvzan5P2TQbEZ4zj4Gt6FYowDQYJKoZIhvcN -AQELBQADggEBAIopqco/k9RSjouTeKP4z0EVUxdD4qnNh1GLSRqyAVe0aChyKF5f -qt1Bd1XCY8D16RgekkKGHDpJhGCpel+vtIoXPBxUaGQNYxmJCf5OzLMODlcrZk5i -jHIcv/FMeK02NBcz/WQ3mbWHVwXLhmwqa2zBsF4FmPCJAbFLchLhkAv1HJifHbnD -jQzlKyl5jxam/wtjWxSm0iyso0z2TgyzY+MESqjEqB1hZkCFzD1xtUOCxbXgtKae -dgfHVFuovr3fNLV3GvQk0s9okDwDUcqV7DSH61e5bUMfE84o3of8YA7+HUoPV5Du -8sTOKRf7ncGXdDRA8aofW268pTCuIu3+g/Y= +Q29tIEluYy4gUm9vdCBDQYIUDWQJmWZ9xBTsQUeOt9F5YSPpqOIwDQYJKoZIhvcN +AQELBQADggEBAL3Q3AHUhMiLUy6OlLSt8wX9I2oNGDKbBu0atpUNDztk/0s3YLQC +YuXgN4KrIcMXQIuAXCx407c+pIlT/T1FNn+VQXwi56PYzxQKtlpoKUL3oPQE1d0V +6EoiNk+6UodvyZqpdQu7fXVentRMk1QX7D9otmiiNuX+GSxJhJC2Lyzw65O9EUgG +1yVJon6RkUGtqBqKIuLksKwEr//ELnjmXit4LQKSnqKr0FTCB7seIrKJNyb35Qnq +qy9a/Unhokrmdda1tr6MbqU8l7HmxLuSd/Ky+L0eDNtYv6YfMewtjg0TtAnFyQov +rdXmeq1dy9HLo3Ds4AFz3Gx9076TxcRS/iI= -----END CERTIFICATE----- From e40efdc0fbf901a8b932bdb36d966ff769cadf53 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 22 Feb 2024 17:25:58 -0500 Subject: [PATCH 122/204] Redact sensitive URL parameters from audit logging (#4067) Signed-off-by: Craig Perkins Signed-off-by: Craig Perkins Co-authored-by: Peter Nied --- .../JwtAuthenticationWithUrlParamTests.java | 20 +++++++++- .../test/framework/AuditFilters.java | 8 ++++ .../audit/AuditMessagePredicate.java | 30 ++++++++++++++- .../auth/http/jwt/HTTPJwtAuthenticator.java | 10 +++++ .../security/auditlog/config/AuditConfig.java | 38 +++++++++++++++++++ .../auditlog/impl/AbstractAuditLog.java | 18 +++++++++ .../security/auditlog/impl/AuditMessage.java | 14 +++++-- .../security/auth/HTTPAuthenticator.java | 12 ++++++ .../security/rest/SecurityInfoAction.java | 3 +- .../config/AuditConfigSerializeTest.java | 5 +++ .../dlic/rest/api/AuditApiActionTest.java | 2 +- 11 files changed, 152 insertions(+), 8 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/http/JwtAuthenticationWithUrlParamTests.java b/src/integrationTest/java/org/opensearch/security/http/JwtAuthenticationWithUrlParamTests.java index 6dfb3c3bbc..e10ad82e8c 100644 --- a/src/integrationTest/java/org/opensearch/security/http/JwtAuthenticationWithUrlParamTests.java +++ b/src/integrationTest/java/org/opensearch/security/http/JwtAuthenticationWithUrlParamTests.java @@ -21,8 +21,12 @@ import org.junit.Test; import org.junit.runner.RunWith; +import org.opensearch.test.framework.AuditCompliance; +import org.opensearch.test.framework.AuditConfiguration; +import org.opensearch.test.framework.AuditFilters; import org.opensearch.test.framework.JwtConfigBuilder; import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.audit.AuditLogsRule; import org.opensearch.test.framework.cluster.ClusterManager; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; @@ -36,9 +40,11 @@ import static org.apache.http.HttpHeaders.AUTHORIZATION; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.rest.RestRequest.Method.GET; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.BASIC_AUTH_DOMAIN_ORDER; import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS; +import static org.opensearch.test.framework.audit.AuditMessagePredicate.userAuthenticated; @RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) @ThreadLeakScope(ThreadLeakScope.Scope.NONE) @@ -69,12 +75,19 @@ public class JwtAuthenticationWithUrlParamTests { new JwtConfigBuilder().jwtUrlParameter(TOKEN_URL_PARAM).signingKey(PUBLIC_KEY).subjectKey(CLAIM_USERNAME).rolesKey(CLAIM_ROLES) ).backend("noop"); + @Rule + public AuditLogsRule auditLogsRule = new AuditLogsRule(); + @ClassRule public static final LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.SINGLENODE) .anonymousAuth(false) .nodeSettings( Map.of("plugins.security.restapi.roles_enabled", List.of("user_" + ADMIN_USER.getName() + "__" + ALL_ACCESS.getName())) ) + .audit( + new AuditConfiguration(true).compliance(new AuditCompliance().enabled(true)) + .filters(new AuditFilters().enabledRest(true).enabledTransport(true)) + ) .authc(AUTHC_HTTPBASIC_INTERNAL) .authc(JWT_AUTH_DOMAIN) .users(ADMIN_USER) @@ -88,11 +101,16 @@ public void shouldAuthenticateWithJwtTokenInUrl_positive() { Header jwtToken = tokenFactory.generateValidToken(ADMIN_USER.getName()); String jwtTokenValue = jwtToken.getValue(); try (TestRestClient client = cluster.getRestClient()) { - HttpResponse response = client.getAuthInfo(Map.of(TOKEN_URL_PARAM, jwtTokenValue)); + HttpResponse response = client.getAuthInfo(Map.of(TOKEN_URL_PARAM, jwtTokenValue, "verbose", "true")); response.assertStatusCode(200); String username = response.getTextFromJsonBody(POINTER_USERNAME); assertThat(username, equalTo(ADMIN_USER.getName())); + Map expectedParams = Map.of("token", "REDACTED", "verbose", "true"); + + auditLogsRule.assertExactlyOne( + userAuthenticated(ADMIN_USER).withRestRequest(GET, "/_opendistro/_security/authinfo").withRestParams(expectedParams) + ); } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/AuditFilters.java b/src/integrationTest/java/org/opensearch/test/framework/AuditFilters.java index 087342eb6f..5e63665f41 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/AuditFilters.java +++ b/src/integrationTest/java/org/opensearch/test/framework/AuditFilters.java @@ -35,6 +35,7 @@ public class AuditFilters implements ToXContentObject { private List ignoreRequests; private List ignoreHeaders; + private List ignoreUrlParams; private List disabledRestCategories; @@ -52,6 +53,7 @@ public AuditFilters() { this.ignoreUsers = Collections.emptyList(); this.ignoreRequests = Collections.emptyList(); this.ignoreHeaders = Collections.emptyList(); + this.ignoreUrlParams = Collections.emptyList(); this.disabledRestCategories = Collections.emptyList(); this.disabledTransportCategories = Collections.emptyList(); } @@ -101,6 +103,11 @@ public AuditFilters ignoreHeaders(List ignoreHeaders) { return this; } + public AuditFilters ignoreUrlParams(List ignoreUrlParams) { + this.ignoreUrlParams = ignoreUrlParams; + return this; + } + public AuditFilters disabledRestCategories(List disabledRestCategories) { this.disabledRestCategories = disabledRestCategories; return this; @@ -123,6 +130,7 @@ public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params xContentBuilder.field("ignore_users", ignoreUsers); xContentBuilder.field("ignore_requests", ignoreRequests); xContentBuilder.field("ignore_headers", ignoreHeaders); + xContentBuilder.field("ignore_url_params", ignoreUrlParams); xContentBuilder.field("disabled_rest_categories", disabledRestCategories); xContentBuilder.field("disabled_transport_categories", disabledTransportCategories); xContentBuilder.endObject(); diff --git a/src/integrationTest/java/org/opensearch/test/framework/audit/AuditMessagePredicate.java b/src/integrationTest/java/org/opensearch/test/framework/audit/AuditMessagePredicate.java index 4935bf0387..34565e9926 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/audit/AuditMessagePredicate.java +++ b/src/integrationTest/java/org/opensearch/test/framework/audit/AuditMessagePredicate.java @@ -29,6 +29,7 @@ import static org.opensearch.security.auditlog.impl.AuditCategory.MISSING_PRIVILEGES; import static org.opensearch.security.auditlog.impl.AuditMessage.REQUEST_LAYER; import static org.opensearch.security.auditlog.impl.AuditMessage.RESOLVED_INDICES; +import static org.opensearch.security.auditlog.impl.AuditMessage.REST_REQUEST_PARAMS; import static org.opensearch.security.auditlog.impl.AuditMessage.REST_REQUEST_PATH; public class AuditMessagePredicate implements Predicate { @@ -36,6 +37,7 @@ public class AuditMessagePredicate implements Predicate { private final AuditCategory category; private final Origin requestLayer; private final String restRequestPath; + private final Map restParams; private final String initiatingUser; private final Method requestMethod; private final String transportRequestType; @@ -47,6 +49,7 @@ private AuditMessagePredicate( AuditCategory category, Origin requestLayer, String restRequestPath, + Map restParams, String initiatingUser, Method requestMethod, String transportRequestType, @@ -57,6 +60,7 @@ private AuditMessagePredicate( this.category = category; this.requestLayer = requestLayer; this.restRequestPath = restRequestPath; + this.restParams = restParams; this.initiatingUser = initiatingUser; this.requestMethod = requestMethod; this.transportRequestType = transportRequestType; @@ -66,7 +70,7 @@ private AuditMessagePredicate( } private AuditMessagePredicate(AuditCategory category) { - this(category, null, null, null, null, null, null, null, null); + this(category, null, null, null, null, null, null, null, null, null); } public static AuditMessagePredicate auditPredicate(AuditCategory category) { @@ -110,6 +114,7 @@ public AuditMessagePredicate withLayer(Origin layer) { category, layer, restRequestPath, + restParams, initiatingUser, requestMethod, transportRequestType, @@ -124,6 +129,22 @@ public AuditMessagePredicate withRequestPath(String path) { category, requestLayer, path, + restParams, + initiatingUser, + requestMethod, + transportRequestType, + effectiveUser, + index, + privilege + ); + } + + public AuditMessagePredicate withRestParams(Map params) { + return new AuditMessagePredicate( + category, + requestLayer, + restRequestPath, + params, initiatingUser, requestMethod, transportRequestType, @@ -138,6 +159,7 @@ public AuditMessagePredicate withInitiatingUser(String user) { category, requestLayer, restRequestPath, + restParams, user, requestMethod, transportRequestType, @@ -156,6 +178,7 @@ public AuditMessagePredicate withRestMethod(Method method) { category, requestLayer, restRequestPath, + restParams, initiatingUser, method, transportRequestType, @@ -170,6 +193,7 @@ public AuditMessagePredicate withTransportRequestType(String type) { category, requestLayer, restRequestPath, + restParams, initiatingUser, requestMethod, type, @@ -184,6 +208,7 @@ public AuditMessagePredicate withEffectiveUser(String user) { category, requestLayer, restRequestPath, + restParams, initiatingUser, requestMethod, transportRequestType, @@ -206,6 +231,7 @@ public AuditMessagePredicate withIndex(String indexName) { category, requestLayer, restRequestPath, + restParams, initiatingUser, requestMethod, transportRequestType, @@ -220,6 +246,7 @@ public AuditMessagePredicate withPrivilege(String privilegeAction) { category, requestLayer, restRequestPath, + restParams, initiatingUser, requestMethod, transportRequestType, @@ -235,6 +262,7 @@ public boolean test(AuditMessage auditMessage) { predicates.add(audit -> Objects.isNull(category) || category.equals(audit.getCategory())); predicates.add(audit -> Objects.isNull(requestLayer) || requestLayer.equals(audit.getAsMap().get(REQUEST_LAYER))); predicates.add(audit -> Objects.isNull(restRequestPath) || restRequestPath.equals(audit.getAsMap().get(REST_REQUEST_PATH))); + predicates.add(audit -> Objects.isNull(restParams) || restParams.equals(auditMessage.getAsMap().get(REST_REQUEST_PARAMS))); predicates.add(audit -> Objects.isNull(initiatingUser) || initiatingUser.equals(audit.getInitiatingUser())); predicates.add(audit -> Objects.isNull(requestMethod) || requestMethod.equals(audit.getRequestMethod())); predicates.add(audit -> Objects.isNull(transportRequestType) || transportRequestType.equals(audit.getRequestType())); diff --git a/src/main/java/com/amazon/dlic/auth/http/jwt/HTTPJwtAuthenticator.java b/src/main/java/com/amazon/dlic/auth/http/jwt/HTTPJwtAuthenticator.java index 9bf22bf7f3..a6ff27eb6b 100644 --- a/src/main/java/com/amazon/dlic/auth/http/jwt/HTTPJwtAuthenticator.java +++ b/src/main/java/com/amazon/dlic/auth/http/jwt/HTTPJwtAuthenticator.java @@ -15,9 +15,11 @@ import java.security.AccessController; import java.security.PrivilegedAction; import java.util.Collection; +import java.util.Collections; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; +import java.util.Set; import java.util.regex.Pattern; import org.apache.http.HttpStatus; @@ -194,6 +196,14 @@ public Optional reRequestAuthentication(final SecurityRequest ); } + @Override + public Set getSensitiveUrlParams() { + if (jwtUrlParameter != null) { + return Set.of(jwtUrlParameter); + } + return Collections.emptySet(); + } + @Override public String getType() { return "jwt"; diff --git a/src/main/java/org/opensearch/security/auditlog/config/AuditConfig.java b/src/main/java/org/opensearch/security/auditlog/config/AuditConfig.java index 7b173099b5..3b3ee742b6 100644 --- a/src/main/java/org/opensearch/security/auditlog/config/AuditConfig.java +++ b/src/main/java/org/opensearch/security/auditlog/config/AuditConfig.java @@ -12,6 +12,7 @@ package org.opensearch.security.auditlog.config; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -142,9 +143,12 @@ public static class Filter { private final Set ignoredAuditRequests; @JsonProperty("ignore_headers") private final Set ignoredCustomHeaders; + @JsonProperty("ignore_url_params") + private Set ignoredUrlParams; private final WildcardMatcher ignoredAuditUsersMatcher; private final WildcardMatcher ignoredAuditRequestsMatcher; private final WildcardMatcher ignoredCustomHeadersMatcher; + private WildcardMatcher ignoredUrlParamsMatcher; private final Set disabledRestCategories; private final Set disabledTransportCategories; @@ -159,6 +163,7 @@ public static class Filter { final Set ignoredAuditUsers, final Set ignoredAuditRequests, final Set ignoredCustomHeaders, + final Set ignoredUrlParams, final Set disabledRestCategories, final Set disabledTransportCategories ) { @@ -174,6 +179,8 @@ public static class Filter { this.ignoredAuditRequestsMatcher = WildcardMatcher.from(ignoredAuditRequests); this.ignoredCustomHeaders = ignoredCustomHeaders; this.ignoredCustomHeadersMatcher = WildcardMatcher.from(ignoredCustomHeaders); + this.ignoredUrlParams = ignoredUrlParams; + this.ignoredUrlParamsMatcher = WildcardMatcher.from(ignoredUrlParams); this.disabledRestCategories = disabledRestCategories; this.disabledTransportCategories = disabledTransportCategories; } @@ -269,6 +276,7 @@ public static Filter from(Map properties) throws JsonProcessingE ignoredAuditUsers, ignoreAuditRequests, ignoreHeaders, + new HashSet<>(), disabledRestCategories, disabledTransportCategories ); @@ -314,6 +322,7 @@ public static Filter from(Settings settings) { ignoredAuditUsers, ignoreAuditRequests, ignoreHeaders, + new HashSet<>(), disabledRestCategories, disabledTransportCategories ); @@ -422,6 +431,21 @@ WildcardMatcher getIgnoredCustomHeadersMatcher() { return ignoredCustomHeadersMatcher; } + @VisibleForTesting + WildcardMatcher getIgnoredUrlParamsMatcher() { + return ignoredUrlParamsMatcher; + } + + /** + * Check if the specified url param is excluded from the audit + * + * @param param + * @return true if header should be excluded + */ + public boolean shouldExcludeUrlParam(String param) { + return ignoredUrlParamsMatcher.test(param); + } + /** * Check if the specified header is excluded from the audit * @@ -441,6 +465,17 @@ public boolean isRequestAuditDisabled(String action) { return ignoredAuditRequestsMatcher.test(action); } + /** + * URL Params to redact for auditing + */ + public void setIgnoredUrlParams(Set ignoredUrlParams) { + if (ignoredUrlParams == null) { + return; + } + this.ignoredUrlParamsMatcher = WildcardMatcher.from(ignoredUrlParams); + this.ignoredUrlParams = ignoredUrlParams; + } + /** * Disabled categories for REST API auditing * @return set of categories @@ -470,6 +505,7 @@ public void log(Logger logger) { logger.info("Sensitive headers auditing is {}.", excludeSensitiveHeaders ? "enabled" : "disabled"); logger.info("Auditing requests from {} users is disabled.", ignoredAuditUsersMatcher); logger.info("Auditing request headers {} is disabled.", ignoredCustomHeadersMatcher); + logger.info("Auditing request url params {} is disabled.", ignoredUrlParamsMatcher); } @Override @@ -497,6 +533,8 @@ public String toString() { + ignoredAuditRequestsMatcher + ", ignoredCustomHeaders=" + ignoredCustomHeadersMatcher + + ", ignoredUrlParamsMatcher=" + + ignoredUrlParamsMatcher + '}'; } } diff --git a/src/main/java/org/opensearch/security/auditlog/impl/AbstractAuditLog.java b/src/main/java/org/opensearch/security/auditlog/impl/AbstractAuditLog.java index e5f314cd29..a5dd5290f6 100644 --- a/src/main/java/org/opensearch/security/auditlog/impl/AbstractAuditLog.java +++ b/src/main/java/org/opensearch/security/auditlog/impl/AbstractAuditLog.java @@ -19,10 +19,14 @@ import java.security.AccessController; import java.security.PrivilegedAction; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; +import java.util.SortedSet; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -62,9 +66,11 @@ import org.opensearch.security.DefaultObjectMapper; import org.opensearch.security.auditlog.AuditLog; import org.opensearch.security.auditlog.config.AuditConfig; +import org.opensearch.security.auth.AuthDomain; import org.opensearch.security.compliance.ComplianceConfig; import org.opensearch.security.dlic.rest.support.Utils; import org.opensearch.security.filter.SecurityRequest; +import org.opensearch.security.securityconf.DynamicConfigModel; import org.opensearch.security.support.Base64Helper; import org.opensearch.security.support.ConfigConstants; import org.opensearch.security.user.User; @@ -73,6 +79,7 @@ import org.opensearch.transport.TransportRequest; import com.flipkart.zjsonpatch.JsonDiff; +import org.greenrobot.eventbus.Subscribe; import static org.opensearch.core.xcontent.DeprecationHandler.THROW_UNSUPPORTED_OPERATION; @@ -88,6 +95,7 @@ public abstract class AbstractAuditLog implements AuditLog { private volatile ComplianceConfig complianceConfig; private final Environment environment; private AtomicBoolean externalConfigLogged = new AtomicBoolean(); + private final Set ignoredUrlParams = new HashSet<>(); protected abstract void enableRoutes(); @@ -120,6 +128,7 @@ protected AbstractAuditLog( } protected void onAuditConfigFilterChanged(AuditConfig.Filter auditConfigFilter) { + auditConfigFilter.setIgnoredUrlParams(ignoredUrlParams); this.auditConfigFilter = auditConfigFilter; this.auditConfigFilter.log(log); } @@ -930,4 +939,13 @@ boolean checkRestFilter(final AuditCategory category, final String effectiveUser } protected abstract void save(final AuditMessage msg); + + @Subscribe + public void onDynamicConfigModelChanged(DynamicConfigModel dcm) { + SortedSet authDomains = Collections.unmodifiableSortedSet(dcm.getRestAuthDomains()); + ignoredUrlParams.clear(); + for (AuthDomain authDomain : authDomains) { + ignoredUrlParams.addAll(authDomain.getHttpAuthenticator().getSensitiveUrlParams()); + } + } } diff --git a/src/main/java/org/opensearch/security/auditlog/impl/AuditMessage.java b/src/main/java/org/opensearch/security/auditlog/impl/AuditMessage.java index b57becc359..716e141ffd 100644 --- a/src/main/java/org/opensearch/security/auditlog/impl/AuditMessage.java +++ b/src/main/java/org/opensearch/security/auditlog/impl/AuditMessage.java @@ -350,9 +350,17 @@ public void addTaskParentId(String id) { } } - public void addRestParams(Map params) { + public void addRestParams(Map params, AuditConfig.Filter filter) { if (params != null && !params.isEmpty()) { - auditInfo.put(REST_REQUEST_PARAMS, new HashMap<>(params)); + Map redactedParams = new HashMap<>(); + for (Entry param : params.entrySet()) { + if (filter != null && filter.shouldExcludeUrlParam(param.getKey())) { + redactedParams.put(param.getKey(), "REDACTED"); + } else { + redactedParams.put(param.getKey(), param.getValue()); + } + } + auditInfo.put(REST_REQUEST_PARAMS, redactedParams); } } @@ -380,7 +388,7 @@ void addRestRequestInfo(final SecurityRequest request, final AuditConfig.Filter final String path = request.path().toString(); addPath(path); addRestHeaders(request.getHeaders(), filter.shouldExcludeSensitiveHeaders(), filter); - addRestParams(request.params()); + addRestParams(request.params(), filter); addRestMethod(request.method()); if (filter.shouldLogRequestBody()) { diff --git a/src/main/java/org/opensearch/security/auth/HTTPAuthenticator.java b/src/main/java/org/opensearch/security/auth/HTTPAuthenticator.java index c79576ef5f..927dc0e286 100644 --- a/src/main/java/org/opensearch/security/auth/HTTPAuthenticator.java +++ b/src/main/java/org/opensearch/security/auth/HTTPAuthenticator.java @@ -26,7 +26,9 @@ package org.opensearch.security.auth; +import java.util.Collections; import java.util.Optional; +import java.util.Set; import org.opensearch.OpenSearchSecurityException; import org.opensearch.common.util.concurrent.ThreadContext; @@ -92,4 +94,14 @@ public interface HTTPAuthenticator { default boolean supportsImpersonation() { return true; } + + /** + * Returns a set of URL parameters this authenticator supports that are considered sensitive + * and should be redacted in the audit logs + * + * @return The set of URL parameters considered sensitive for this authenticator. + */ + default Set getSensitiveUrlParams() { + return Collections.emptySet(); + } } diff --git a/src/main/java/org/opensearch/security/rest/SecurityInfoAction.java b/src/main/java/org/opensearch/security/rest/SecurityInfoAction.java index 9300cf72f2..469c7f81b4 100644 --- a/src/main/java/org/opensearch/security/rest/SecurityInfoAction.java +++ b/src/main/java/org/opensearch/security/rest/SecurityInfoAction.java @@ -88,6 +88,7 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + final boolean verbose = request.paramAsBoolean("verbose", false); return new RestChannelConsumer() { @Override @@ -97,8 +98,6 @@ public void accept(RestChannel channel) throws Exception { try { - final boolean verbose = request.paramAsBoolean("verbose", false); - final X509Certificate[] certs = threadContext.getTransient(ConfigConstants.OPENDISTRO_SECURITY_SSL_PEER_CERTIFICATES); final User user = threadContext.getTransient(ConfigConstants.OPENDISTRO_SECURITY_USER); final TransportAddress remoteAddress = threadContext.getTransient(ConfigConstants.OPENDISTRO_SECURITY_REMOTE_ADDRESS); diff --git a/src/test/java/org/opensearch/security/auditlog/config/AuditConfigSerializeTest.java b/src/test/java/org/opensearch/security/auditlog/config/AuditConfigSerializeTest.java index 04cea3dc05..52cb39f41e 100644 --- a/src/test/java/org/opensearch/security/auditlog/config/AuditConfigSerializeTest.java +++ b/src/test/java/org/opensearch/security/auditlog/config/AuditConfigSerializeTest.java @@ -73,6 +73,7 @@ public void testDefaultSerialize() throws IOException { .field("ignore_users", Collections.singletonList("kibanaserver")) .field("ignore_requests", Collections.emptyList()) .field("ignore_headers", Collections.emptyList()) + .field("ignore_url_params", Collections.emptyList()) .endObject() .startObject("compliance") .field("enabled", true) @@ -136,6 +137,7 @@ public void testDeserialize() throws IOException { .field("ignore_users", Collections.singletonList("test-user-1")) .field("ignore_requests", Collections.singletonList("test-request")) .field("ignore_headers", Collections.singletonList("test-headers")) + .field("ignore_url_params", Collections.singletonList("test-param")) .endObject() .startObject("compliance") .field("enabled", true) @@ -200,6 +202,7 @@ public void testSerialize() throws IOException { ImmutableSet.of("ignore-user-1", "ignore-user-2"), ImmutableSet.of("ignore-request-1"), ImmutableSet.of("test-header"), + ImmutableSet.of("test-param"), EnumSet.of(AuditCategory.FAILED_LOGIN, AuditCategory.GRANTED_PRIVILEGES), EnumSet.of(AUTHENTICATED) ); @@ -233,6 +236,7 @@ public void testSerialize() throws IOException { .field("ignore_users", ImmutableList.of("ignore-user-1", "ignore-user-2")) .field("ignore_requests", Collections.singletonList("ignore-request-1")) .field("ignore_headers", Collections.singletonList("test-header")) + .field("ignore_url_params", Collections.singletonList("test-param")) .endObject() .startObject("compliance") .field("enabled", true) @@ -276,6 +280,7 @@ public void testNullSerialize() throws IOException { .field("ignore_users", ImmutableList.of("kibanaserver")) .field("ignore_requests", Collections.emptyList()) .field("ignore_headers", Collections.emptyList()) + .field("ignore_url_params", Collections.emptyList()) .endObject() .startObject("compliance") .field("enabled", true) diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/AuditApiActionTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/AuditApiActionTest.java index b3d916e8ed..92ce7c9112 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/AuditApiActionTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/AuditApiActionTest.java @@ -682,7 +682,7 @@ private String getTestPayload() { + "\"enable_rest\":true,\"disabled_rest_categories\":[\"AUTHENTICATED\"]," + "\"enable_transport\":true,\"disabled_transport_categories\":[\"SSL_EXCEPTION\"]," + "\"resolve_bulk_requests\":true,\"log_request_body\":true,\"resolve_indices\":true,\"exclude_sensitive_headers\":true," - + "\"ignore_users\":[\"test-user-1\"],\"ignore_requests\":[\"test-request\"], \"ignore_headers\":[\"\"]}," + + "\"ignore_users\":[\"test-user-1\"],\"ignore_requests\":[\"test-request\"], \"ignore_headers\":[\"\"], \"ignore_url_params\":[]}," + "\"compliance\":{" + "\"enabled\":true," + "\"internal_config\":true,\"external_config\":true," From 77ffba4838915d67b6f049e5cdd2aa4d0e77fd90 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Fri, 23 Feb 2024 07:10:22 -0500 Subject: [PATCH 123/204] Use GHA `derek-ho/start-opensearch` for OpenSearch plugin install tests (#4063) Signed-off-by: Derek Ho --- .../action.yml | 127 ------------------ .github/workflows/plugin_install.yml | 21 +-- 2 files changed, 3 insertions(+), 145 deletions(-) delete mode 100644 .github/actions/start-opensearch-with-one-plugin/action.yml diff --git a/.github/actions/start-opensearch-with-one-plugin/action.yml b/.github/actions/start-opensearch-with-one-plugin/action.yml deleted file mode 100644 index e8e0f4eb77..0000000000 --- a/.github/actions/start-opensearch-with-one-plugin/action.yml +++ /dev/null @@ -1,127 +0,0 @@ -name: 'Launch OpenSearch with a single plugin installed' -description: 'Downloads latest build of OpenSearch, installs a plugin, executes a script and then starts OpenSearch on localhost:9200' - -inputs: - opensearch-version: - description: 'The version of OpenSearch that should be used, e.g "3.0.0"' - required: true - - plugin-name: - description: 'The name of the plugin to use, such as opensearch-security' - required: true - - setup-script-name: - description: 'The name of the setup script you want to run i.e. "setup" (do not include file extension). Leave empty to indicate one should not be run.' - required: false - - admin-password: - description: 'The admin password uses for the cluster' - required: true - -runs: - using: "composite" - steps: - - # Configure longpath names if on Windows - - name: Enable Longpaths if on Windows - if: ${{ runner.os == 'Windows' }} - run: git config --system core.longpaths true - shell: pwsh - - # Download OpenSearch - - name: Download OpenSearch for Windows - uses: peternied/download-file@v2 - if: ${{ runner.os == 'Windows' }} - with: - url: https://artifacts.opensearch.org/snapshots/core/opensearch/${{ inputs.opensearch-version }}-SNAPSHOT/opensearch-min-${{ inputs.opensearch-version }}-SNAPSHOT-windows-x64-latest.zip - - - - name: Download OpenSearch for Linux - uses: peternied/download-file@v2 - if: ${{ runner.os == 'Linux' }} - with: - url: https://artifacts.opensearch.org/snapshots/core/opensearch/${{ inputs.opensearch-version }}-SNAPSHOT/opensearch-min-${{ inputs.opensearch-version }}-SNAPSHOT-linux-x64-latest.tar.gz - - # Extract downloaded zip - - name: Extract downloaded tar - if: ${{ runner.os == 'Linux' }} - run: | - tar -xzf opensearch-*.tar.gz - rm -f opensearch-*.tar.gz - shell: bash - - - name: Extract downloaded zip - if: ${{ runner.os == 'Windows' }} - run: | - tar -xzf opensearch-min-${{ inputs.opensearch-version }}-SNAPSHOT-windows-x64-latest.zip - del opensearch-min-${{ inputs.opensearch-version }}-SNAPSHOT-windows-x64-latest.zip - shell: pwsh - - # Install the plugin - - name: Install Plugin into OpenSearch for Linux - if: ${{ runner.os == 'Linux'}} - run: | - chmod +x ./opensearch-${{ inputs.opensearch-version }}-SNAPSHOT/bin/opensearch-plugin - /bin/bash -c "yes | ./opensearch-${{ inputs.opensearch-version }}-SNAPSHOT/bin/opensearch-plugin install file:$(pwd)/opensearch-security.zip" - shell: bash - - - name: Install Plugin into OpenSearch for Windows - if: ${{ runner.os == 'Windows'}} - run: | - 'y' | .\opensearch-${{ inputs.opensearch-version }}-SNAPSHOT\bin\opensearch-plugin.bat install file:$(pwd)\${{ inputs.plugin-name }}.zip - shell: pwsh - - # Run any configuration scripts - - name: Run Setup Script for Linux - if: ${{ runner.os == 'Linux' && inputs.setup-script-name != '' }} - run: | - echo "running linux setup" - export OPENSEARCH_INITIAL_ADMIN_PASSWORD=${{ inputs.admin-password }} - chmod +x ./${{ inputs.setup-script-name }}.sh - ./${{ inputs.setup-script-name }}.sh - shell: bash - - - name: Run Setup Script for Windows - if: ${{ runner.os == 'Windows' && inputs.setup-script-name != '' }} - run: | - echo "running windows setup" - $env:OPENSEARCH_INITIAL_ADMIN_PASSWORD="${{ inputs.admin-password }}" - .\${{ inputs.setup-script-name }}.bat - shell: pwsh - - # Run OpenSearch - - name: Run OpenSearch with plugin on Linux - if: ${{ runner.os == 'Linux'}} - run: /bin/bash -c "./opensearch-${{ inputs.opensearch-version }}-SNAPSHOT/bin/opensearch &" - shell: bash - - - name: Run OpenSearch with plugin on Windows - if: ${{ runner.os == 'Windows'}} - run: start .\opensearch-${{ inputs.opensearch-version }}-SNAPSHOT\bin\opensearch.bat - shell: pwsh - - # Give the OpenSearch process some time to boot up before sending any requires, might need to increase the default time! - - name: Sleep while OpenSearch starts - uses: peternied/action-sleep@v1 - with: - seconds: 30 - - # Verify that the server is operational - - name: Check OpenSearch Running on Linux - if: ${{ runner.os != 'Windows'}} - run: curl https://localhost:9200/_cat/plugins -u 'admin:${{ inputs.admin-password }}' -k -v --fail-with-body - shell: bash - - - name: Check OpenSearch Running on Windows - if: ${{ runner.os == 'Windows'}} - run: | - $credentialBytes = [Text.Encoding]::ASCII.GetBytes("admin:${{ inputs.admin-password }}") - $encodedCredentials = [Convert]::ToBase64String($credentialBytes) - $baseCredentials = "Basic $encodedCredentials" - $Headers = @{ Authorization = $baseCredentials } - Invoke-WebRequest -SkipCertificateCheck -Uri 'https://localhost:9200/_cat/plugins' -Headers $Headers; - shell: pwsh - - - if: always() - run: cat ./opensearch-${{ inputs.opensearch-version }}-SNAPSHOT/logs/opensearch.log - shell: bash diff --git a/.github/workflows/plugin_install.yml b/.github/workflows/plugin_install.yml index b88cfb166f..fb86b915e0 100644 --- a/.github/workflows/plugin_install.yml +++ b/.github/workflows/plugin_install.yml @@ -39,27 +39,12 @@ jobs: run: mv ./build/distributions/${{ env.PLUGIN_NAME }}-*.zip ${{ env.PLUGIN_NAME }}.zip shell: bash - - name: Create Setup Script - if: ${{ runner.os == 'Linux' }} - run: | - cat > setup.sh <<'EOF' - chmod +x ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/plugins/${{ env.PLUGIN_NAME }}/tools/install_demo_configuration.sh - /bin/bash -c "yes | ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/plugins/${{ env.PLUGIN_NAME }}/tools/install_demo_configuration.sh -t" - EOF - - - name: Create Setup Script - if: ${{ runner.os == 'Windows' }} - run: | - New-Item .\setup.bat -type file - Set-Content .\setup.bat -Value "powershell.exe .\opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT\plugins\${{ env.PLUGIN_NAME }}\tools\install_demo_configuration.bat -i -c -y -t" - Get-Content .\setup.bat - - name: Run Opensearch with A Single Plugin - uses: ./.github/actions/start-opensearch-with-one-plugin + uses: derek-ho/start-opensearch@v2 with: opensearch-version: ${{ env.OPENSEARCH_VERSION }} - plugin-name: ${{ env.PLUGIN_NAME }} - setup-script-name: setup + plugins: "file:$(pwd)/${{ env.PLUGIN_NAME }}.zip" + security-enabled: true admin-password: ${{ steps.random-password.outputs.generated_name }} - name: Run sanity tests From a843b561c1a25f97b4ae48cff7e82f0492ca4d9a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Feb 2024 13:52:30 +0100 Subject: [PATCH 124/204] Bump com.netflix.nebula.ospackage from 11.8.0 to 11.8.1 (#4073) Bumps com.netflix.nebula.ospackage from 11.8.0 to 11.8.1. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.netflix.nebula.ospackage&package-manager=gradle&previous-version=11.8.0&new-version=11.8.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 76974f469b..a8dfcda965 100644 --- a/build.gradle +++ b/build.gradle @@ -64,7 +64,7 @@ plugins { id 'maven-publish' id 'com.diffplug.spotless' version '6.25.0' id 'checkstyle' - id 'com.netflix.nebula.ospackage' version "11.8.0" + id 'com.netflix.nebula.ospackage' version "11.8.1" id "org.gradle.test-retry" version "1.5.8" id 'eclipse' id "com.github.spotbugs" version "5.2.5" From 90377a5fb6bf2bf281769e28815f58a58b99274e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Feb 2024 13:53:05 +0100 Subject: [PATCH 125/204] Bump org.scala-lang:scala-library from 2.13.12 to 2.13.13 (#4072) Bumps [org.scala-lang:scala-library](https://github.com/scala/scala) from 2.13.12 to 2.13.13.
Commits
  • fcc67cd Merge pull request #10695 from SethTisue/scala-dev-755
  • 842d69d fix scala task in our sbt build so REPL has JLine features again
  • ec6078f Merge pull request #10694 from scala/tasty/test-classpath-standalone-obj
  • 5936c32 Use stripSuffix over dropRight for clarity
  • d9d1307 Align tasty file lookup with dotty, add test case
  • 232521f Merge pull request #10689 from scalacenter/tasty/switch-3.4.0-stable
  • 515883b Merge pull request #10688 from scalacenter/tasty/temp-patch-windows
  • da72476 upgrade to 3.4.0 stable
  • baadd95 Disable tasty pipelining tests on windows
  • 17c0c74 Merge pull request #10686 from som-snytt/tweak/12944-hashset-issubset
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.scala-lang:scala-library&package-manager=gradle&previous-version=2.13.12&new-version=2.13.13)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index a8dfcda965..b065fd1acf 100644 --- a/build.gradle +++ b/build.gradle @@ -476,7 +476,7 @@ configurations { resolutionStrategy { force 'commons-codec:commons-codec:1.16.1' force 'org.slf4j:slf4j-api:1.7.36' - force 'org.scala-lang:scala-library:2.13.12' + force 'org.scala-lang:scala-library:2.13.13' force "com.fasterxml.jackson:jackson-bom:${versions.jackson}" force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" force "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${versions.jackson}" @@ -702,7 +702,7 @@ dependencies { testRuntimeOnly ("org.springframework:spring-core:${spring_version}") { exclude(group:'org.springframework', module: 'spring-jcl' ) } - testRuntimeOnly 'org.scala-lang:scala-library:2.13.12' + testRuntimeOnly 'org.scala-lang:scala-library:2.13.13' testRuntimeOnly 'com.yammer.metrics:metrics-core:2.2.0' testRuntimeOnly 'com.typesafe.scala-logging:scala-logging_3:3.9.5' testRuntimeOnly('org.apache.zookeeper:zookeeper:3.9.1') { From 26b224fa1e2986431f22cf865592c4ce6fa6f49c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Feb 2024 14:06:59 +0100 Subject: [PATCH 126/204] Bump com.google.googlejavaformat:google-java-format from 1.19.2 to 1.20.0 (#4074) Bumps [com.google.googlejavaformat:google-java-format](https://github.com/google/google-java-format) from 1.19.2 to 1.20.0.
Release notes

Sourced from com.google.googlejavaformat:google-java-format's releases.

v1.20.0

This release includes GraalVM native-image binaries for google-java-format for windows, linux, and mac. These binaries offer improved startup performance.

Changes:

  • Keep type-use annotation with type when wrapping (03efe44d9affc989eee8623651fbdf1bcc7240dc)
  • Handle 'any' patterns (#1037)

Full Changelog: https://github.com/google/google-java-format/compare/v1.19.2...v1.20.0

Commits
  • 38a7b73 Release google-java-format 1.20.0
  • 92c609a Set -march=compatibility for native-image builds
  • 571c2b6 Update maven native-image configuration for google-java-format
  • 910586c Handle .exe extensions for windows native-image
  • 250fa9b Update release.yml
  • 865cdf8 Apply suggestions from code review
  • 0bc08ab Update release.yml
  • 4e9aa25 Update .github/workflows/release.yml
  • 9851a39 Generate native-image binaries for google-java-format
  • b9b41fa Add Windows native build
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.googlejavaformat:google-java-format&package-manager=gradle&previous-version=1.19.2&new-version=1.20.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index b065fd1acf..439212ab03 100644 --- a/build.gradle +++ b/build.gradle @@ -741,7 +741,7 @@ dependencies { integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5" //spotless - implementation('com.google.googlejavaformat:google-java-format:1.19.2') { + implementation('com.google.googlejavaformat:google-java-format:1.20.0') { exclude group: 'com.google.guava' } } From 142913e149ffe6c775e975867207b53208c955fb Mon Sep 17 00:00:00 2001 From: Andrey Pleskach Date: Wed, 28 Feb 2024 16:40:56 +0100 Subject: [PATCH 127/204] Add deprecate message that TLSv1 and TLSv1.1 support will be removed in the next major version (#4053) ### Description Since TLSv1.1 was deprecated in 2021 (RFC [8996](https://datatracker.ietf.org/doc/html/rfc8996)) and new deprication message was added. By default JDK 18 uses TLS 1.2 and latest stable 1.3 ### Check List - [ ] New functionality includes testing - [ ] New functionality has been documented - [x] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). Signed-off-by: Andrey Pleskach --- .../security/OpenSearchSecurityPlugin.java | 28 ++++++++++ .../security/ssl/util/SSLConfigConstants.java | 4 +- .../ssl/util/SSLConfigConstantsTest.java | 55 +++++++++++++++++++ 3 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 src/test/java/org/opensearch/security/ssl/util/SSLConfigConstantsTest.java diff --git a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java index bde984531e..688b797e85 100644 --- a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java +++ b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java @@ -310,6 +310,20 @@ public OpenSearchSecurityPlugin(final Settings settings, final Path configPath) return; } + if (settings.hasValue(SSLConfigConstants.SECURITY_SSL_HTTP_ENABLED_PROTOCOLS)) { + verifyTLSVersion( + SSLConfigConstants.SECURITY_SSL_HTTP_ENABLED_PROTOCOLS, + settings.getAsList(SSLConfigConstants.SECURITY_SSL_HTTP_ENABLED_PROTOCOLS) + ); + } + + if (settings.hasValue(SSLConfigConstants.SECURITY_SSL_TRANSPORT_ENABLED_PROTOCOLS)) { + verifyTLSVersion( + SSLConfigConstants.SECURITY_SSL_TRANSPORT_ENABLED_PROTOCOLS, + settings.getAsList(SSLConfigConstants.SECURITY_SSL_TRANSPORT_ENABLED_PROTOCOLS) + ); + } + if (SSLConfig.isSslOnlyMode()) { this.sslCertReloadEnabled = false; log.warn("OpenSearch Security plugin run in ssl only mode. No authentication or authorization is performed"); @@ -437,6 +451,20 @@ public List run() { } } + private void verifyTLSVersion(final String settings, final List configuredProtocols) { + for (final var tls : configuredProtocols) { + if (tls.equalsIgnoreCase("TLSv1") || tls.equalsIgnoreCase("TLSv1.1")) { + deprecationLogger.deprecate( + settings, + "The '{}' setting contains {} protocol version which was deprecated since 2021 (RFC 8996). " + + "Support for it will be removed in the next major release.", + settings, + tls + ); + } + } + } + private String sha256(Path p) { if (!Files.isRegularFile(p, LinkOption.NOFOLLOW_LINKS)) { diff --git a/src/main/java/org/opensearch/security/ssl/util/SSLConfigConstants.java b/src/main/java/org/opensearch/security/ssl/util/SSLConfigConstants.java index 2449146b39..a3b9348496 100644 --- a/src/main/java/org/opensearch/security/ssl/util/SSLConfigConstants.java +++ b/src/main/java/org/opensearch/security/ssl/util/SSLConfigConstants.java @@ -101,7 +101,7 @@ public final class SSLConfigConstants { private static final String[] _SECURE_SSL_PROTOCOLS = { "TLSv1.3", "TLSv1.2", "TLSv1.1" }; - public static final String[] getSecureSSLProtocols(Settings settings, boolean http) { + public static String[] getSecureSSLProtocols(Settings settings, boolean http) { List configuredProtocols = null; if (settings != null) { @@ -233,7 +233,7 @@ public static final String[] getSecureSSLProtocols(Settings settings, boolean ht }; // @formatter:on - public static final List getSecureSSLCiphers(Settings settings, boolean http) { + public static List getSecureSSLCiphers(Settings settings, boolean http) { List configuredCiphers = null; diff --git a/src/test/java/org/opensearch/security/ssl/util/SSLConfigConstantsTest.java b/src/test/java/org/opensearch/security/ssl/util/SSLConfigConstantsTest.java new file mode 100644 index 0000000000..b51efeda03 --- /dev/null +++ b/src/test/java/org/opensearch/security/ssl/util/SSLConfigConstantsTest.java @@ -0,0 +1,55 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ +package org.opensearch.security.ssl.util; + +import java.util.List; + +import org.junit.Test; + +import org.opensearch.common.settings.Settings; + +import static org.opensearch.security.ssl.util.SSLConfigConstants.SECURITY_SSL_HTTP_ENABLED_PROTOCOLS; +import static org.opensearch.security.ssl.util.SSLConfigConstants.SECURITY_SSL_TRANSPORT_ENABLED_PROTOCOLS; +import static org.junit.Assert.assertArrayEquals; + +public class SSLConfigConstantsTest { + + @Test + public void testDefaultTLSProtocols() { + final var tlsDefaultProtocols = SSLConfigConstants.getSecureSSLProtocols(Settings.EMPTY, false); + assertArrayEquals(new String[] { "TLSv1.3", "TLSv1.2", "TLSv1.1" }, tlsDefaultProtocols); + } + + @Test + public void testDefaultSSLProtocols() { + final var sslDefaultProtocols = SSLConfigConstants.getSecureSSLProtocols(Settings.EMPTY, true); + assertArrayEquals(new String[] { "TLSv1.3", "TLSv1.2", "TLSv1.1" }, sslDefaultProtocols); + } + + @Test + public void testCustomTLSProtocols() { + final var tlsDefaultProtocols = SSLConfigConstants.getSecureSSLProtocols( + Settings.builder().putList(SECURITY_SSL_TRANSPORT_ENABLED_PROTOCOLS, List.of("TLSv1", "TLSv1.1")).build(), + false + ); + assertArrayEquals(new String[] { "TLSv1", "TLSv1.1" }, tlsDefaultProtocols); + } + + @Test + public void testCustomSSLProtocols() { + final var sslDefaultProtocols = SSLConfigConstants.getSecureSSLProtocols( + Settings.builder().putList(SECURITY_SSL_HTTP_ENABLED_PROTOCOLS, List.of("TLSv1", "TLSv1.1")).build(), + true + ); + assertArrayEquals(new String[] { "TLSv1", "TLSv1.1" }, sslDefaultProtocols); + } + +} From f3b5727044d041937790cc29511dbdad6016fdd0 Mon Sep 17 00:00:00 2001 From: Cam <17013462+camerondurham@users.noreply.github.com> Date: Wed, 28 Feb 2024 08:41:28 -0700 Subject: [PATCH 128/204] Log password requirement details in demo environment (#4071) Signed-off-by: Cameron Durham --- .../SecuritySettingsConfigurer.java | 21 ++++++++---- .../SecuritySettingsConfigurerTests.java | 33 ++++++++++++++++++- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java index a68c93f03f..5b497d0f20 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java @@ -76,6 +76,7 @@ public class SecuritySettingsConfigurer { ".plugins-flow-framework-templates", ".plugins-flow-framework-state" ); + static final Integer DEFAULT_PASSWORD_MIN_LENGTH = 8; static String ADMIN_PASSWORD = ""; static String ADMIN_USERNAME = "admin"; @@ -131,7 +132,7 @@ void updateAdminPassword() { final PasswordValidator passwordValidator = PasswordValidator.of( Settings.builder() .put(SECURITY_RESTAPI_PASSWORD_VALIDATION_REGEX, "(?=.*[A-Z])(?=.*[^a-zA-Z\\\\d])(?=.*[0-9])(?=.*[a-z]).{8,}") - .put(SECURITY_RESTAPI_PASSWORD_MIN_LENGTH, 8) + .put(SECURITY_RESTAPI_PASSWORD_MIN_LENGTH, DEFAULT_PASSWORD_MIN_LENGTH) .build() ); @@ -142,11 +143,19 @@ void updateAdminPassword() { } // If script execution environment is set to demo, validate custom password, else if set to test, skip validation - if (shouldValidatePassword - && !ADMIN_PASSWORD.isEmpty() - && passwordValidator.validate(ADMIN_USERNAME, ADMIN_PASSWORD) != RequestContentValidator.ValidationError.NONE) { - System.out.println("Password " + ADMIN_PASSWORD + " is weak. Please re-try with a stronger password."); - System.exit(-1); + if (shouldValidatePassword && !ADMIN_PASSWORD.isEmpty()) { + RequestContentValidator.ValidationError response = passwordValidator.validate(ADMIN_USERNAME, ADMIN_PASSWORD); + if (!RequestContentValidator.ValidationError.NONE.equals(response)) { + System.out.println( + String.format( + "Password %s failed validation: \"%s\". Please re-try with a minimum %d character password and must contain at least one uppercase letter, one lowercase letter, one digit, and one special character that is strong. Password strength can be tested here: https://lowe.github.io/tryzxcvbn", + ADMIN_PASSWORD, + response.message(), + DEFAULT_PASSWORD_MIN_LENGTH + ) + ); + System.exit(-1); + } } // if ADMIN_PASSWORD is still an empty string, it implies no custom password was provided. We exit the setup. diff --git a/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java b/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java index 280d704fb8..50a65e7fa2 100644 --- a/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java +++ b/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java @@ -37,6 +37,9 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.opensearch.security.dlic.rest.validation.RequestContentValidator.ValidationError.INVALID_PASSWORD_INVALID_REGEX; +import static org.opensearch.security.dlic.rest.validation.RequestContentValidator.ValidationError.INVALID_PASSWORD_TOO_SHORT; +import static org.opensearch.security.tools.democonfig.SecuritySettingsConfigurer.DEFAULT_PASSWORD_MIN_LENGTH; import static org.opensearch.security.tools.democonfig.SecuritySettingsConfigurer.REST_ENABLED_ROLES; import static org.opensearch.security.tools.democonfig.SecuritySettingsConfigurer.SYSTEM_INDICES; import static org.opensearch.security.tools.democonfig.SecuritySettingsConfigurer.isKeyPresentInYMLFile; @@ -55,6 +58,9 @@ public class SecuritySettingsConfigurerTests { private final String adminPasswordKey = ConfigConstants.OPENSEARCH_INITIAL_ADMIN_PASSWORD; + private static final String PASSWORD_VALIDATION_FAILURE_MESSAGE = + "Password %s failed validation: \"%s\". Please re-try with a minimum %d character password and must contain at least one uppercase letter, one lowercase letter, one digit, and one special character that is strong. Password strength can be tested here: https://lowe.github.io/tryzxcvbn"; + private static SecuritySettingsConfigurer securitySettingsConfigurer; private static Installer installer; @@ -125,7 +131,32 @@ public void testUpdateAdminPasswordWithWeakPassword() throws NoSuchFieldExceptio System.setSecurityManager(null); } - verifyStdOutContainsString("Password weakpassword is weak. Please re-try with a stronger password."); + verifyStdOutContainsString( + String.format( + PASSWORD_VALIDATION_FAILURE_MESSAGE, + "weakpassword", + INVALID_PASSWORD_INVALID_REGEX.message(), + DEFAULT_PASSWORD_MIN_LENGTH + ) + ); + } + + @Test + public void testUpdateAdminPasswordWithShortPassword() throws NoSuchFieldException, IllegalAccessException { + + setEnv(adminPasswordKey, "short"); + try { + System.setSecurityManager(new NoExitSecurityManager()); + securitySettingsConfigurer.updateAdminPassword(); + } catch (SecurityException e) { + assertThat(e.getMessage(), equalTo("System.exit(-1) blocked to allow print statement testing.")); + } finally { + System.setSecurityManager(null); + } + + verifyStdOutContainsString( + String.format(PASSWORD_VALIDATION_FAILURE_MESSAGE, "short", INVALID_PASSWORD_TOO_SHORT.message(), DEFAULT_PASSWORD_MIN_LENGTH) + ); } @Test From b34d3d53c704a28ea510213099a8f7b75da600d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 09:20:08 -0600 Subject: [PATCH 129/204] Bump ch.qos.logback:logback-classic from 1.2.13 to 1.5.2 (#4088) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 439212ab03..06b7645f72 100644 --- a/build.gradle +++ b/build.gradle @@ -501,7 +501,7 @@ configurations { force "org.apache.httpcomponents:httpcore:4.4.16" force "com.google.errorprone:error_prone_annotations:2.25.0" force "org.checkerframework:checker-qual:3.42.0" - force "ch.qos.logback:logback-classic:1.2.13" + force "ch.qos.logback:logback-classic:1.5.2" } } From ca653c705743b94114e0ac90e6ba976d5d583502 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 09:20:26 -0600 Subject: [PATCH 130/204] Bump Wandalen/wretry.action from 1.4.4 to 1.4.5 (#4090) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fbb4d6c266..fdbdda24ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,7 +81,7 @@ jobs: working-directory: downloaded-artifacts - name: Upload Coverage with retry - uses: Wandalen/wretry.action@v1.4.4 + uses: Wandalen/wretry.action@v1.4.5 with: attempt_limit: 5 attempt_delay: 2000 From 548e21827b1897baf0e22044090e42486462199a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 09:20:33 -0600 Subject: [PATCH 131/204] Bump com.fasterxml.woodstox:woodstox-core from 6.6.0 to 6.6.1 (#4089) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 06b7645f72..8f0c1ee934 100644 --- a/build.gradle +++ b/build.gradle @@ -649,7 +649,7 @@ dependencies { runtimeOnly 'org.xerial.snappy:snappy-java:1.1.10.5' runtimeOnly 'org.codehaus.woodstox:stax2-api:4.2.2' runtimeOnly "org.glassfish.jaxb:txw2:${jaxb_version}" - runtimeOnly 'com.fasterxml.woodstox:woodstox-core:6.6.0' + runtimeOnly 'com.fasterxml.woodstox:woodstox-core:6.6.1' runtimeOnly 'org.apache.ws.xmlschema:xmlschema-core:2.3.1' runtimeOnly 'org.apache.santuario:xmlsec:2.3.4' runtimeOnly "com.github.luben:zstd-jni:${versions.zstd}" From d235d97840a8450426f3c9d272e91f6afbd6ddce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Mar 2024 16:41:17 +0100 Subject: [PATCH 132/204] Bump kafka_version from 3.6.1 to 3.7.0 (#4087) Bumps `kafka_version` from 3.6.1 to 3.7.0. Updates `org.apache.kafka:kafka-clients` from 3.6.1 to 3.7.0 Updates `org.apache.kafka:kafka_2.13` from 3.6.1 to 3.7.0 Updates `org.apache.kafka:kafka-server-common` from 3.6.1 to 3.7.0 Updates `org.apache.kafka:kafka-group-coordinator` from 3.6.1 to 3.7.0 Updates `org.apache.kafka:kafka-metadata` from 3.6.1 to 3.7.0 Updates `org.apache.kafka:kafka-storage` from 3.6.1 to 3.7.0 Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Signed-off-by: Craig Perkins Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Craig Perkins --- build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8f0c1ee934..dae173e2d4 100644 --- a/build.gradle +++ b/build.gradle @@ -25,7 +25,7 @@ buildscript { opensearch_build = version_tokens[0] + '.0' common_utils_version = System.getProperty("common_utils.version", '3.0.0.0-SNAPSHOT') - kafka_version = '3.6.1' + kafka_version = '3.7.0' apache_cxf_version = '4.0.3' open_saml_version = '4.3.0' one_login_java_saml = '2.9.0' @@ -677,6 +677,7 @@ dependencies { testImplementation 'org.apache.httpcomponents:fluent-hc:4.5.14' testImplementation "org.apache.httpcomponents.client5:httpclient5-fluent:${versions.httpclient5}" testImplementation "org.apache.kafka:kafka_2.13:${kafka_version}" + testImplementation "org.apache.kafka:kafka-server:${kafka_version}" testImplementation "org.apache.kafka:kafka-server-common:${kafka_version}" testImplementation "org.apache.kafka:kafka-server-common:${kafka_version}:test" testImplementation "org.apache.kafka:kafka-group-coordinator:${kafka_version}" From 84c86e77a5b22a13826461a01cd339a0aac2872f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 07:50:20 -0400 Subject: [PATCH 133/204] Bump Wandalen/wretry.action from 1.4.5 to 1.4.8 (#4104) Bumps [Wandalen/wretry.action](https://github.com/wandalen/wretry.action) from 1.4.5 to 1.4.8.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Wandalen/wretry.action&package-manager=github_actions&previous-version=1.4.5&new-version=1.4.8)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fdbdda24ec..c2e97f8e3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,7 +81,7 @@ jobs: working-directory: downloaded-artifacts - name: Upload Coverage with retry - uses: Wandalen/wretry.action@v1.4.5 + uses: Wandalen/wretry.action@v1.4.8 with: attempt_limit: 5 attempt_delay: 2000 From ec01a669e2745ffa4e423934f17c5eb5c63a27a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 07:50:44 -0400 Subject: [PATCH 134/204] Bump jakarta.xml.bind:jakarta.xml.bind-api from 4.0.1 to 4.0.2 (#4105) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [jakarta.xml.bind:jakarta.xml.bind-api](https://github.com/jakartaee/jaxb-api) from 4.0.1 to 4.0.2.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=jakarta.xml.bind:jakarta.xml.bind-api&package-manager=gradle&previous-version=4.0.1&new-version=4.0.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index dae173e2d4..a16ac314c7 100644 --- a/build.gradle +++ b/build.gradle @@ -611,7 +611,7 @@ dependencies { runtimeOnly 'org.cryptacular:cryptacular:1.2.6' compileOnly 'com.google.errorprone:error_prone_annotations:2.25.0' runtimeOnly 'com.sun.istack:istack-commons-runtime:4.2.0' - runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.1' + runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.2' runtimeOnly 'org.ow2.asm:asm:9.6' testImplementation 'org.apache.camel:camel-xmlsecurity:3.22.1' From 941d3bb985f4c957dc19f939880ab46ae195c695 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 07:51:09 -0400 Subject: [PATCH 135/204] Bump com.google.googlejavaformat:google-java-format from 1.20.0 to 1.21.0 (#4106) Bumps [com.google.googlejavaformat:google-java-format](https://github.com/google/google-java-format) from 1.20.0 to 1.21.0.
Release notes

Sourced from com.google.googlejavaformat:google-java-format's releases.

v1.21.0

Formatting changes:

  • Adjust indentation of text blocks (ce3cb59a8d649359a8e6e7fcc5f2f21bb79b3df1)

Bug fixes:

  • Fix a crash with comments inside string templates (e946e82801eb5bbd52bea00355ba20450bc0725c)
  • Native image -version reports HEAD-SNAPSHOT instead of the correct version (#1068)
  • Improve compatibility with older glibc versions (#1072)

Full Changelog: https://github.com/google/google-java-format/compare/v1.20.0...v1.21.0

Commits
  • ee72f3a Release google-java-format 1.21.0
  • f20d393 Bump the version number for native image builds
  • 74c510a Update the IntelliJ plugin to gfj 1.20.0.
  • cea3782 Update release.yml
  • 32d14f0 Build native on Ubuntu 20.04 instead of latest 22.04 (re. #1072).
  • 29b7f93 Remove an un-used portion of CI YAML
  • d8216e8 Migrate google-java-format to JSpecify
  • e946e82 Work around a crash on comments inside string template arguments
  • ce3cb59 Re-indent text blocks
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.googlejavaformat:google-java-format&package-manager=gradle&previous-version=1.20.0&new-version=1.21.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a16ac314c7..32e6e11b4c 100644 --- a/build.gradle +++ b/build.gradle @@ -742,7 +742,7 @@ dependencies { integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5" //spotless - implementation('com.google.googlejavaformat:google-java-format:1.20.0') { + implementation('com.google.googlejavaformat:google-java-format:1.21.0') { exclude group: 'com.google.guava' } } From d526c9f6c2a438c14db8b413148204510b9fe2e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 07:52:00 -0400 Subject: [PATCH 136/204] Bump ch.qos.logback:logback-classic from 1.5.2 to 1.5.3 (#4107) Bumps [ch.qos.logback:logback-classic](https://github.com/qos-ch/logback) from 1.5.2 to 1.5.3.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=ch.qos.logback:logback-classic&package-manager=gradle&previous-version=1.5.2&new-version=1.5.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 32e6e11b4c..5b145b9412 100644 --- a/build.gradle +++ b/build.gradle @@ -501,7 +501,7 @@ configurations { force "org.apache.httpcomponents:httpcore:4.4.16" force "com.google.errorprone:error_prone_annotations:2.25.0" force "org.checkerframework:checker-qual:3.42.0" - force "ch.qos.logback:logback-classic:1.5.2" + force "ch.qos.logback:logback-classic:1.5.3" } } From 65c5b69615609bd93da6495e0412e8584db12819 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 19 Mar 2024 08:51:53 -0400 Subject: [PATCH 137/204] Bump org.apache.zookeeper:zookeeper from 3.9.1. to 3.9.2 (#4129) ### Description Bumps org.apache.zookeeper:zookeeper from 3.9.1. to 3.9.2 Resolves WhiteSource Security check seen on byte-buddy upgrade: https://github.com/opensearch-project/security/pull/4127/checks?check_run_id=22820167922 * Category Maintenance ### Check List - [ ] New functionality includes testing - [ ] New functionality has been documented - [ ] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). Signed-off-by: Craig Perkins --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 5b145b9412..e63135a98a 100644 --- a/build.gradle +++ b/build.gradle @@ -706,7 +706,7 @@ dependencies { testRuntimeOnly 'org.scala-lang:scala-library:2.13.13' testRuntimeOnly 'com.yammer.metrics:metrics-core:2.2.0' testRuntimeOnly 'com.typesafe.scala-logging:scala-logging_3:3.9.5' - testRuntimeOnly('org.apache.zookeeper:zookeeper:3.9.1') { + testRuntimeOnly('org.apache.zookeeper:zookeeper:3.9.2') { exclude(group:'ch.qos.logback', module: 'logback-classic' ) exclude(group:'ch.qos.logback', module: 'logback-core' ) } From 582d2cddbe3c23640370698647752b39ff58eed0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Mar 2024 14:08:30 +0100 Subject: [PATCH 138/204] Bump com.google.errorprone:error_prone_annotations from 2.25.0 to 2.26.1 (#4126) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [com.google.errorprone:error_prone_annotations](https://github.com/google/error-prone) from 2.25.0 to 2.26.1.
Release notes

Sourced from com.google.errorprone:error_prone_annotations's releases.

Error Prone 2.26.1

This release contains all of the changes in 2.26.0, plus a bug fix to the module name of the annotations artifact com.google.errorprone.annotations (https://github.com/google/error-prone/commit/9d99ee76f2ca8568b69150f5df7fe845c8545d16)

Starting in 2.26.x, the 'annotations' artifact now includes a module-info.java for Java Platform Module System support, thanks to @​sgammon in #4311.

Compatibility note:

Now that the annotations artifact explicit declares a module instead of relying on Automatic-Module-Name, JDK 17 and newer perform stricter module encapsulation checks. Modularized libraries depending on Error Prone annotations 2.26.x and newer may see errors like:

error: package com.google.errorprone.annotations is not
visible
import com.google.errorprone.annotations.CheckReturnValue;
                            ^
(package com.google.errorprone.annotations is declared in module
com.google.errorprone.annotations, but module ... does not read it)

The fix is to add requires static to the module declaration of modularized libraries that depend on Error Prone annotations:

 module your.module {
...
+  requires static com.google.errorprone.annotations;
 }

Full Changelog: https://github.com/google/error-prone/compare/v2.26.0...v2.26.1

Error Prone 2.26.0

Warning: This release contains a bug, please use 2.26.1 or newer instead.

Changes:

  • The 'annotations' artifact now includes a module-info.java for Java Platform Module System support, thanks to @​sgammon in #4311.
  • Disabled checks passed to -XepPatchChecks are now ignored, instead of causing a crash. Thanks to @​oxkitsune in #4028.

New checks:

  • SystemConsoleNull: Null-checking System.console() is not a reliable way to detect if the console is connected to a terminal.
  • EnumOrdinal: Discourage uses of Enum.ordinal()

Closed issues: #2649, #3908, #4028, #4311, #4314

Full Changelog: https://github.com/google/error-prone/compare/v2.25.0...v2.26.0

Commits
  • b380572 Release Error Prone 2.26.1
  • 9d99ee7 fix: module name → com.google.errorprone.annotations
  • ea5ef6d Add the 'compile' goal for 'compile-java9'
  • 0e95364 feat: add jpms definition for annotations
  • 9da2d55 Ignore disabled checks passed to -XepPatchChecks
  • 3292632 Increase year range on Date usages.
  • ad513d5 Recommend using var for var unused = ...; and `var thrown = assertThrows(...
  • af37d35 ImpossibleNullComparison: emit empty fixes.
  • 297019c Fix some mistakes in the EnumOrdinal examples
  • f3dbb09 Move the EnumOrdinal.md doc to the right place (it got overwritten by automat...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.errorprone:error_prone_annotations&package-manager=gradle&previous-version=2.25.0&new-version=2.26.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index e63135a98a..8df9a52212 100644 --- a/build.gradle +++ b/build.gradle @@ -499,7 +499,7 @@ configurations { // For integrationTest force "org.apache.httpcomponents:httpclient:4.5.14" force "org.apache.httpcomponents:httpcore:4.4.16" - force "com.google.errorprone:error_prone_annotations:2.25.0" + force "com.google.errorprone:error_prone_annotations:2.26.1" force "org.checkerframework:checker-qual:3.42.0" force "ch.qos.logback:logback-classic:1.5.3" } @@ -609,7 +609,7 @@ dependencies { runtimeOnly 'com.eclipsesource.minimal-json:minimal-json:0.9.5' runtimeOnly 'commons-codec:commons-codec:1.16.1' runtimeOnly 'org.cryptacular:cryptacular:1.2.6' - compileOnly 'com.google.errorprone:error_prone_annotations:2.25.0' + compileOnly 'com.google.errorprone:error_prone_annotations:2.26.1' runtimeOnly 'com.sun.istack:istack-commons-runtime:4.2.0' runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.2' runtimeOnly 'org.ow2.asm:asm:9.6' From e2893639b32d4a7cf22fafb721cfc48c72591488 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Mar 2024 13:17:37 +0000 Subject: [PATCH 139/204] Bump org.eclipse.platform:org.eclipse.core.runtime from 3.30.0 to 3.31.0 (#4122) Bumps [org.eclipse.platform:org.eclipse.core.runtime](https://github.com/eclipse-platform/eclipse.platform) from 3.30.0 to 3.31.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.eclipse.platform:org.eclipse.core.runtime&package-manager=gradle&previous-version=3.30.0&new-version=3.31.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8df9a52212..27061092f5 100644 --- a/build.gradle +++ b/build.gradle @@ -494,7 +494,7 @@ configurations { force "org.apache.commons:commons-lang3:${versions.commonslang}" // for spotless transitive dependency CVE - force "org.eclipse.platform:org.eclipse.core.runtime:3.30.0" + force "org.eclipse.platform:org.eclipse.core.runtime:3.31.0" // For integrationTest force "org.apache.httpcomponents:httpclient:4.5.14" From 8ccee785d88e43d8f2090cc2d8ba728e75ed4908 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Mar 2024 13:18:07 +0000 Subject: [PATCH 140/204] Bump apache_cxf_version from 4.0.3 to 4.0.4 (#4124) Bumps `apache_cxf_version` from 4.0.3 to 4.0.4. Updates `org.apache.cxf:cxf-core` from 4.0.3 to 4.0.4 Updates `org.apache.cxf:cxf-rt-rs-json-basic` from 4.0.3 to 4.0.4 Updates `org.apache.cxf:cxf-rt-security` from 4.0.3 to 4.0.4 Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 27061092f5..6513ca87d6 100644 --- a/build.gradle +++ b/build.gradle @@ -26,7 +26,7 @@ buildscript { common_utils_version = System.getProperty("common_utils.version", '3.0.0.0-SNAPSHOT') kafka_version = '3.7.0' - apache_cxf_version = '4.0.3' + apache_cxf_version = '4.0.4' open_saml_version = '4.3.0' one_login_java_saml = '2.9.0' jjwt_version = '0.12.5' From 6d2fb70cec1bc3b6aa6ec1e5b218668d7abb10bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Mar 2024 14:18:49 +0100 Subject: [PATCH 141/204] Bump spring_version from 5.3.32 to 5.3.33 (#4125) Bumps `spring_version` from 5.3.32 to 5.3.33. Updates `org.springframework:spring-beans` from 5.3.32 to 5.3.33
Release notes

Sourced from org.springframework:spring-beans's releases.

v5.3.33

:star: New Features

  • Extract reusable method for URI validations #32442
  • Allow UriTemplate to be built with an empty template #32438
  • Refine *HttpMessageConverter#getContentLength return value null safety #32332

:lady_beetle: Bug Fixes

  • AopUtils.getMostSpecificMethod does not return original method for proxy-derived method anymore #32369
  • Better protect against concurrent error handling for async requests #32342
  • Restore Jetty 10 compatibility in JettyClientHttpResponse #32337
  • ContentCachingResponseWrapper no longer honors Content-Type and Content-Length #32322

:notebook_with_decorative_cover: Documentation

  • Build KDoc against 5.3.x Spring Framework Javadoc #32414

:hammer: Dependency Upgrades

  • Upgrade to Reactor 2020.0.42 #32422
Commits
  • df041ba Release v5.3.33
  • 297cbae Extract reusable checkSchemeAndPort method
  • 274fba4 Additional unit tests for operations on empty UriTemplate
  • 5dfec09 Allow UriTemplate to be built with an empty template
  • 5056e8c Upgrade to Reactor 2020.0.42
  • 4566e86 Polishing
  • 1b84f97 Disable external Javadoc URLs not supported on JDK 8
  • 41bc43b Build KDoc against 5.3.x Spring Framework Javadoc
  • 915d5bd Polishing
  • dc86fea Remove IOException that's not thrown from Javadoc
  • Additional commits viewable in compare view

Updates `org.springframework:spring-core` from 5.3.32 to 5.3.33
Release notes

Sourced from org.springframework:spring-core's releases.

v5.3.33

:star: New Features

  • Extract reusable method for URI validations #32442
  • Allow UriTemplate to be built with an empty template #32438
  • Refine *HttpMessageConverter#getContentLength return value null safety #32332

:lady_beetle: Bug Fixes

  • AopUtils.getMostSpecificMethod does not return original method for proxy-derived method anymore #32369
  • Better protect against concurrent error handling for async requests #32342
  • Restore Jetty 10 compatibility in JettyClientHttpResponse #32337
  • ContentCachingResponseWrapper no longer honors Content-Type and Content-Length #32322

:notebook_with_decorative_cover: Documentation

  • Build KDoc against 5.3.x Spring Framework Javadoc #32414

:hammer: Dependency Upgrades

  • Upgrade to Reactor 2020.0.42 #32422
Commits
  • df041ba Release v5.3.33
  • 297cbae Extract reusable checkSchemeAndPort method
  • 274fba4 Additional unit tests for operations on empty UriTemplate
  • 5dfec09 Allow UriTemplate to be built with an empty template
  • 5056e8c Upgrade to Reactor 2020.0.42
  • 4566e86 Polishing
  • 1b84f97 Disable external Javadoc URLs not supported on JDK 8
  • 41bc43b Build KDoc against 5.3.x Spring Framework Javadoc
  • 915d5bd Polishing
  • dc86fea Remove IOException that's not thrown from Javadoc
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 6513ca87d6..faee33aa5c 100644 --- a/build.gradle +++ b/build.gradle @@ -32,7 +32,7 @@ buildscript { jjwt_version = '0.12.5' guava_version = '32.1.3-jre' jaxb_version = '2.3.9' - spring_version = '5.3.32' + spring_version = '5.3.33' if (buildVersionQualifier) { opensearch_build += "-${buildVersionQualifier}" From 9806699cc727489c8d46637986f7860e81ed7066 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Mar 2024 14:19:02 +0100 Subject: [PATCH 142/204] Bump org.awaitility:awaitility from 4.2.0 to 4.2.1 (#4123) Bumps [org.awaitility:awaitility](https://github.com/awaitility/awaitility) from 4.2.0 to 4.2.1.
Changelog

Sourced from org.awaitility:awaitility's changelog.

Changelog 4.2.1 (2024-03-15)

  • Upgraded Kotlin to 1.9.22

  • Added extension properties forever, then, and, given to the Kotlin extension. This allows you to do e.g.:

    await.forever until { .. }

  • Added shortcut for enabling logging. Before you had to do e.g.

    await() .with() .conditionEvaluationListener(new ConditionEvaluationLogger(log::info)) .pollInterval(ONE_HUNDRED_MILLISECONDS) .until(logs::size, is(4));

    You can now instead use the "logging" shortcut:

    await() .with() .logging(log::info) .pollInterval(ONE_HUNDRED_MILLISECONDS) .until(logs::size, is(4));

    or simply ".logging()" for "System.out".

    This shortcut has also been added globally:

    Awaitility.setLogging(log::info);

    or

    Awaitility.setDefaultLogging();

  • Improved lambda detection for Java 17 and Java 21

  • Upgraded Groovy to 4.0.19

Commits
  • ff13b72 [maven-release-plugin] prepare release awaitility-4.2.1
  • f80c299 [ci skip] Preparing changelog for release
  • 4be5236 [ci skip] Fixed typo in changelog
  • e15b975 Fixed failing tests
  • 7f7656e Adding 17 and 21 to tests
  • 32eafb6 Improved lambda detection and upgraded groovy/scala
  • 8012936 Trying to fix failing test
  • b01855d Revert "Added java 21 tests"
  • 0e7dff0 Revert "Revert "Use Duration factories in Durations.java (#268)""
  • 97076a9 Added java 21 tests
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.awaitility:awaitility&package-manager=gradle&previous-version=4.2.0&new-version=4.2.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index faee33aa5c..7fcd272c32 100644 --- a/build.gradle +++ b/build.gradle @@ -688,7 +688,7 @@ dependencies { testImplementation "org.springframework:spring-beans:${spring_version}" testImplementation 'org.junit.jupiter:junit-jupiter:5.10.2' testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.2' - testImplementation('org.awaitility:awaitility:4.2.0') { + testImplementation('org.awaitility:awaitility:4.2.1') { exclude(group: 'org.hamcrest', module: 'hamcrest') } // Only osx-x86_64, osx-aarch_64, linux-x86_64, linux-aarch_64, windows-x86_64 are available @@ -731,7 +731,7 @@ dependencies { integrationTestImplementation 'org.hamcrest:hamcrest:2.2' integrationTestImplementation "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}" integrationTestImplementation "org.bouncycastle:bcutil-jdk18on:${versions.bouncycastle}" - integrationTestImplementation('org.awaitility:awaitility:4.2.0') { + integrationTestImplementation('org.awaitility:awaitility:4.2.1') { exclude(group: 'org.hamcrest', module: 'hamcrest') } integrationTestImplementation 'com.unboundid:unboundid-ldapsdk:4.0.14' From 343fc7764afa84c6dab5022730edb43443bef64f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Mar 2024 13:39:33 +0000 Subject: [PATCH 143/204] Bump derek-ho/start-opensearch from 2 to 3 (#4120) Bumps [derek-ho/start-opensearch](https://github.com/derek-ho/start-opensearch) from 2 to 3.
Release notes

Sourced from derek-ho/start-opensearch's releases.

Release v3

Allow port flexibility

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=derek-ho/start-opensearch&package-manager=github_actions&previous-version=2&new-version=3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/plugin_install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/plugin_install.yml b/.github/workflows/plugin_install.yml index fb86b915e0..6fa8c74beb 100644 --- a/.github/workflows/plugin_install.yml +++ b/.github/workflows/plugin_install.yml @@ -40,7 +40,7 @@ jobs: shell: bash - name: Run Opensearch with A Single Plugin - uses: derek-ho/start-opensearch@v2 + uses: derek-ho/start-opensearch@v3 with: opensearch-version: ${{ env.OPENSEARCH_VERSION }} plugins: "file:$(pwd)/${{ env.PLUGIN_NAME }}.zip" From 32ba887290161eebbdd0788ffc02c0fb9d1b38c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Mar 2024 09:41:58 -0400 Subject: [PATCH 144/204] Bump Wandalen/wretry.action from 1.4.8 to 1.4.10 (#4121) Bumps [Wandalen/wretry.action](https://github.com/wandalen/wretry.action) from 1.4.8 to 1.4.10.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Wandalen/wretry.action&package-manager=github_actions&previous-version=1.4.8&new-version=1.4.10)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c2e97f8e3a..0595106ce7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,7 +81,7 @@ jobs: working-directory: downloaded-artifacts - name: Upload Coverage with retry - uses: Wandalen/wretry.action@v1.4.8 + uses: Wandalen/wretry.action@v1.4.10 with: attempt_limit: 5 attempt_delay: 2000 From 25e2e51edecdd1a436223a2bae5fab8e975d2069 Mon Sep 17 00:00:00 2001 From: David Osorno <48450162+davidosorno@users.noreply.github.com> Date: Tue, 19 Mar 2024 08:57:45 -0700 Subject: [PATCH 145/204] Dynamic sign in options (#3869) ### Description **New Feature** Allow admins to define the sign-in options that will be displayed on OpenSearch Dashboard login page. There are couple of sign-in options defined in [Security documentation](https://opensearch.org/docs/latest/security/configuration/multi-auth/), and theses options must be available in security _config.yml_ file to be able to change them dynamically in Security Dashboard. Furthermore, if `anonymous_auth_enabled` is true it will be available in Security Dashboard sign-in options to allow admins enable or disable it. *Old Behavior* Admins have to update _opensearch_dashboards.yml_ adding or removing sign-in options, and then restart Dashboards to be able to log in using other sign-in option. *New Behavior* Admins can change sign-in options dynamically without having to restart the Dashboards, and the changes are applied immediately. Users just need to logout in order to see the sign-in options available. ### Issues Resolved - Related https://github.com/opensearch-project/security-dashboards-plugin/issues/1573 ### Testing Unit Testing, Integration Testing, and Manual Testing. ### Check List - [x] New functionality includes testing - [ ] New functionality has been documented - [x] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: David Osorno --- .../security/api/DashboardsInfoTest.java | 12 ++++ .../rest/api/MultiTenancyConfigApiAction.java | 31 +++++++++- .../privileges/PrivilegesEvaluator.java | 5 ++ .../security/rest/DashboardsInfoAction.java | 1 + .../securityconf/DynamicConfigModel.java | 3 + .../securityconf/DynamicConfigModelV6.java | 6 ++ .../securityconf/DynamicConfigModelV7.java | 6 ++ .../impl/DashboardSignInOption.java | 29 +++++++++ .../securityconf/impl/v6/ConfigV6.java | 7 +++ .../securityconf/impl/v7/ConfigV7.java | 8 +++ .../rest/api/MultiTenancyConfigApiTest.java | 61 +++++++++++++++++++ .../securityconf/impl/v6/ConfigV6Test.java | 8 +++ .../securityconf/impl/v7/ConfigV7Test.java | 8 +++ src/test/resources/restapi/config.yml | 12 ++++ .../restapi/securityconfig_nondefault.json | 3 +- 15 files changed, 198 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/opensearch/security/securityconf/impl/DashboardSignInOption.java diff --git a/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoTest.java b/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoTest.java index 8bfcd3b8a8..15634e8890 100644 --- a/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoTest.java +++ b/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoTest.java @@ -17,6 +17,7 @@ import org.junit.Test; import org.junit.runner.RunWith; +import org.opensearch.security.securityconf.impl.DashboardSignInOption; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; import org.opensearch.test.framework.cluster.ClusterManager; @@ -25,6 +26,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.opensearch.security.rest.DashboardsInfoAction.DEFAULT_PASSWORD_MESSAGE; import static org.opensearch.security.rest.DashboardsInfoAction.DEFAULT_PASSWORD_REGEX; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; @@ -53,4 +55,14 @@ public void testDashboardsInfoValidationMessage() throws Exception { assertThat(response.getTextFromJsonBody("/password_validation_regex"), equalTo(DEFAULT_PASSWORD_REGEX)); } } + + @Test + public void testDashboardsInfoContainsSignInOptions() throws Exception { + + try (TestRestClient client = cluster.getRestClient(DASHBOARDS_USER)) { + TestRestClient.HttpResponse response = client.get("_plugins/_security/dashboardsinfo"); + assertThat(response.getStatusCode(), equalTo(HttpStatus.SC_OK)); + assertThat(response.getTextArrayFromJsonBody("/sign_in_options").contains(DashboardSignInOption.BASIC.toString()), is(true)); + } + } } diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/MultiTenancyConfigApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/MultiTenancyConfigApiAction.java index d56025aec1..2be5778956 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/MultiTenancyConfigApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/MultiTenancyConfigApiAction.java @@ -18,6 +18,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.IntStream; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -33,8 +34,10 @@ import org.opensearch.security.dlic.rest.validation.RequestContentValidator; import org.opensearch.security.dlic.rest.validation.RequestContentValidator.DataType; import org.opensearch.security.securityconf.impl.CType; +import org.opensearch.security.securityconf.impl.DashboardSignInOption; import org.opensearch.security.securityconf.impl.SecurityDynamicConfiguration; import org.opensearch.security.securityconf.impl.v7.ConfigV7; +import org.opensearch.security.securityconf.impl.v7.ConfigV7.Authc; import org.opensearch.security.support.ConfigConstants; import org.opensearch.threadpool.ThreadPool; @@ -49,6 +52,7 @@ public class MultiTenancyConfigApiAction extends AbstractApiAction { public static final String DEFAULT_TENANT_JSON_PROPERTY = "default_tenant"; public static final String PRIVATE_TENANT_ENABLED_JSON_PROPERTY = "private_tenant_enabled"; public static final String MULTITENANCY_ENABLED_JSON_PROPERTY = "multitenancy_enabled"; + public static final String SIGN_IN_OPTIONS = "sign_in_options"; private static final List ROUTES = addRoutesPrefix( ImmutableList.of(new Route(GET, "/tenancy/config"), new Route(PUT, "/tenancy/config")) @@ -119,7 +123,9 @@ public Map allowedKeys() { PRIVATE_TENANT_ENABLED_JSON_PROPERTY, DataType.BOOLEAN, MULTITENANCY_ENABLED_JSON_PROPERTY, - DataType.BOOLEAN + DataType.BOOLEAN, + SIGN_IN_OPTIONS, + DataType.ARRAY ); } }); @@ -132,6 +138,7 @@ private ToXContent multitenancyContent(final ConfigV7 config) { .field(DEFAULT_TENANT_JSON_PROPERTY, config.dynamic.kibana.default_tenant) .field(PRIVATE_TENANT_ENABLED_JSON_PROPERTY, config.dynamic.kibana.private_tenant_enabled) .field(MULTITENANCY_ENABLED_JSON_PROPERTY, config.dynamic.kibana.multitenancy_enabled) + .field(SIGN_IN_OPTIONS, config.dynamic.kibana.sign_in_options) .endObject(); } @@ -177,6 +184,12 @@ private void updateAndValidatesValues(final ConfigV7 config, final JsonNode json if (Objects.nonNull(jsonContent.findValue(MULTITENANCY_ENABLED_JSON_PROPERTY))) { config.dynamic.kibana.multitenancy_enabled = jsonContent.findValue(MULTITENANCY_ENABLED_JSON_PROPERTY).asBoolean(); } + if (jsonContent.hasNonNull(SIGN_IN_OPTIONS) && jsonContent.findValue(SIGN_IN_OPTIONS).isEmpty() == false) { + JsonNode newOptions = jsonContent.findValue(SIGN_IN_OPTIONS); + List options = getNewSignInOptions(newOptions, config.dynamic.authc); + config.dynamic.kibana.sign_in_options = options; + } + final String defaultTenant = Optional.ofNullable(config.dynamic.kibana.default_tenant).map(String::toLowerCase).orElse(""); if (!config.dynamic.kibana.private_tenant_enabled && ConfigConstants.TENANCY_PRIVATE_TENANT_NAME.equals(defaultTenant)) { @@ -202,4 +215,20 @@ private void updateAndValidatesValues(final ConfigV7 config, final JsonNode json } } + private List getNewSignInOptions(JsonNode newOptions, Authc authc) { + + Set domains = authc.getDomains().keySet(); + + return IntStream.range(0, newOptions.size()).mapToObj(newOptions::get).map(JsonNode::asText).filter(option -> { + // Checking if the new sign-in options are set in backend. + if (option.equals(DashboardSignInOption.ANONYMOUS.toString()) + || domains.stream().anyMatch(domain -> domain.contains(option.toLowerCase()))) { + return true; + } else { + throw new IllegalArgumentException( + "Validation failure: " + option.toUpperCase() + " authentication provider is not available for this cluster." + ); + } + }).map(DashboardSignInOption::valueOf).collect(Collectors.toList()); + } } diff --git a/src/main/java/org/opensearch/security/privileges/PrivilegesEvaluator.java b/src/main/java/org/opensearch/security/privileges/PrivilegesEvaluator.java index 0f8d132e3e..06e41a2aaa 100644 --- a/src/main/java/org/opensearch/security/privileges/PrivilegesEvaluator.java +++ b/src/main/java/org/opensearch/security/privileges/PrivilegesEvaluator.java @@ -90,6 +90,7 @@ import org.opensearch.security.securityconf.ConfigModel; import org.opensearch.security.securityconf.DynamicConfigModel; import org.opensearch.security.securityconf.SecurityRoles; +import org.opensearch.security.securityconf.impl.DashboardSignInOption; import org.opensearch.security.support.ConfigConstants; import org.opensearch.security.support.WildcardMatcher; import org.opensearch.security.user.User; @@ -620,6 +621,10 @@ public String dashboardsOpenSearchRole() { return dcm.getDashboardsOpenSearchRole(); } + public List getSignInOptions() { + return dcm.getSignInOptions(); + } + private Set evaluateAdditionalIndexPermissions(final ActionRequest request, final String originalAction) { // --- check inner bulk requests final Set additionalPermissionsRequired = new HashSet<>(); diff --git a/src/main/java/org/opensearch/security/rest/DashboardsInfoAction.java b/src/main/java/org/opensearch/security/rest/DashboardsInfoAction.java index 2b286d0c3d..070648ed92 100644 --- a/src/main/java/org/opensearch/security/rest/DashboardsInfoAction.java +++ b/src/main/java/org/opensearch/security/rest/DashboardsInfoAction.java @@ -108,6 +108,7 @@ public void accept(RestChannel channel) throws Exception { builder.field("multitenancy_enabled", evaluator.multitenancyEnabled()); builder.field("private_tenant_enabled", evaluator.privateTenantEnabled()); builder.field("default_tenant", evaluator.dashboardsDefaultTenant()); + builder.field("sign_in_options", evaluator.getSignInOptions()); builder.field( "password_validation_error_message", client.settings().get(ConfigConstants.SECURITY_RESTAPI_PASSWORD_VALIDATION_ERROR_MESSAGE, DEFAULT_PASSWORD_MESSAGE) diff --git a/src/main/java/org/opensearch/security/securityconf/DynamicConfigModel.java b/src/main/java/org/opensearch/security/securityconf/DynamicConfigModel.java index e3d10878da..064f555a75 100644 --- a/src/main/java/org/opensearch/security/securityconf/DynamicConfigModel.java +++ b/src/main/java/org/opensearch/security/securityconf/DynamicConfigModel.java @@ -52,6 +52,7 @@ import org.opensearch.security.http.HTTPClientCertAuthenticator; import org.opensearch.security.http.HTTPProxyAuthenticator; import org.opensearch.security.http.proxy.HTTPExtendedProxyAuthenticator; +import org.opensearch.security.securityconf.impl.DashboardSignInOption; public abstract class DynamicConfigModel { @@ -105,6 +106,8 @@ public abstract class DynamicConfigModel { public abstract Multimap> getAuthBackendClientBlockRegistries(); + public abstract List getSignInOptions(); + public abstract Settings getDynamicOnBehalfOfSettings(); protected final Map authImplMap = new HashMap<>(); diff --git a/src/main/java/org/opensearch/security/securityconf/DynamicConfigModelV6.java b/src/main/java/org/opensearch/security/securityconf/DynamicConfigModelV6.java index b652893bdd..c7edaf938c 100644 --- a/src/main/java/org/opensearch/security/securityconf/DynamicConfigModelV6.java +++ b/src/main/java/org/opensearch/security/securityconf/DynamicConfigModelV6.java @@ -54,6 +54,7 @@ import org.opensearch.security.auth.HTTPAuthenticator; import org.opensearch.security.auth.blocking.ClientBlockRegistry; import org.opensearch.security.auth.internal.InternalAuthenticationBackend; +import org.opensearch.security.securityconf.impl.DashboardSignInOption; import org.opensearch.security.securityconf.impl.v6.ConfigV6; import org.opensearch.security.securityconf.impl.v6.ConfigV6.Authc; import org.opensearch.security.securityconf.impl.v6.ConfigV6.AuthcDomain; @@ -205,6 +206,11 @@ public Multimap> getAuthBackendClientBlockRe return Multimaps.unmodifiableMultimap(authBackendClientBlockRegistries); } + @Override + public List getSignInOptions() { + return config.dynamic.kibana.sign_in_options; + } + @Override public Settings getDynamicOnBehalfOfSettings() { return Settings.EMPTY; diff --git a/src/main/java/org/opensearch/security/securityconf/DynamicConfigModelV7.java b/src/main/java/org/opensearch/security/securityconf/DynamicConfigModelV7.java index 91bb59db64..ca237bc054 100644 --- a/src/main/java/org/opensearch/security/securityconf/DynamicConfigModelV7.java +++ b/src/main/java/org/opensearch/security/securityconf/DynamicConfigModelV7.java @@ -60,6 +60,7 @@ import org.opensearch.security.auth.internal.NoOpAuthenticationBackend; import org.opensearch.security.configuration.ClusterInfoHolder; import org.opensearch.security.http.OnBehalfOfAuthenticator; +import org.opensearch.security.securityconf.impl.DashboardSignInOption; import org.opensearch.security.securityconf.impl.v7.ConfigV7; import org.opensearch.security.securityconf.impl.v7.ConfigV7.Authc; import org.opensearch.security.securityconf.impl.v7.ConfigV7.AuthcDomain; @@ -221,6 +222,11 @@ public Multimap> getAuthBackendClientBlockRe return Multimaps.unmodifiableMultimap(authBackendClientBlockRegistries); } + @Override + public List getSignInOptions() { + return config.dynamic.kibana.sign_in_options; + } + @Override public Settings getDynamicOnBehalfOfSettings() { return Settings.builder() diff --git a/src/main/java/org/opensearch/security/securityconf/impl/DashboardSignInOption.java b/src/main/java/org/opensearch/security/securityconf/impl/DashboardSignInOption.java new file mode 100644 index 0000000000..3d9fa20e97 --- /dev/null +++ b/src/main/java/org/opensearch/security/securityconf/impl/DashboardSignInOption.java @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.securityconf.impl; + +public enum DashboardSignInOption { + BASIC("basic"), + SAML("saml"), + OPENID("openid"), + ANONYMOUS("anonymous"); + + private String option; + + DashboardSignInOption(String option) { + this.option = option; + } + + public String getOption() { + return option; + } +} diff --git a/src/main/java/org/opensearch/security/securityconf/impl/v6/ConfigV6.java b/src/main/java/org/opensearch/security/securityconf/impl/v6/ConfigV6.java index c5b954675b..78758e0603 100644 --- a/src/main/java/org/opensearch/security/securityconf/impl/v6/ConfigV6.java +++ b/src/main/java/org/opensearch/security/securityconf/impl/v6/ConfigV6.java @@ -27,8 +27,10 @@ package org.opensearch.security.securityconf.impl.v6; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.regex.Pattern; @@ -43,6 +45,7 @@ import org.opensearch.security.DefaultObjectMapper; import org.opensearch.security.auth.internal.InternalAuthenticationBackend; +import org.opensearch.security.securityconf.impl.DashboardSignInOption; import org.opensearch.security.setting.DeprecatedSettings; public class ConfigV6 { @@ -100,6 +103,8 @@ public static class Kibana { public String opendistro_role = null; public String index = ".kibana"; public boolean do_not_fail_on_forbidden; + @JsonInclude(JsonInclude.Include.NON_NULL) + public List sign_in_options = Arrays.asList(DashboardSignInOption.BASIC); @Override public String toString() { @@ -113,6 +118,8 @@ public String toString() { + index + ", do_not_fail_on_forbidden=" + do_not_fail_on_forbidden + + ", sign_in_options=" + + sign_in_options + "]"; } diff --git a/src/main/java/org/opensearch/security/securityconf/impl/v7/ConfigV7.java b/src/main/java/org/opensearch/security/securityconf/impl/v7/ConfigV7.java index 4028719379..dc9be395b1 100644 --- a/src/main/java/org/opensearch/security/securityconf/impl/v7/ConfigV7.java +++ b/src/main/java/org/opensearch/security/securityconf/impl/v7/ConfigV7.java @@ -27,8 +27,10 @@ package org.opensearch.security.securityconf.impl.v7; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -44,6 +46,7 @@ import org.opensearch.security.DefaultObjectMapper; import org.opensearch.security.auth.internal.InternalAuthenticationBackend; +import org.opensearch.security.securityconf.impl.DashboardSignInOption; import org.opensearch.security.securityconf.impl.v6.ConfigV6; import org.opensearch.security.setting.DeprecatedSettings; @@ -76,6 +79,7 @@ public ConfigV7(ConfigV6 c6) { dynamic.kibana.private_tenant_enabled = true; dynamic.kibana.default_tenant = ""; dynamic.kibana.server_username = c6.dynamic.kibana.server_username; + dynamic.kibana.sign_in_options = c6.dynamic.kibana.sign_in_options; dynamic.http = new Http(); @@ -168,6 +172,8 @@ public static class Kibana { public String server_username = "kibanaserver"; public String opendistro_role = null; public String index = ".kibana"; + @JsonInclude(JsonInclude.Include.NON_NULL) + public List sign_in_options = Arrays.asList(DashboardSignInOption.BASIC); @Override public String toString() { @@ -183,6 +189,8 @@ public String toString() { + opendistro_role + ", index=" + index + + ", sign_in_options=" + + sign_in_options + "]"; } diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/MultiTenancyConfigApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/MultiTenancyConfigApiTest.java index 8438338869..752335b802 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/MultiTenancyConfigApiTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/MultiTenancyConfigApiTest.java @@ -15,10 +15,13 @@ import org.apache.http.HttpStatus; import org.junit.Test; +import org.opensearch.security.securityconf.impl.DashboardSignInOption; import org.opensearch.security.support.ConfigConstants; import org.opensearch.security.test.helper.rest.RestHelper.HttpResponse; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.not; import static org.hamcrest.core.IsEqual.equalTo; import static org.hamcrest.core.StringContains.containsString; @@ -54,9 +57,43 @@ private void verifyTenantUpdate(final Header... header) throws Exception { setPrivateTenantAsDefaultResponse.getStatusCode(), equalTo(HttpStatus.SC_OK) ); + assertThat(getDashboardsinfoResponse.findArrayInJson("sign_in_options"), hasItem(DashboardSignInOption.BASIC.toString())); + assertThat(getDashboardsinfoResponse.findArrayInJson("sign_in_options"), not(hasItem(DashboardSignInOption.SAML.toString()))); + assertThat(getDashboardsinfoResponse.findArrayInJson("sign_in_options"), not(hasItem(DashboardSignInOption.OPENID.toString()))); + + final HttpResponse updateDashboardSignInOptions = rh.executePutRequest( + "/_plugins/_security/api/tenancy/config", + "{\"sign_in_options\": [\"BASIC\", \"OPENID\"]}", + header + ); + assertThat(updateDashboardSignInOptions.getBody(), updateDashboardSignInOptions.getStatusCode(), equalTo(HttpStatus.SC_OK)); + getDashboardsinfoResponse = rh.executeGetRequest("/_plugins/_security/dashboardsinfo", ADMIN_FULL_ACCESS_USER); assertThat(getDashboardsinfoResponse.getStatusCode(), equalTo(HttpStatus.SC_OK)); assertThat(getDashboardsinfoResponse.findValueInJson("default_tenant"), equalTo("Private")); + + assertThat(getDashboardsinfoResponse.findArrayInJson("sign_in_options"), hasItem((DashboardSignInOption.BASIC.toString()))); + assertThat(getDashboardsinfoResponse.findArrayInJson("sign_in_options"), hasItem((DashboardSignInOption.OPENID.toString()))); + + final HttpResponse updateUnavailableSignInOption = rh.executePutRequest( + "/_plugins/_security/api/tenancy/config", + "{\"sign_in_options\": [\"BASIC\", \"SAML\"]}", + header + ); + assertThat(updateUnavailableSignInOption.getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST)); + assertThat( + updateUnavailableSignInOption.findValueInJson("error.reason"), + containsString("Validation failure: SAML authentication provider is not available for this cluster.") + ); + + // Ensuring the sign in options array has not been modified due to the Bad Request response. + getDashboardsinfoResponse = rh.executeGetRequest("/_plugins/_security/dashboardsinfo", ADMIN_FULL_ACCESS_USER); + assertThat(getDashboardsinfoResponse.getStatusCode(), equalTo(HttpStatus.SC_OK)); + + assertThat(getDashboardsinfoResponse.findArrayInJson("sign_in_options").size(), equalTo(2)); + assertThat(getDashboardsinfoResponse.findArrayInJson("sign_in_options"), hasItem(DashboardSignInOption.BASIC.toString())); + assertThat(getDashboardsinfoResponse.findArrayInJson("sign_in_options"), hasItem(DashboardSignInOption.OPENID.toString())); + assertThat(getDashboardsinfoResponse.findArrayInJson("sign_in_options"), not(hasItem(DashboardSignInOption.SAML.toString()))); } @Test @@ -148,6 +185,30 @@ private void verifyTenantUpdateFailed(final Header... header) throws Exception { setRandomStringAsDefaultTenant.findValueInJson("error.reason"), containsString("Default tenant should be selected from one of the available tenants.") ); + + final HttpResponse signInOptionsNonArrayValue = rh.executePutRequest( + "/_plugins/_security/api/tenancy/config", + "{\"sign_in_options\": \"BASIC\"}", + header + ); + assertThat(signInOptionsNonArrayValue.getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST)); + assertThat( + signInOptionsNonArrayValue.getBody(), + signInOptionsNonArrayValue.findValueInJson("reason"), + containsString("Wrong datatype") + ); + + final HttpResponse invalidSignInOption = rh.executePutRequest( + "/_plugins/_security/api/tenancy/config", + "{\"sign_in_options\": [\"INVALID_OPTION\"]}", + header + ); + assertThat(invalidSignInOption.getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST)); + assertThat( + invalidSignInOption.getBody(), + invalidSignInOption.findValueInJson("error.reason"), + containsString("authentication provider is not available for this cluster") + ); } @Test diff --git a/src/test/java/org/opensearch/security/securityconf/impl/v6/ConfigV6Test.java b/src/test/java/org/opensearch/security/securityconf/impl/v6/ConfigV6Test.java index 2983fc6064..a780b0066f 100644 --- a/src/test/java/org/opensearch/security/securityconf/impl/v6/ConfigV6Test.java +++ b/src/test/java/org/opensearch/security/securityconf/impl/v6/ConfigV6Test.java @@ -20,6 +20,10 @@ import org.opensearch.security.DefaultObjectMapper; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; + @RunWith(Parameterized.class) public class ConfigV6Test { private final boolean omitDefaults; @@ -31,6 +35,9 @@ public static Iterable omitDefaults() { public void assertEquals(ConfigV6.Kibana expected, JsonNode node) { Assert.assertEquals(expected.multitenancy_enabled, node.get("multitenancy_enabled").asBoolean()); + assertThat(node.get("sign_in_options").isArray(), is(true)); + assertThat(node.get("sign_in_options").toString(), containsString(expected.sign_in_options.get(0).toString())); + if (expected.server_username == null) { Assert.assertNull(node.get("server_username")); } else { @@ -57,6 +64,7 @@ public void assertEquals(ConfigV6.Kibana expected, JsonNode node) { private void assertEquals(ConfigV6.Kibana expected, ConfigV6.Kibana actual) { Assert.assertEquals(expected.multitenancy_enabled, actual.multitenancy_enabled); + assertThat(expected.sign_in_options, is(actual.sign_in_options)); if (expected.server_username == null) { // null is restored to default instead of null Assert.assertEquals(new ConfigV6.Kibana().server_username, actual.server_username); diff --git a/src/test/java/org/opensearch/security/securityconf/impl/v7/ConfigV7Test.java b/src/test/java/org/opensearch/security/securityconf/impl/v7/ConfigV7Test.java index 542ce878bd..246247c6d9 100644 --- a/src/test/java/org/opensearch/security/securityconf/impl/v7/ConfigV7Test.java +++ b/src/test/java/org/opensearch/security/securityconf/impl/v7/ConfigV7Test.java @@ -20,6 +20,10 @@ import org.opensearch.security.DefaultObjectMapper; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; + @RunWith(Parameterized.class) public class ConfigV7Test { private final boolean omitDefaults; @@ -31,6 +35,9 @@ public static Iterable omitDefaults() { public void assertEquals(ConfigV7.Kibana expected, JsonNode node) { Assert.assertEquals(expected.multitenancy_enabled, node.get("multitenancy_enabled").asBoolean()); + assertThat(node.get("sign_in_options").isArray(), is(true)); + assertThat(node.get("sign_in_options").toString(), containsString(expected.sign_in_options.get(0).toString())); + if (expected.server_username == null) { Assert.assertNull(node.get("server_username")); } else { @@ -51,6 +58,7 @@ public void assertEquals(ConfigV7.Kibana expected, JsonNode node) { private void assertEquals(ConfigV7.Kibana expected, ConfigV7.Kibana actual) { Assert.assertEquals(expected.multitenancy_enabled, actual.multitenancy_enabled); + assertThat(expected.sign_in_options, is(actual.sign_in_options)); if (expected.server_username == null) { // null is restored to default instead of null Assert.assertEquals(new ConfigV7.Kibana().server_username, actual.server_username); diff --git a/src/test/resources/restapi/config.yml b/src/test/resources/restapi/config.yml index 2ed865657a..7a7d3d0e98 100644 --- a/src/test/resources/restapi/config.yml +++ b/src/test/resources/restapi/config.yml @@ -21,6 +21,18 @@ config: internalProxies: "192\\.168\\.0\\.10|192\\.168\\.0\\.11" remoteIpHeader: "x-forwarded-for" authc: + openid_auth_domain: + http_enabled: true + transport_enabled: true + order: 4 + http_authenticator: + type: openid + challenge: false + config: {} + authentication_backend: + type: "noop" + config: {} + description: "Migrated from v6" authentication_domain_kerb: http_enabled: false transport_enabled: false diff --git a/src/test/resources/restapi/securityconfig_nondefault.json b/src/test/resources/restapi/securityconfig_nondefault.json index 2482e99674..aae6948b01 100644 --- a/src/test/resources/restapi/securityconfig_nondefault.json +++ b/src/test/resources/restapi/securityconfig_nondefault.json @@ -9,7 +9,8 @@ "private_tenant_enabled" : true, "default_tenant" : "", "server_username" : "kibanaserver", - "index" : ".kibana" + "index" : ".kibana", + "sign_in_options":["BASIC"] }, "http" : { "anonymous_auth_enabled" : false, From 91efc3b3f063624c661967806a42fdc3943a06aa Mon Sep 17 00:00:00 2001 From: Darshit Chanpura <35282393+DarshitChanpura@users.noreply.github.com> Date: Tue, 19 Mar 2024 11:58:59 -0400 Subject: [PATCH 146/204] Updates admin password string only if correct hash is present (#4100) Signed-off-by: Darshit Chanpura --- .../security/tools/democonfig/Installer.java | 5 +- .../SecuritySettingsConfigurer.java | 71 +++++++++------- .../SecuritySettingsConfigurerTests.java | 81 +++++++++++++++++-- 3 files changed, 120 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/opensearch/security/tools/democonfig/Installer.java b/src/main/java/org/opensearch/security/tools/democonfig/Installer.java index 864607a9c6..f1ee81f84e 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/Installer.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/Installer.java @@ -14,6 +14,7 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileReader; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -97,7 +98,7 @@ public static Installer getInstance() { * Installs the demo security configuration * @param options the options passed to the script */ - public void installDemoConfiguration(String[] options) { + public void installDemoConfiguration(String[] options) throws IOException { readOptions(options); printScriptHeaders(); gatherUserInputs(); @@ -108,7 +109,7 @@ public void installDemoConfiguration(String[] options) { finishScriptExecution(); } - public static void main(String[] options) { + public static void main(String[] options) throws IOException { Installer installer = Installer.getInstance(); installer.buildOptions(); installer.installDemoConfiguration(options); diff --git a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java index 5b497d0f20..92a7915114 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java @@ -12,24 +12,22 @@ package org.opensearch.security.tools.democonfig; import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.File; +import java.io.FileInputStream; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; +import org.bouncycastle.crypto.generators.OpenBSDBCrypt; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; -import org.opensearch.security.DefaultObjectMapper; import org.opensearch.security.dlic.rest.validation.PasswordValidator; import org.opensearch.security.dlic.rest.validation.RequestContentValidator; import org.opensearch.security.support.ConfigConstants; @@ -38,6 +36,7 @@ import org.yaml.snakeyaml.DumperOptions; import org.yaml.snakeyaml.Yaml; +import static org.opensearch.security.DefaultObjectMapper.YAML_MAPPER; import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_PASSWORD_MIN_LENGTH; import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_PASSWORD_VALIDATION_REGEX; @@ -81,6 +80,7 @@ public class SecuritySettingsConfigurer { static String ADMIN_USERNAME = "admin"; private final Installer installer; + static final String DEFAULT_ADMIN_PASSWORD = "admin"; public SecuritySettingsConfigurer(Installer installer) { this.installer = installer; @@ -92,7 +92,7 @@ public SecuritySettingsConfigurer(Installer installer) { * 2. Sets the custom admin password (Generates one if none is provided) * 3. Write the security config to opensearch.yml */ - public void configureSecuritySettings() { + public void configureSecuritySettings() throws IOException { checkIfSecurityPluginIsAlreadyConfigured(); updateAdminPassword(); writeSecurityConfigToOpenSearchYML(); @@ -125,9 +125,17 @@ void checkIfSecurityPluginIsAlreadyConfigured() { /** * Replaces the admin password in internal_users.yml with the custom or generated password */ - void updateAdminPassword() { + void updateAdminPassword() throws IOException { String INTERNAL_USERS_FILE_PATH = installer.OPENSEARCH_CONF_DIR + "opensearch-security" + File.separator + "internal_users.yml"; boolean shouldValidatePassword = installer.environment.equals(ExecutionEnvironment.DEMO); + + // check if the password `admin` is present, if not skip updating admin password + if (!isAdminPasswordSetToAdmin(INTERNAL_USERS_FILE_PATH)) { + System.out.println("Admin password seems to be custom configured. Skipping update to admin password."); + return; + } + + // if hashed value for default password "admin" is found, update it with the custom password. try { final PasswordValidator passwordValidator = PasswordValidator.of( Settings.builder() @@ -169,17 +177,29 @@ void updateAdminPassword() { System.exit(-1); } - // Print an update to the logs - System.out.println("Admin password set successfully."); - + // Update the custom password in internal_users.yml file writePasswordToInternalUsersFile(ADMIN_PASSWORD, INTERNAL_USERS_FILE_PATH); + System.out.println("Admin password set successfully."); + } catch (IOException e) { System.out.println("Exception updating the admin password : " + e.getMessage()); System.exit(-1); } } + /** + * Check if the password for admin user was already updated. (Possibly via a custom internal_users.yml) + * @param internalUsersFile Path to internal_users.yml file + * @return true if password was already updated, false otherwise + * @throws IOException if there was an error while reading the file + */ + private boolean isAdminPasswordSetToAdmin(String internalUsersFile) throws IOException { + JsonNode internalUsers = YAML_MAPPER.readTree(new FileInputStream(internalUsersFile)); + return internalUsers.has("admin") + && OpenBSDBCrypt.checkPassword(internalUsers.get("admin").get("hash").asText(), DEFAULT_ADMIN_PASSWORD.toCharArray()); + } + /** * Generate password hash and update it in the internal_users.yml file * @param adminPassword the password to be hashed and updated @@ -190,31 +210,24 @@ void writePasswordToInternalUsersFile(String adminPassword, String internalUsers String hashedAdminPassword = Hasher.hash(adminPassword.toCharArray()); if (hashedAdminPassword.isEmpty()) { - System.out.println("Hash the admin password failure, see console for details"); + System.out.println("Failure while hashing the admin password, see console for details."); System.exit(-1); } - Path tempFilePath = Paths.get(internalUsersFile + ".tmp"); - Path internalUsersPath = Paths.get(internalUsersFile); - - try ( - BufferedReader reader = new BufferedReader(new FileReader(internalUsersFile, StandardCharsets.UTF_8)); - BufferedWriter writer = new BufferedWriter(new FileWriter(tempFilePath.toFile(), StandardCharsets.UTF_8)) - ) { - String line; - while ((line = reader.readLine()) != null) { - if (line.matches(" *hash: *\"\\$2a\\$12\\$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG\"")) { - line = line.replace( - "\"$2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG\"", - "\"" + hashedAdminPassword + "\"" - ); - } - writer.write(line + System.lineSeparator()); + try { + var map = YAML_MAPPER.readValue(new File(internalUsersFile), new TypeReference>>() { + }); + var admin = map.get("admin"); + if (admin != null) { + // Replace the password since the default password was found via the check: isAdminPasswordSetToAdmin(..) + admin.put("hash", hashedAdminPassword); } + + // Write the updated map back to the internal_users.yml file + YAML_MAPPER.writeValue(new File(internalUsersFile), map); } catch (IOException e) { throw new IOException("Unable to update the internal users file with the hashed password."); } - Files.move(tempFilePath, internalUsersPath, java.nio.file.StandardCopyOption.REPLACE_EXISTING); } /** @@ -329,7 +342,7 @@ static boolean isNodeMaxLocalStorageNodesAlreadyPresent(String filePath) { static boolean isKeyPresentInYMLFile(String filePath, String key) throws IOException { JsonNode node; try { - node = DefaultObjectMapper.YAML_MAPPER.readTree(new File(filePath)); + node = YAML_MAPPER.readTree(new File(filePath)); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java b/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java index 50a65e7fa2..f4b56e6f76 100644 --- a/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java +++ b/src/test/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurerTests.java @@ -21,16 +21,22 @@ import java.io.PrintStream; import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; +import org.apache.commons.lang3.RandomStringUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.opensearch.security.support.ConfigConstants; +import org.opensearch.security.tools.Hasher; import org.opensearch.security.tools.democonfig.util.NoExitSecurityManager; import static org.hamcrest.MatcherAssert.assertThat; @@ -39,6 +45,7 @@ import static org.hamcrest.Matchers.is; import static org.opensearch.security.dlic.rest.validation.RequestContentValidator.ValidationError.INVALID_PASSWORD_INVALID_REGEX; import static org.opensearch.security.dlic.rest.validation.RequestContentValidator.ValidationError.INVALID_PASSWORD_TOO_SHORT; +import static org.opensearch.security.tools.democonfig.SecuritySettingsConfigurer.DEFAULT_ADMIN_PASSWORD; import static org.opensearch.security.tools.democonfig.SecuritySettingsConfigurer.DEFAULT_PASSWORD_MIN_LENGTH; import static org.opensearch.security.tools.democonfig.SecuritySettingsConfigurer.REST_ENABLED_ROLES; import static org.opensearch.security.tools.democonfig.SecuritySettingsConfigurer.SYSTEM_INDICES; @@ -66,13 +73,14 @@ public class SecuritySettingsConfigurerTests { private static Installer installer; @Before - public void setUp() { + public void setUp() throws IOException { System.setOut(new PrintStream(outContent)); System.setErr(new PrintStream(outContent)); installer = Installer.getInstance(); installer.buildOptions(); securitySettingsConfigurer = new SecuritySettingsConfigurer(installer); setUpConf(); + setUpInternalUsersYML(); } @After @@ -87,7 +95,7 @@ public void tearDown() throws NoSuchFieldException, IllegalAccessException { } @Test - public void testUpdateAdminPasswordWithCustomPassword() throws NoSuchFieldException, IllegalAccessException { + public void testUpdateAdminPasswordWithCustomPassword() throws NoSuchFieldException, IllegalAccessException, IOException { String customPassword = "myStrongPassword123"; setEnv(adminPasswordKey, customPassword); @@ -104,7 +112,7 @@ public void testUpdateAdminPassword_noPasswordSupplied() { try { System.setSecurityManager(new NoExitSecurityManager()); securitySettingsConfigurer.updateAdminPassword(); - } catch (SecurityException e) { + } catch (SecurityException | IOException e) { assertThat(e.getMessage(), equalTo("System.exit(-1) blocked to allow print statement testing.")); } finally { System.setSecurityManager(null); @@ -125,7 +133,7 @@ public void testUpdateAdminPasswordWithWeakPassword() throws NoSuchFieldExceptio try { System.setSecurityManager(new NoExitSecurityManager()); securitySettingsConfigurer.updateAdminPassword(); - } catch (SecurityException e) { + } catch (SecurityException | IOException e) { assertThat(e.getMessage(), equalTo("System.exit(-1) blocked to allow print statement testing.")); } finally { System.setSecurityManager(null); @@ -148,7 +156,7 @@ public void testUpdateAdminPasswordWithShortPassword() throws NoSuchFieldExcepti try { System.setSecurityManager(new NoExitSecurityManager()); securitySettingsConfigurer.updateAdminPassword(); - } catch (SecurityException e) { + } catch (SecurityException | IOException e) { assertThat(e.getMessage(), equalTo("System.exit(-1) blocked to allow print statement testing.")); } finally { System.setSecurityManager(null); @@ -160,7 +168,8 @@ public void testUpdateAdminPasswordWithShortPassword() throws NoSuchFieldExcepti } @Test - public void testUpdateAdminPasswordWithWeakPassword_skipPasswordValidation() throws NoSuchFieldException, IllegalAccessException { + public void testUpdateAdminPasswordWithWeakPassword_skipPasswordValidation() throws NoSuchFieldException, IllegalAccessException, + IOException { setEnv(adminPasswordKey, "weakpassword"); installer.environment = ExecutionEnvironment.TEST; securitySettingsConfigurer.updateAdminPassword(); @@ -170,6 +179,49 @@ public void testUpdateAdminPasswordWithWeakPassword_skipPasswordValidation() thr verifyStdOutContainsString("Admin password set successfully."); } + @Test + public void testUpdateAdminPasswordWithCustomInternalUsersYML() throws IOException { + String internalUsersFile = installer.OPENSEARCH_CONF_DIR + "opensearch-security" + File.separator + "internal_users.yml"; + Path internalUsersFilePath = Paths.get(internalUsersFile); + + List newContent = Arrays.asList( + "_meta:", + " type: \"internalusers\"", + " config_version: 2", + "admin:", + " hash: " + Hasher.hash(RandomStringUtils.randomAlphanumeric(16).toCharArray()), + " backend_roles:", + " - \"admin\"" + ); + // overwriting existing content + Files.write(internalUsersFilePath, newContent, StandardCharsets.UTF_8); + + securitySettingsConfigurer.updateAdminPassword(); + + verifyStdOutContainsString("Admin password seems to be custom configured. Skipping update to admin password."); + } + + @Test + public void testUpdateAdminPasswordWithDefaultInternalUsersYml() { + + SecuritySettingsConfigurer.ADMIN_PASSWORD = ""; // to ensure 0 flaky-ness + try { + System.setSecurityManager(new NoExitSecurityManager()); + securitySettingsConfigurer.updateAdminPassword(); + } catch (SecurityException | IOException e) { + assertThat(e.getMessage(), equalTo("System.exit(-1) blocked to allow print statement testing.")); + } finally { + System.setSecurityManager(null); + } + + verifyStdOutContainsString( + String.format( + "No custom admin password found. Please provide a password via the environment variable %s.", + ConfigConstants.OPENSEARCH_INITIAL_ADMIN_PASSWORD + ) + ); + } + @Test public void testSecurityPluginAlreadyConfigured() { securitySettingsConfigurer.writeSecurityConfigToOpenSearchYML(); @@ -353,4 +405,21 @@ void setUpConf() { private void verifyStdOutContainsString(String s) { assertThat(outContent.toString(), containsString(s)); } + + private void setUpInternalUsersYML() throws IOException { + String internalUsersFile = installer.OPENSEARCH_CONF_DIR + "opensearch-security" + File.separator + "internal_users.yml"; + Path internalUsersFilePath = Paths.get(internalUsersFile); + List defaultContent = Arrays.asList( + "_meta:", + " type: \"internalusers\"", + " config_version: 2", + "admin:", + " hash: " + Hasher.hash(DEFAULT_ADMIN_PASSWORD.toCharArray()), + " reserved: " + true, + " backend_roles:", + " - \"admin\"", + " description: Demo admin user" + ); + Files.write(internalUsersFilePath, defaultContent, StandardCharsets.UTF_8); + } } From 633ff9b30cd34acfb81a264fc49b3d716dd89c64 Mon Sep 17 00:00:00 2001 From: Sicheng Song Date: Tue, 19 Mar 2024 16:10:55 -0700 Subject: [PATCH 147/204] Add query assistant role and new ml system indices (#4141) Signed-off-by: Sicheng Song --- config/roles.yml | 9 +++++++++ .../tools/democonfig/SecuritySettingsConfigurer.java | 2 ++ 2 files changed, 11 insertions(+) diff --git a/config/roles.yml b/config/roles.yml index efa83ed02e..e90a4e62a6 100644 --- a/config/roles.yml +++ b/config/roles.yml @@ -271,6 +271,15 @@ cross_cluster_search_remote_full_access: - 'indices:admin/shards/search_shards' - 'indices:data/read/search' +# Allow users to operate query assistant +ml_query_assistant_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opensearch/ml/execute' + - 'cluster:admin/opensearch/ml/memory/conversation/create' + - 'cluster:admin/opensearch/ml/memory/interaction/create' + - 'cluster:admin/opensearch/ml/predict' + # Allow users to read ML stats/models/tasks ml_read_access: reserved: true diff --git a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java index 92a7915114..e66215ac9a 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java @@ -47,8 +47,10 @@ public class SecuritySettingsConfigurer { static final List REST_ENABLED_ROLES = List.of("all_access", "security_rest_api_access"); static final List SYSTEM_INDICES = List.of( + ".plugins-ml-agent", ".plugins-ml-config", ".plugins-ml-connector", + ".plugins-ml-controller", ".plugins-ml-model-group", ".plugins-ml-model", ".plugins-ml-task", From fa877babe3dac13c30bcc2bcbe4d484bcdb6101f Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Wed, 20 Mar 2024 12:05:06 -0500 Subject: [PATCH 148/204] [Feature] Check for and perform upgrades on security configurations (#4102) ### Description This adds a new API that allows for checking and updating configurations from the default configurations on disk. Initial feature supports only Roles. #### Response when no upgrade is available ``` GET _plugins/_security/api/_upgrade_check 200 { "status": "ok", "upgradeAvailable" : false } ``` #### Response when a new role is available ``` GET _plugins/_security/api/_upgrade_check 200 { "status": "ok", "upgradeAvailable" : true, "upgradeActions" : { "roles" : { "add" : [ "flow_framework_full_access" ] } } } ``` #### Response when a new role + existing role were updated ``` GET _plugins/_security/api/_upgrade_check 200 { "status": "ok", "upgradeAvailable" : true, "upgradeActions" : { "roles" : { "add" : [ "flow_framework_full_access" ], "modify" : [ "flow_framework_read_access" ] } } } ``` #### Perform an upgrade ``` POST _plugins/_security/api/_upgrade_perform 200 { "status" : "OK", "upgrades" : { "roles" : { "add" : [ "flow_framework_full_access" ], "modify" : [ "flow_framework_read_access" ] } } } ``` #### Perform an upgrade when unneeded ``` POST _plugins/_security/api/_upgrade_perform 400 { "status": "BAD_REQUEST", "message": "Unable to upgrade, no differences found in 'roles' config" } ``` ### Issues Resolved - https://github.com/opensearch-project/security/issues/2316 ### Testing New unit test and integration test cases ### Check List - [X] New functionality includes testing - [ ] New functionality has been documented - [X] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: Peter Nied Signed-off-by: Andrey Pleskach Signed-off-by: Peter Nied --- .../security/DefaultConfigurationTests.java | 100 ++++- src/integrationTest/resources/roles.yml | 18 + .../ConfigurationRepository.java | 13 +- .../dlic/rest/api/AbstractApiAction.java | 54 ++- .../dlic/rest/api/ConfigUpgradeApiAction.java | 400 ++++++++++++++++++ .../dlic/rest/api/SecurityRestApiActions.java | 3 +- .../validation/RequestContentValidator.java | 4 +- .../rest/validation/ValidationResult.java | 19 +- .../security/securityconf/impl/CType.java | 59 ++- .../api/AbstractApiActionValidationTest.java | 4 +- .../api/ConfigUpgradeApiActionUnitTest.java | 291 +++++++++++++ .../InternalUsersApiActionValidationTest.java | 20 +- 12 files changed, 929 insertions(+), 56 deletions(-) create mode 100644 src/main/java/org/opensearch/security/dlic/rest/api/ConfigUpgradeApiAction.java create mode 100644 src/test/java/org/opensearch/security/dlic/rest/api/ConfigUpgradeApiActionUnitTest.java diff --git a/src/integrationTest/java/org/opensearch/security/DefaultConfigurationTests.java b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationTests.java index 8bb5b96145..eb028c74e4 100644 --- a/src/integrationTest/java/org/opensearch/security/DefaultConfigurationTests.java +++ b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationTests.java @@ -11,10 +11,13 @@ import java.io.IOException; import java.nio.file.Path; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import com.fasterxml.jackson.databind.JsonNode; import org.apache.commons.io.FileUtils; import org.awaitility.Awaitility; import org.junit.AfterClass; @@ -22,6 +25,7 @@ import org.junit.Test; import org.junit.runner.RunWith; +import org.opensearch.test.framework.TestSecurityConfig.User; import org.opensearch.test.framework.cluster.ClusterManager; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; @@ -32,16 +36,16 @@ import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.not; @RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) @ThreadLeakScope(ThreadLeakScope.Scope.NONE) public class DefaultConfigurationTests { private final static Path configurationFolder = ConfigurationFiles.createConfigurationDirectory(); - public static final String ADMIN_USER_NAME = "admin"; - public static final String DEFAULT_PASSWORD = "secret"; - public static final String NEW_USER = "new-user"; - public static final String LIMITED_USER = "limited-user"; + private static final User ADMIN_USER = new User("admin"); + private static final User NEW_USER = new User("new-user"); + private static final User LIMITED_USER = new User("limited-user"); @ClassRule public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.SINGLENODE) @@ -64,15 +68,95 @@ public static void cleanConfigurationDirectory() throws IOException { @Test public void shouldLoadDefaultConfiguration() { - try (TestRestClient client = cluster.getRestClient(NEW_USER, DEFAULT_PASSWORD)) { + try (TestRestClient client = cluster.getRestClient(NEW_USER)) { Awaitility.await().alias("Load default configuration").until(() -> client.getAuthInfo().getStatusCode(), equalTo(200)); } - try (TestRestClient client = cluster.getRestClient(ADMIN_USER_NAME, DEFAULT_PASSWORD)) { - client.confirmCorrectCredentials(ADMIN_USER_NAME); + try (TestRestClient client = cluster.getRestClient(ADMIN_USER)) { + client.confirmCorrectCredentials(ADMIN_USER.getName()); HttpResponse response = client.get("_plugins/_security/api/internalusers"); response.assertStatusCode(200); Map users = response.getBodyAs(Map.class); - assertThat(users, allOf(aMapWithSize(3), hasKey(ADMIN_USER_NAME), hasKey(NEW_USER), hasKey(LIMITED_USER))); + assertThat( + users, + allOf(aMapWithSize(3), hasKey(ADMIN_USER.getName()), hasKey(NEW_USER.getName()), hasKey(LIMITED_USER.getName())) + ); } } + + @Test + public void securityRolesUgrade() throws Exception { + try (var client = cluster.getRestClient(ADMIN_USER)) { + // Setup: Make sure the config is ready before starting modifications + Awaitility.await().alias("Load default configuration").until(() -> client.getAuthInfo().getStatusCode(), equalTo(200)); + + // Setup: Collect default roles after cluster start + final var expectedRoles = client.get("_plugins/_security/api/roles/"); + final var expectedRoleNames = extractFieldNames(expectedRoles.getBodyAs(JsonNode.class)); + + // Verify: Before any changes, nothing to upgrade + final var upgradeCheck = client.get("_plugins/_security/api/_upgrade_check"); + upgradeCheck.assertStatusCode(200); + assertThat(upgradeCheck.getBooleanFromJsonBody("/upgradeAvailable"), equalTo(false)); + + // Action: Select a role that is part of the defaults and delete that role + final var roleToDelete = "flow_framework_full_access"; + client.delete("_plugins/_security/api/roles/" + roleToDelete).assertStatusCode(200); + + // Action: Select a role that is part of the defaults and alter that role with removal, edits, and additions + final var roleToAlter = "flow_framework_read_access"; + final var originalRoleConfig = client.get("_plugins/_security/api/roles/" + roleToAlter).getBodyAs(JsonNode.class); + final var alteredRoleReponse = client.patch("_plugins/_security/api/roles/" + roleToAlter, "[\n" + // + " {\n" + // + " \"op\": \"replace\",\n" + // + " \"path\": \"/cluster_permissions\",\n" + // + " \"value\": [\"a\", \"b\", \"c\"]\n" + // + " },\n" + // + " {\n" + // + " \"op\": \"add\",\n" + // + " \"path\": \"/index_permissions\",\n" + // + " \"value\": [ {\n" + // + " \"index_patterns\": [\"*\"],\n" + // + " \"allowed_actions\": [\"*\"]\n" + // + " }\n" + // + " ]\n" + // + " }\n" + // + "]"); + alteredRoleReponse.assertStatusCode(200); + final var alteredRoleJson = alteredRoleReponse.getBodyAs(JsonNode.class); + assertThat(originalRoleConfig, not(equalTo(alteredRoleJson))); + + // Verify: Confirm that the upgrade check detects the changes associated with both role resources + final var upgradeCheckAfterChanges = client.get("_plugins/_security/api/_upgrade_check"); + upgradeCheckAfterChanges.assertStatusCode(200); + assertThat( + upgradeCheckAfterChanges.getTextArrayFromJsonBody("/upgradeActions/roles/add"), + equalTo(List.of("flow_framework_full_access")) + ); + assertThat( + upgradeCheckAfterChanges.getTextArrayFromJsonBody("/upgradeActions/roles/modify"), + equalTo(List.of("flow_framework_read_access")) + ); + + // Action: Perform the upgrade to the roles configuration + final var performUpgrade = client.post("_plugins/_security/api/_upgrade_perform"); + performUpgrade.assertStatusCode(200); + assertThat(performUpgrade.getTextArrayFromJsonBody("/upgrades/roles/add"), equalTo(List.of("flow_framework_full_access"))); + assertThat(performUpgrade.getTextArrayFromJsonBody("/upgrades/roles/modify"), equalTo(List.of("flow_framework_read_access"))); + + // Verify: Same roles as the original state - the deleted role has been restored + final var afterUpgradeRoles = client.get("_plugins/_security/api/roles/"); + final var afterUpgradeRolesNames = extractFieldNames(afterUpgradeRoles.getBodyAs(JsonNode.class)); + assertThat(afterUpgradeRolesNames, equalTo(expectedRoleNames)); + + // Verify: Altered role was restored to its expected state + final var afterUpgradeAlteredRoleConfig = client.get("_plugins/_security/api/roles/" + roleToAlter).getBodyAs(JsonNode.class); + assertThat(originalRoleConfig, equalTo(afterUpgradeAlteredRoleConfig)); + } + } + + private Set extractFieldNames(final JsonNode json) { + final var set = new HashSet(); + json.fieldNames().forEachRemaining(set::add); + return set; + } } diff --git a/src/integrationTest/resources/roles.yml b/src/integrationTest/resources/roles.yml index 02de9bf3d5..2ea7548ad6 100644 --- a/src/integrationTest/resources/roles.yml +++ b/src/integrationTest/resources/roles.yml @@ -17,3 +17,21 @@ user_limited-user__limited-role: allowed_actions: - "indices:data/read/get" - "indices:data/read/search" +flow_framework_full_access: + cluster_permissions: + - 'cluster:admin/opensearch/flow_framework/*' + - 'cluster_monitor' + index_permissions: + - index_patterns: + - '*' + allowed_actions: + - 'indices:admin/aliases/get' + - 'indices:admin/mappings/get' + - 'indices_monitor' +flow_framework_read_access: + cluster_permissions: + - 'cluster:admin/opensearch/flow_framework/workflow/get' + - 'cluster:admin/opensearch/flow_framework/workflow/search' + - 'cluster:admin/opensearch/flow_framework/workflow_state/get' + - 'cluster:admin/opensearch/flow_framework/workflow_state/search' + - 'cluster:admin/opensearch/flow_framework/workflow_step/get' diff --git a/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java b/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java index dfbeb16cb3..353286fc4a 100644 --- a/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java +++ b/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java @@ -123,6 +123,14 @@ private ConfigurationRepository( configCache = CacheBuilder.newBuilder().build(); } + public String getConfigDirectory() { + String lookupDir = System.getProperty("security.default_init.dir"); + final String cd = lookupDir != null + ? (lookupDir + "/") + : new Environment(settings, configPath).configDir().toAbsolutePath().toString() + "/opensearch-security/"; + return cd; + } + private void initalizeClusterConfiguration(final boolean installDefaultConfig) { try { LOGGER.info("Background init thread started. Install default config?: " + installDefaultConfig); @@ -135,10 +143,7 @@ private void initalizeClusterConfiguration(final boolean installDefaultConfig) { if (installDefaultConfig) { try { - String lookupDir = System.getProperty("security.default_init.dir"); - final String cd = lookupDir != null - ? (lookupDir + "/") - : new Environment(settings, configPath).configDir().toAbsolutePath().toString() + "/opensearch-security/"; + final String cd = getConfigDirectory(); File confFile = new File(cd + "config.yml"); if (confFile.exists()) { final ThreadContext threadContext = threadPool.getThreadContext(); diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/AbstractApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/AbstractApiAction.java index ef8a00d700..a38d618da0 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/AbstractApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/AbstractApiAction.java @@ -35,13 +35,16 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.CheckedSupplier; +import org.opensearch.common.action.ActionFuture; import org.opensearch.common.util.concurrent.ThreadContext.StoredContext; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.Strings; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentHelper; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; @@ -226,7 +229,7 @@ protected final ValidationResult patchEntity( ); } - protected final ValidationResult patchEntities( + protected ValidationResult patchEntities( final RestRequest request, final JsonNode patchContent, final SecurityConfiguration securityConfiguration @@ -336,7 +339,7 @@ final void saveOrUpdateConfiguration( final SecurityDynamicConfiguration configuration, final OnSucessActionListener onSucessActionListener ) { - saveAndUpdateConfigs(securityApiDependencies.securityIndexName(), client, getConfigType(), configuration, onSucessActionListener); + saveAndUpdateConfigsAsync(securityApiDependencies, client, getConfigType(), configuration, onSucessActionListener); } protected final String nameParam(final RestRequest request) { @@ -367,7 +370,7 @@ protected final ValidationResult loadConfiguration(final ); } - protected final ValidationResult> loadConfiguration( + protected ValidationResult> loadConfiguration( final CType cType, boolean omitSensitiveData, final boolean logComplianceEvent @@ -485,30 +488,45 @@ public final void onFailure(Exception e) { } - public static void saveAndUpdateConfigs( - final String indexName, + public static ActionFuture saveAndUpdateConfigs( + final SecurityApiDependencies dependencies, + final Client client, + final CType cType, + final SecurityDynamicConfiguration configuration + ) { + final var request = createIndexRequestForConfig(dependencies, cType, configuration); + return client.index(request); + } + + public static void saveAndUpdateConfigsAsync( + final SecurityApiDependencies dependencies, final Client client, final CType cType, final SecurityDynamicConfiguration configuration, final ActionListener actionListener ) { - final IndexRequest ir = new IndexRequest(indexName); - final String id = cType.toLCString(); + final var ir = createIndexRequestForConfig(dependencies, cType, configuration); + client.index(ir, new ConfigUpdatingActionListener<>(new String[] { cType.toLCString() }, client, actionListener)); + } + private static IndexRequest createIndexRequestForConfig( + final SecurityApiDependencies dependencies, + final CType cType, + final SecurityDynamicConfiguration configuration + ) { configuration.removeStatic(); - + final BytesReference content; try { - client.index( - ir.id(id) - .setRefreshPolicy(RefreshPolicy.IMMEDIATE) - .setIfSeqNo(configuration.getSeqNo()) - .setIfPrimaryTerm(configuration.getPrimaryTerm()) - .source(id, XContentHelper.toXContent(configuration, XContentType.JSON, false)), - new ConfigUpdatingActionListener<>(new String[] { id }, client, actionListener) - ); - } catch (IOException e) { + content = XContentHelper.toXContent(configuration, XContentType.JSON, ToXContent.EMPTY_PARAMS, false); + } catch (final IOException e) { throw ExceptionsHelper.convertToOpenSearchException(e); } + + return new IndexRequest(dependencies.securityIndexName()).id(cType.toLCString()) + .setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .setIfSeqNo(configuration.getSeqNo()) + .setIfPrimaryTerm(configuration.getPrimaryTerm()) + .source(cType.toLCString(), content); } protected static class ConfigUpdatingActionListener implements ActionListener { diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/ConfigUpgradeApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/ConfigUpgradeApiAction.java new file mode 100644 index 0000000000..f295ab8c1c --- /dev/null +++ b/src/main/java/org/opensearch/security/dlic/rest/api/ConfigUpgradeApiAction.java @@ -0,0 +1,400 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.dlic.rest.api; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.collect.Tuple; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestRequest.Method; +import org.opensearch.security.configuration.ConfigurationRepository; +import org.opensearch.security.dlic.rest.support.Utils; +import org.opensearch.security.dlic.rest.validation.EndpointValidator; +import org.opensearch.security.dlic.rest.validation.RequestContentValidator; +import org.opensearch.security.dlic.rest.validation.RequestContentValidator.DataType; +import org.opensearch.security.dlic.rest.validation.ValidationResult; +import org.opensearch.security.securityconf.impl.CType; +import org.opensearch.security.securityconf.impl.SecurityDynamicConfiguration; +import org.opensearch.security.support.ConfigHelper; +import org.opensearch.threadpool.ThreadPool; + +import com.flipkart.zjsonpatch.DiffFlags; +import com.flipkart.zjsonpatch.JsonDiff; + +import static org.opensearch.security.dlic.rest.api.Responses.badRequestMessage; +import static org.opensearch.security.dlic.rest.api.Responses.response; +import static org.opensearch.security.dlic.rest.support.Utils.addRoutesPrefix; +import static org.opensearch.security.dlic.rest.support.Utils.withIOException; + +public class ConfigUpgradeApiAction extends AbstractApiAction { + + private final static Logger LOGGER = LogManager.getLogger(ConfigUpgradeApiAction.class); + + private final static Set SUPPORTED_CTYPES = ImmutableSet.of(CType.ROLES); + + private final static String REQUEST_PARAM_CONFIGS_KEY = "configs"; + + private static final List routes = addRoutesPrefix( + ImmutableList.of(new Route(Method.GET, "/_upgrade_check"), new Route(Method.POST, "/_upgrade_perform")) + ); + + @Inject + public ConfigUpgradeApiAction( + final ClusterService clusterService, + final ThreadPool threadPool, + final SecurityApiDependencies securityApiDependencies + ) { + super(Endpoint.CONFIG, clusterService, threadPool, securityApiDependencies); + this.requestHandlersBuilder.configureRequestHandlers(rhb -> { + rhb.allMethodsNotImplemented().add(Method.GET, this::canUpgrade).add(Method.POST, this::performUpgrade); + }); + } + + void canUpgrade(final RestChannel channel, final RestRequest request, final Client client) throws IOException { + getAndValidateConfigurationsToUpgrade(request).map(this::configurationDifferences).valid(differencesList -> { + final var allConfigItemChanges = differencesList.stream() + .map(kvp -> new ConfigItemChanges(kvp.v1(), kvp.v2())) + .collect(Collectors.toList()); + + final var upgradeAvailable = allConfigItemChanges.stream().anyMatch(ConfigItemChanges::hasChanges); + + final ObjectNode response = JsonNodeFactory.instance.objectNode(); + response.put("status", "OK"); + response.put("upgradeAvailable", upgradeAvailable); + + if (upgradeAvailable) { + final ObjectNode differences = JsonNodeFactory.instance.objectNode(); + allConfigItemChanges.forEach(configItemChanges -> configItemChanges.addToNode(differences)); + response.set("upgradeActions", differences); + } + channel.sendResponse(new BytesRestResponse(RestStatus.OK, XContentType.JSON.mediaType(), response.toPrettyString())); + }).error((status, toXContent) -> response(channel, status, toXContent)); + } + + void performUpgrade(final RestChannel channel, final RestRequest request, final Client client) throws IOException { + getAndValidateConfigurationsToUpgrade(request).map(this::configurationDifferences) + .map(this::verifyHasDifferences) + .map(diffs -> applyDifferences(request, client, diffs)) + .valid(updatedConfigs -> { + final var response = JsonNodeFactory.instance.objectNode(); + response.put("status", "OK"); + + final var allUpdates = JsonNodeFactory.instance.objectNode(); + updatedConfigs.forEach(configItemChanges -> configItemChanges.addToNode(allUpdates)); + response.set("upgrades", allUpdates); + + channel.sendResponse(new BytesRestResponse(RestStatus.OK, XContentType.JSON.mediaType(), response.toPrettyString())); + }) + .error((status, toXContent) -> response(channel, status, toXContent)); + } + + private ValidationResult> applyDifferences( + final RestRequest request, + final Client client, + final List> differencesToUpdate + ) { + try { + final var updatedResources = new ArrayList>(); + for (final Tuple difference : differencesToUpdate) { + updatedResources.add( + loadConfiguration(difference.v1(), false, false).map( + configuration -> patchEntities(request, difference.v2(), SecurityConfiguration.of(null, configuration)).map( + patchResults -> { + final var response = saveAndUpdateConfigs( + securityApiDependencies, + client, + difference.v1(), + patchResults.configuration() + ); + return ValidationResult.success(response.actionGet()); + } + ).map(indexResponse -> { + + final var itemsGroupedByOperation = new ConfigItemChanges(difference.v1(), difference.v2()); + return ValidationResult.success(itemsGroupedByOperation); + }) + ) + ); + } + + return ValidationResult.merge(updatedResources); + } catch (final Exception ioe) { + LOGGER.debug("Error while applying differences", ioe); + return ValidationResult.error( + RestStatus.BAD_REQUEST, + badRequestMessage("Error applying configuration, see the log file to troubleshoot.") + ); + } + + } + + ValidationResult>> verifyHasDifferences(List> diffs) { + if (diffs.isEmpty()) { + return ValidationResult.error(RestStatus.BAD_REQUEST, badRequestMessage("Unable to upgrade, no differences found")); + } + + for (final var diff : diffs) { + if (diff.v2().size() == 0) { + return ValidationResult.error( + RestStatus.BAD_REQUEST, + badRequestMessage("Unable to upgrade, no differences found in '" + diff.v1().toLCString() + "' config") + ); + } + } + return ValidationResult.success(diffs); + } + + private ValidationResult>> configurationDifferences(final Set configurations) { + try { + final var differences = new ArrayList>>(); + for (final var configuration : configurations) { + differences.add(computeDifferenceToUpdate(configuration)); + } + return ValidationResult.merge(differences); + } catch (final UncheckedIOException ioe) { + LOGGER.error("Error while processing differences", ioe.getCause()); + return ValidationResult.error( + RestStatus.BAD_REQUEST, + badRequestMessage("Error processing configuration, see the log file to troubleshoot.") + ); + } + } + + ValidationResult> computeDifferenceToUpdate(final CType configType) { + return withIOException(() -> loadConfiguration(configType, false, false).map(activeRoles -> { + final var activeRolesJson = Utils.convertJsonToJackson(activeRoles, true); + final var defaultRolesJson = loadConfigFileAsJson(configType); + final var rawDiff = JsonDiff.asJson(activeRolesJson, defaultRolesJson, EnumSet.of(DiffFlags.OMIT_VALUE_ON_REMOVE)); + return ValidationResult.success(new Tuple<>(configType, filterRemoveOperations(rawDiff))); + })); + } + + private ValidationResult> getAndValidateConfigurationsToUpgrade(final RestRequest request) { + final String[] configs = request.paramAsStringArray(REQUEST_PARAM_CONFIGS_KEY, null); + + final Set configurations; + try { + configurations = Optional.ofNullable(configs).map(CType::fromStringValues).orElse(SUPPORTED_CTYPES); + } catch (final IllegalArgumentException iae) { + return ValidationResult.error( + RestStatus.BAD_REQUEST, + badRequestMessage("Found invalid configuration option, valid options are: " + CType.lcStringValues()) + ); + } + + if (!configurations.stream().allMatch(SUPPORTED_CTYPES::contains)) { + // Remove all supported configurations + configurations.removeAll(SUPPORTED_CTYPES); + return ValidationResult.error( + RestStatus.BAD_REQUEST, + badRequestMessage("Unsupported configurations for upgrade, " + configurations) + ); + } + + return ValidationResult.success(configurations); + } + + private JsonNode filterRemoveOperations(final JsonNode diff) { + final ArrayNode filteredDiff = JsonNodeFactory.instance.arrayNode(); + diff.forEach(node -> { + if (!isRemoveOperation(node)) { + filteredDiff.add(node); + return; + } else { + if (!hasRootLevelPath(node)) { + filteredDiff.add(node); + } + } + }); + return filteredDiff; + } + + private static String pathRoot(final JsonNode node) { + return node.get("path").asText().split("/")[1]; + } + + private static boolean hasRootLevelPath(final JsonNode node) { + final var jsonPath = node.get("path").asText(); + return jsonPath.charAt(0) == '/' && !jsonPath.substring(1).contains("/"); + } + + private static boolean isRemoveOperation(final JsonNode node) { + return node.get("op").asText().equals("remove"); + } + + private SecurityDynamicConfiguration loadYamlFile(final String filepath, final CType cType) throws IOException { + return ConfigHelper.fromYamlFile(filepath, cType, ConfigurationRepository.DEFAULT_CONFIG_VERSION, 0, 0); + } + + JsonNode loadConfigFileAsJson(final CType cType) throws IOException { + final var cd = securityApiDependencies.configurationRepository().getConfigDirectory(); + final var filepath = cType.configFile(Path.of(cd)).toString(); + try { + return AccessController.doPrivileged((PrivilegedExceptionAction) () -> { + final var loadedConfiguration = loadYamlFile(filepath, cType); + return Utils.convertJsonToJackson(loadedConfiguration, true); + }); + } catch (final PrivilegedActionException e) { + LOGGER.error("Error when loading configuration from file", e); + throw (IOException) e.getCause(); + } + } + + @Override + public List routes() { + return routes; + } + + @Override + protected CType getConfigType() { + throw new UnsupportedOperationException("This class supports multiple configuration types"); + } + + @Override + protected EndpointValidator createEndpointValidator() { + return new EndpointValidator() { + + @Override + public Endpoint endpoint() { + return endpoint; + } + + @Override + public RestApiAdminPrivilegesEvaluator restApiAdminPrivilegesEvaluator() { + return securityApiDependencies.restApiAdminPrivilegesEvaluator(); + } + + @Override + public ValidationResult entityReserved(SecurityConfiguration securityConfiguration) { + // Allow modification of reserved entities + return ValidationResult.success(securityConfiguration); + } + + @Override + public ValidationResult entityHidden(SecurityConfiguration securityConfiguration) { + // Allow modification of hidden entities + return ValidationResult.success(securityConfiguration); + } + + @Override + public RequestContentValidator createRequestContentValidator(final Object... params) { + return new ConfigUpgradeContentValidator(new RequestContentValidator.ValidationContext() { + @Override + public Object[] params() { + return params; + } + + @Override + public Settings settings() { + return securityApiDependencies.settings(); + } + + @Override + public Map allowedKeys() { + return Map.of(REQUEST_PARAM_CONFIGS_KEY, DataType.ARRAY); + } + }); + } + }; + } + + /** More permissions validation that default ContentValidator */ + static class ConfigUpgradeContentValidator extends RequestContentValidator { + + protected ConfigUpgradeContentValidator(final ValidationContext validationContext) { + super(validationContext); + } + + @Override + public ValidationResult validate(final RestRequest request, final JsonNode jsonContent) throws IOException { + return validateContentSize(jsonContent); + } + } + + /** Tranforms config changes from a raw PATCH into simplier view */ + static class ConfigItemChanges { + + private final CType config; + private final Map> itemsGroupedByOperation; + + public ConfigItemChanges(final CType config, final JsonNode differences) { + this.config = config; + this.itemsGroupedByOperation = classifyChanges(differences); + } + + public boolean hasChanges() { + return !itemsGroupedByOperation.isEmpty(); + } + + /** Adds the config item changes to the json node */ + public void addToNode(final ObjectNode node) { + final var allOperations = JsonNodeFactory.instance.objectNode(); + itemsGroupedByOperation.forEach((operation, items) -> { + final var arrayNode = allOperations.putArray(operation); + items.forEach(arrayNode::add); + }); + node.set(config.toLCString(), allOperations); + } + + /** + * Classifies the changes to this config into groupings by the type of change, for + * multiple changes types on the same item they are groupped as 'modify' + */ + private static Map> classifyChanges(final JsonNode differences) { + final var items = new HashMap(); + differences.forEach(node -> { + final var item = pathRoot(node); + final var operation = node.get("op").asText(); + if (items.containsKey(item) && !items.get(item).equals(operation)) { + items.put(item, "modify"); + } else { + items.put(item, operation); + } + }); + + final var itemsGroupedByOperation = items.entrySet() + .stream() + .collect(Collectors.groupingBy(Map.Entry::getValue, Collectors.mapping(Map.Entry::getKey, Collectors.toList()))); + return itemsGroupedByOperation; + } + } +} diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/SecurityRestApiActions.java b/src/main/java/org/opensearch/security/dlic/rest/api/SecurityRestApiActions.java index b0d46f8774..f38cf0580d 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/SecurityRestApiActions.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/SecurityRestApiActions.java @@ -95,7 +95,8 @@ public static Collection getHandler( new AllowlistApiAction(Endpoint.ALLOWLIST, clusterService, threadPool, securityApiDependencies), new AuditApiAction(clusterService, threadPool, securityApiDependencies), new MultiTenancyConfigApiAction(clusterService, threadPool, securityApiDependencies), - new SecuritySSLCertsApiAction(clusterService, threadPool, securityKeyStore, certificatesReloadEnabled, securityApiDependencies) + new SecuritySSLCertsApiAction(clusterService, threadPool, securityKeyStore, certificatesReloadEnabled, securityApiDependencies), + new ConfigUpgradeApiAction(clusterService, threadPool, securityApiDependencies) ); } diff --git a/src/main/java/org/opensearch/security/dlic/rest/validation/RequestContentValidator.java b/src/main/java/org/opensearch/security/dlic/rest/validation/RequestContentValidator.java index 452bdd72e4..4d9faf096c 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/validation/RequestContentValidator.java +++ b/src/main/java/org/opensearch/security/dlic/rest/validation/RequestContentValidator.java @@ -142,7 +142,7 @@ private ValidationResult parseRequestContent(final RestRequest request } } - private ValidationResult validateContentSize(final JsonNode jsonContent) { + protected ValidationResult validateContentSize(final JsonNode jsonContent) { if (jsonContent.isEmpty()) { this.validationError = ValidationError.PAYLOAD_MANDATORY; return ValidationResult.error(RestStatus.BAD_REQUEST, this); @@ -150,7 +150,7 @@ private ValidationResult validateContentSize(final JsonNode jsonConten return ValidationResult.success(jsonContent); } - private ValidationResult validateJsonKeys(final JsonNode jsonContent) { + protected ValidationResult validateJsonKeys(final JsonNode jsonContent) { final Set requestedKeys = new HashSet<>(); jsonContent.fieldNames().forEachRemaining(requestedKeys::add); // mandatory settings, one of ... diff --git a/src/main/java/org/opensearch/security/dlic/rest/validation/ValidationResult.java b/src/main/java/org/opensearch/security/dlic/rest/validation/ValidationResult.java index ea782ea504..921ed4675d 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/validation/ValidationResult.java +++ b/src/main/java/org/opensearch/security/dlic/rest/validation/ValidationResult.java @@ -12,7 +12,9 @@ package org.opensearch.security.dlic.rest.validation; import java.io.IOException; +import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; import org.opensearch.common.CheckedBiConsumer; import org.opensearch.common.CheckedConsumer; @@ -50,6 +52,22 @@ public static ValidationResult error(final RestStatus status, final ToXCo return new ValidationResult<>(status, errorMessage); } + /** + * Transforms a list of validation results into a single validation result of that lists contents. + * If any of the validation results are not valid, the first is returned as the error. + */ + public static ValidationResult> merge(final List> results) { + if (results.stream().allMatch(ValidationResult::isValid)) { + return success(results.stream().map(result -> result.content).collect(Collectors.toList())); + } + + return results.stream() + .filter(result -> !result.isValid()) + .map(failedResult -> new ValidationResult>(failedResult.status, failedResult.errorMessage)) + .findFirst() + .get(); + } + public ValidationResult map(final CheckedFunction, IOException> mapper) throws IOException { if (content != null) { return Objects.requireNonNull(mapper).apply(content); @@ -82,5 +100,4 @@ public boolean isValid() { public ToXContent errorMessage() { return errorMessage; } - } diff --git a/src/main/java/org/opensearch/security/securityconf/impl/CType.java b/src/main/java/org/opensearch/security/securityconf/impl/CType.java index 4e5e2de496..23158e5850 100644 --- a/src/main/java/org/opensearch/security/securityconf/impl/CType.java +++ b/src/main/java/org/opensearch/security/securityconf/impl/CType.java @@ -27,14 +27,17 @@ package org.opensearch.security.securityconf.impl; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Collectors; +import com.google.common.collect.ImmutableMap; + import org.opensearch.security.auditlog.config.AuditConfig; import org.opensearch.security.securityconf.impl.v6.ActionGroupsV6; import org.opensearch.security.securityconf.impl.v6.ConfigV6; @@ -50,21 +53,39 @@ public enum CType { - INTERNALUSERS(toMap(1, InternalUserV6.class, 2, InternalUserV7.class)), - ACTIONGROUPS(toMap(0, List.class, 1, ActionGroupsV6.class, 2, ActionGroupsV7.class)), - CONFIG(toMap(1, ConfigV6.class, 2, ConfigV7.class)), - ROLES(toMap(1, RoleV6.class, 2, RoleV7.class)), - ROLESMAPPING(toMap(1, RoleMappingsV6.class, 2, RoleMappingsV7.class)), - TENANTS(toMap(2, TenantV7.class)), - NODESDN(toMap(1, NodesDn.class, 2, NodesDn.class)), - WHITELIST(toMap(1, WhitelistingSettings.class, 2, WhitelistingSettings.class)), - ALLOWLIST(toMap(1, AllowlistingSettings.class, 2, AllowlistingSettings.class)), - AUDIT(toMap(1, AuditConfig.class, 2, AuditConfig.class)); + ACTIONGROUPS(toMap(0, List.class, 1, ActionGroupsV6.class, 2, ActionGroupsV7.class), "action_groups.yml", false), + ALLOWLIST(toMap(1, AllowlistingSettings.class, 2, AllowlistingSettings.class), "allowlist.yml", true), + AUDIT(toMap(1, AuditConfig.class, 2, AuditConfig.class), "audit.yml", true), + CONFIG(toMap(1, ConfigV6.class, 2, ConfigV7.class), "config.yml", false), + INTERNALUSERS(toMap(1, InternalUserV6.class, 2, InternalUserV7.class), "internal_users.yml", false), + NODESDN(toMap(1, NodesDn.class, 2, NodesDn.class), "nodes_dn.yml", true), + ROLES(toMap(1, RoleV6.class, 2, RoleV7.class), "roles.yml", false), + ROLESMAPPING(toMap(1, RoleMappingsV6.class, 2, RoleMappingsV7.class), "roles_mapping.yml", false), + TENANTS(toMap(2, TenantV7.class), "tenants.yml", false), + WHITELIST(toMap(1, WhitelistingSettings.class, 2, WhitelistingSettings.class), "whitelist.yml", true); + + public static final List REQUIRED_CONFIG_FILES = Arrays.stream(CType.values()) + .filter(Predicate.not(CType::emptyIfMissing)) + .collect(Collectors.toList()); + + public static final List NOT_REQUIRED_CONFIG_FILES = Arrays.stream(CType.values()) + .filter(CType::emptyIfMissing) + .collect(Collectors.toList()); + + private final Map> implementations; + + private final String configFileName; - private Map> implementations; + private final boolean emptyIfMissing; - private CType(Map> implementations) { + private CType(Map> implementations, final String configFileName, final boolean emptyIfMissing) { this.implementations = implementations; + this.configFileName = configFileName; + this.emptyIfMissing = emptyIfMissing; + } + + public boolean emptyIfMissing() { + return emptyIfMissing; } public Map> getImplementationClass() { @@ -80,18 +101,22 @@ public String toLCString() { } public static Set lcStringValues() { - return Arrays.stream(CType.values()).map(c -> c.toLCString()).collect(Collectors.toSet()); + return Arrays.stream(CType.values()).map(CType::toLCString).collect(Collectors.toSet()); } public static Set fromStringValues(String[] strings) { - return Arrays.stream(strings).map(c -> CType.fromString(c)).collect(Collectors.toSet()); + return Arrays.stream(strings).map(CType::fromString).collect(Collectors.toSet()); + } + + public Path configFile(final Path configDir) { + return configDir.resolve(this.configFileName); } private static Map> toMap(Object... objects) { - final Map> map = new HashMap>(); + final ImmutableMap.Builder> map = ImmutableMap.builder(); for (int i = 0; i < objects.length; i = i + 2) { map.put((Integer) objects[i], (Class) objects[i + 1]); } - return Collections.unmodifiableMap(map); + return map.build(); } } diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/AbstractApiActionValidationTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/AbstractApiActionValidationTest.java index f2df09549f..4b2e9e4417 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/AbstractApiActionValidationTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/AbstractApiActionValidationTest.java @@ -116,10 +116,12 @@ protected CType getConfigType() { } - protected JsonNode xContentToJsonNode(final ToXContent toXContent) throws IOException { + protected JsonNode xContentToJsonNode(final ToXContent toXContent) { try (final var xContentBuilder = XContentFactory.jsonBuilder()) { toXContent.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); return DefaultObjectMapper.readTree(xContentBuilder.toString()); + } catch (final IOException ioe) { + throw new RuntimeException(ioe); } } diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/ConfigUpgradeApiActionUnitTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/ConfigUpgradeApiActionUnitTest.java new file mode 100644 index 0000000000..36407cfbc4 --- /dev/null +++ b/src/test/java/org/opensearch/security/dlic/rest/api/ConfigUpgradeApiActionUnitTest.java @@ -0,0 +1,291 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.dlic.rest.api; + +import java.io.IOException; +import java.util.List; +import java.util.function.Consumer; + +import com.google.common.collect.ImmutableList; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.Before; +import org.junit.Test; + +import org.opensearch.action.index.IndexResponse; +import org.opensearch.client.Client; +import org.opensearch.common.action.ActionFuture; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; +import org.opensearch.security.DefaultObjectMapper; +import org.opensearch.security.dlic.rest.support.Utils; +import org.opensearch.security.dlic.rest.validation.ValidationResult; +import org.opensearch.security.securityconf.impl.CType; +import org.opensearch.security.securityconf.impl.SecurityDynamicConfiguration; + +import org.mockito.Mock; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; + +public class ConfigUpgradeApiActionUnitTest extends AbstractApiActionValidationTest { + + @Mock + private Client client; + + @Mock + private RestChannel restChannel; + + @Mock + private RestRequest restRequest; + + private ConfigUpgradeApiAction configUpgradeApiAction; + + @Before + public void setUp() throws IOException { + setupRolesConfiguration(); + doReturn(XContentFactory.jsonBuilder()).when(restChannel).newBuilder(); + + final var actionFuture = mock(ActionFuture.class); + doReturn(mock(IndexResponse.class)).when(actionFuture).actionGet(); + doReturn(actionFuture).when(client).index(any()); + + configUpgradeApiAction = spy(new ConfigUpgradeApiAction(clusterService, threadPool, securityApiDependencies)); + + final var objectMapper = DefaultObjectMapper.objectMapper; + final var config = objectMapper.createObjectNode(); + config.set("_meta", objectMapper.createObjectNode().put("type", CType.ROLES.toLCString()).put("config_version", 2)); + config.set("kibana_read_only", objectMapper.createObjectNode().put("reserved", true)); + final var newRole = objectMapper.createObjectNode(); + newRole.put("reserved", true); + newRole.putArray("cluster_permissions").add("test-permission-1").add("test-permission-2"); + config.set("new_role", newRole); + + doReturn(config).when(configUpgradeApiAction).loadConfigFileAsJson(any()); + } + + @Test + public void testCanUpgrade_ErrorLoadingConfig() throws Exception { + // Setup + doThrow(new IOException("abc")).when(configUpgradeApiAction).loadConfigFileAsJson(any()); + + // Execute + configUpgradeApiAction.canUpgrade(restChannel, restRequest, client); + + // Assert + verify(restChannel).sendResponse(verifyResponseBody(body -> assertThat(body, containsString("see the log file to troubleshoot")))); + } + + @Test + public void testPerformUpgrade_ErrorLoadingConfig() throws Exception { + // Setup + doThrow(new IOException("abc")).when(configUpgradeApiAction).loadConfigFileAsJson(any()); + + // Execute + configUpgradeApiAction.performUpgrade(restChannel, restRequest, client); + + // Assert + verify(restChannel).sendResponse(verifyResponseBody(body -> assertThat(body, containsString("see the log file to troubleshoot")))); + } + + @Test + public void testPerformUpgrade_ErrorApplyConfig() throws Exception { + // Setup + doThrow(new RuntimeException("abc")).when(configUpgradeApiAction).patchEntities(any(), any(), any()); + + // Execute + configUpgradeApiAction.performUpgrade(restChannel, restRequest, client); + + // Assert + verify(restChannel).sendResponse(verifyResponseBody(body -> assertThat(body, containsString("see the log file to troubleshoot")))); + } + + @Test + public void testPerformUpgrade_NoDifferences() throws Exception { + // Setup + final var rolesCopy = rolesConfiguration.deepClone(); + rolesCopy.removeStatic(); // Statics are added by code, not by config files, they should be omitted + final var rolesJsonNode = Utils.convertJsonToJackson(rolesCopy, true); + doReturn(rolesJsonNode).when(configUpgradeApiAction).loadConfigFileAsJson(any()); + + // Execute + configUpgradeApiAction.performUpgrade(restChannel, restRequest, client); + + // Verify + verify(restChannel).sendResponse(verifyResponseBody(body -> assertThat(body, containsString("no differences found")))); + } + + @Test + public void testPerformUpgrade_WithDifferences() throws Exception { + // Execute + configUpgradeApiAction.performUpgrade(restChannel, restRequest, client); + + // Verify + verify(restChannel).sendResponse(argThat(response -> { + final var rawResponseBody = response.content().utf8ToString(); + final var newlineNormalizedBody = rawResponseBody.replace("\r\n", "\n"); + assertThat(newlineNormalizedBody, equalTo("{\n" + // + " \"status\" : \"OK\",\n" + // + " \"upgrades\" : {\n" + // + " \"roles\" : {\n" + // + " \"add\" : [ \"new_role\" ]\n" + // + " }\n" + // + " }\n" + // + "}")); + return true; + })); + } + + @Test + public void testConfigurationDifferences_OperationBash() throws IOException { + final var testCases = new ImmutableList.Builder(); + + testCases.add( + new OperationTestCase("Missing entry", source -> {}, updated -> updated.put("a", "1"), List.of(List.of("add", "/a", "1"))) + ); + + testCases.add( + new OperationTestCase( + "Same object", + source -> source.set("a", objectMapper.createObjectNode()), + updated -> updated.set("a", objectMapper.createObjectNode()), + List.of() + ) + ); + + testCases.add( + new OperationTestCase("Missing object", source -> source.set("a", objectMapper.createObjectNode()), updated -> {}, List.of()) + ); + + testCases.add(new OperationTestCase("Moved and identical object", source -> { + source.set("a", objectMapper.createObjectNode()); + source.set("b", objectMapper.createObjectNode()); + source.set("c", objectMapper.createObjectNode()); + }, updated -> { + updated.set("a", objectMapper.createObjectNode()); + updated.set("c", objectMapper.createObjectNode()); + updated.set("b", objectMapper.createObjectNode()); + }, List.of())); + + testCases.add(new OperationTestCase("Moved and different object", source -> { + source.set("a", objectMapper.createObjectNode()); + source.set("b", objectMapper.createObjectNode()); + source.set("c", objectMapper.createObjectNode()); + }, updated -> { + updated.set("a", objectMapper.createObjectNode()); + updated.set("c", objectMapper.createObjectNode().put("d", "1")); + updated.set("b", objectMapper.createObjectNode()); + }, List.of(List.of("add", "/c/d", "1")))); + + testCases.add(new OperationTestCase("Removed field object", source -> { + source.set("a", objectMapper.createObjectNode().put("hidden", true)); + source.set("b", objectMapper.createObjectNode()); + }, updated -> { + updated.set("a", objectMapper.createObjectNode()); + updated.set("b", objectMapper.createObjectNode()); + }, List.of(List.of("remove", "/a/hidden", "")))); + + testCases.add(new OperationTestCase("Removed field object", source -> { + final var roleA = objectMapper.createObjectNode(); + roleA.putArray("cluster_permissions").add("1").add("2").add("3"); + source.set("a", roleA); + }, updated -> { + final var roleA = objectMapper.createObjectNode(); + roleA.putArray("cluster_permissions").add("2").add("11").add("3").add("44"); + updated.set("a", roleA); + }, + List.of( + List.of("remove", "/a/cluster_permissions/0", ""), + List.of("add", "/a/cluster_permissions/1", "11"), + List.of("add", "/a/cluster_permissions/3", "44") + ) + )); + + for (final var tc : testCases.build()) { + // Setup + final var source = objectMapper.createObjectNode(); + source.set("_meta", objectMapper.createObjectNode().put("type", CType.ROLES.toLCString()).put("config_version", 2)); + tc.sourceChanges.accept(source); + final var updated = objectMapper.createObjectNode(); + tc.updates.accept(updated); + + var sourceAsConfig = SecurityDynamicConfiguration.fromJson(objectMapper.writeValueAsString(source), CType.ROLES, 2, 1, 1); + + doReturn(ValidationResult.success(sourceAsConfig)).when(configUpgradeApiAction) + .loadConfiguration(any(), anyBoolean(), anyBoolean()); + doReturn(updated).when(configUpgradeApiAction).loadConfigFileAsJson(any()); + + // Execute + var result = configUpgradeApiAction.computeDifferenceToUpdate(CType.ACTIONGROUPS); + + // Verify + result.valid(differences -> { + assertThat(differences.v1(), equalTo(CType.ACTIONGROUPS)); + assertThat(tc.name + ": Number of operations", differences.v2().size(), equalTo(tc.expectedResults.size())); + final var expectedResultsIterator = tc.expectedResults.iterator(); + differences.v2().forEach(operation -> { + final List expected = expectedResultsIterator.next(); + assertThat( + tc.name + ": Operation type" + operation.toPrettyString(), + operation.get("op").asText(), + equalTo(expected.get(0)) + ); + assertThat(tc.name + ": Path" + operation.toPrettyString(), operation.get("path").asText(), equalTo(expected.get(1))); + assertThat( + tc.name + ": Value " + operation.toPrettyString(), + operation.has("value") ? operation.get("value").asText("") : "", + equalTo(expected.get(2)) + ); + }); + }); + } + } + + static class OperationTestCase { + final String name; + final Consumer sourceChanges; + final Consumer updates; + final List> expectedResults; + + OperationTestCase( + final String name, + final Consumer sourceChanges, + final Consumer updates, + final List> expectedResults + ) { + this.name = name; + this.sourceChanges = sourceChanges; + this.updates = updates; + this.expectedResults = expectedResults; + } + + } + + private RestResponse verifyResponseBody(final Consumer test) { + return argThat(response -> { + final String content = response.content().utf8ToString(); + test.accept(content); + return true; + }); + } + +} diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/InternalUsersApiActionValidationTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/InternalUsersApiActionValidationTest.java index 773d356246..2af598f5d5 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/InternalUsersApiActionValidationTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/InternalUsersApiActionValidationTest.java @@ -22,6 +22,7 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.RestRequest; import org.opensearch.security.DefaultObjectMapper; +import org.opensearch.security.dlic.rest.validation.ValidationResult; import org.opensearch.security.securityconf.impl.CType; import org.opensearch.security.securityconf.impl.SecurityDynamicConfiguration; import org.opensearch.security.securityconf.impl.v7.InternalUserV7; @@ -32,9 +33,13 @@ import org.mockito.Mock; import org.mockito.Mockito; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.Mockito.when; public class InternalUsersApiActionValidationTest extends AbstractApiActionValidationTest { @@ -145,19 +150,26 @@ public void validateSecurityRolesWithMutableRolesMappingConfig() throws Exceptio // should ok to set regular role with mutable role mapping var userJson = objectMapper.createObjectNode().set("opendistro_security_roles", objectMapper.createArrayNode().add("regular_role")); var result = internalUsersApiAction.validateSecurityRoles(SecurityConfiguration.of(userJson, "some_user", configuration)); - assertTrue(result.isValid()); + assertValidationResultIsValid(result); // should be ok to set reserved role with mutable role mapping userJson = objectMapper.createObjectNode().set("opendistro_security_roles", objectMapper.createArrayNode().add("kibana_read_only")); result = internalUsersApiAction.validateSecurityRoles(SecurityConfiguration.of(userJson, "some_user", configuration)); - assertTrue(result.isValid()); + assertValidationResultIsValid(result); // should be ok to set static role with mutable role mapping userJson = objectMapper.createObjectNode().set("opendistro_security_roles", objectMapper.createArrayNode().add("all_access")); result = internalUsersApiAction.validateSecurityRoles(SecurityConfiguration.of(userJson, "some_user", configuration)); - assertTrue(result.isValid()); + assertValidationResultIsValid(result); // should not be ok to set hidden role with mutable role mapping userJson = objectMapper.createObjectNode().set("opendistro_security_roles", objectMapper.createArrayNode().add("some_hidden_role")); result = internalUsersApiAction.validateSecurityRoles(SecurityConfiguration.of(userJson, "some_user", configuration)); - assertFalse(result.isValid()); + final var errorMessage = xContentToJsonNode(result.errorMessage()).toPrettyString(); + assertThat(errorMessage, allOf(containsString("NOT_FOUND"), containsString("Resource 'some_hidden_role' is not available."))); + } + + void assertValidationResultIsValid(final ValidationResult result) { + if (!result.isValid()) { + fail("Expected valid result, error message: " + xContentToJsonNode(result.errorMessage()).toPrettyString()); + } } @Test From dd119e59dc157d19f2c5836b3af055d931f566e2 Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Wed, 20 Mar 2024 13:16:51 -0400 Subject: [PATCH 149/204] 2.13 Release notes (#4140) Signed-off-by: Stephen Crawford --- ...nsearch-security.release-notes-2.13.0.0.md | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 release-notes/opensearch-security.release-notes-2.13.0.0.md diff --git a/release-notes/opensearch-security.release-notes-2.13.0.0.md b/release-notes/opensearch-security.release-notes-2.13.0.0.md new file mode 100644 index 0000000000..42ccd8fff0 --- /dev/null +++ b/release-notes/opensearch-security.release-notes-2.13.0.0.md @@ -0,0 +1,30 @@ +## 2024-03-19 Version 2.13.0.0 + +Compatible with OpenSearch 2.13.0 + +### Enhancements +* Admin role for Query insights plugin ([#4022](https://github.com/opensearch-project/security/pull/4022)) +* Add query assistant role and new ml system indices ([#4143](https://github.com/opensearch-project/security/pull/4143)) +* Redact sensitive configuration values when retrieving security configuration ([#4028](https://github.com/opensearch-project/security/pull/4028)) +* v2.12 update roles.yml with new API for experimental alerting plugin feature ([#4035](https://github.com/opensearch-project/security/pull/4035)) +* Add deprecate message that TLSv1 and TLSv1.1 support will be removed in the next major version ([#4083](https://github.com/opensearch-project/security/pull/4083)) +* Log password requirement details in demo environment ([#4082](https://github.com/opensearch-project/security/pull/4082)) +* Redact sensitive URL parameters from audit logging ([#4070](https://github.com/opensearch-project/security/pull/4070)) +* Fix unconsumed parameter exception when authenticating with jwtUrlParameter ([#4065](https://github.com/opensearch-project/security/pull/4065)) +* Regenerates root-ca, kirk and esnode certificates to address already expired root ca certificate ([#4066](https://github.com/opensearch-project/security/pull/4066)) +* Add exclude_roles configuration parameter to LDAP authorization backend ([#4043](https://github.com/opensearch-project/security/pull/4043)) + +### Maintenance +* Add exlusion for logback-core to resolve CVE-2023-6378 ([#4050](https://github.com/opensearch-project/security/pull/4050)) +* Bump com.netflix.nebula.ospackage from 11.7.0 to 11.8.1 ([#4041](https://github.com/opensearch-project/security/pull/4041), [#4075](https://github.com/opensearch-project/security/pull/4075)) +* Bump Wandalen/wretry.action from 1.3.0 to 1.4.10 ([#4042](https://github.com/opensearch-project/security/pull/4042), [#4092](https://github.com/opensearch-project/security/pull/4092), [#4108](https://github.com/opensearch-project/security/pull/4108), [#4135](https://github.com/opensearch-project/security/pull/4135)) +* Bump spring_version from 5.3.31 to 5.3.33 ([#4058](https://github.com/opensearch-project/security/pull/4058), [#4131](https://github.com/opensearch-project/security/pull/4131)) +* Bump org.scala-lang:scala-library from 2.13.12 to 2.13.13 ([#4076](https://github.com/opensearch-project/security/pull/4076)) +* Bump com.google.googlejavaformat:google-java-format from 1.19.1 to 1.21.0 ([#4078](https://github.com/opensearch-project/security/pull/4078), [#4110](https://github.com/opensearch-project/security/pull/4110)) +* Bump ch.qos.logback:logback-classic from 1.2.13 to 1.5.3 ([#4091](https://github.com/opensearch-project/security/pull/4091), [#4111](https://github.com/opensearch-project/security/pull/4111)) +* Bump com.fasterxml.woodstox:woodstox-core from 6.6.0 to 6.6.1 ([#4093](https://github.com/opensearch-project/security/pull/4093)) +* Bump kafka_version from 3.5.1 to 3.7.0 ([#4095](https://github.com/opensearch-project/security/pull/4095)) +* Bump jakarta.xml.bind:jakarta.xml.bind-api from 4.0.1 to 4.0.2 ([#4109](https://github.com/opensearch-project/security/pull/4109)) +* Bump org.apache.zookeeper:zookeeper from 3.9.1. to 3.9.2 ([#4130](https://github.com/opensearch-project/security/pull/4130)) +* Bump org.awaitility:awaitility from 4.2.0 to 4.2.1 ([#4133](https://github.com/opensearch-project/security/pull/4133)) +* Bump com.google.errorprone:error_prone_annotations from 2.25.0 to 2.26.1 ([#4132](https://github.com/opensearch-project/security/pull/4132)) From 6f79d094b80983f36b1b52d4703093f2583ec61e Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Fri, 22 Mar 2024 09:33:57 -0400 Subject: [PATCH 150/204] Update Log4JSink Default from sgaudit to audit and add test for default values (#4146) Signed-off-by: Craig Perkins --- .../org/opensearch/security/auditlog/sink/Log4JSink.java | 2 +- .../opensearch/security/auditlog/sink/SinkProviderTest.java | 6 ++++++ .../auditlog/endpoints/sink/configuration_all_variants.yml | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/security/auditlog/sink/Log4JSink.java b/src/main/java/org/opensearch/security/auditlog/sink/Log4JSink.java index f01043fa21..cf535e48b1 100644 --- a/src/main/java/org/opensearch/security/auditlog/sink/Log4JSink.java +++ b/src/main/java/org/opensearch/security/auditlog/sink/Log4JSink.java @@ -27,7 +27,7 @@ public final class Log4JSink extends AuditLogSink { public Log4JSink(final String name, final Settings settings, final String settingsPrefix, AuditLogSink fallbackSink) { super(name, settings, settingsPrefix, fallbackSink); - loggerName = settings.get(settingsPrefix + ".log4j.logger_name", "sgaudit"); + loggerName = settings.get(settingsPrefix + ".log4j.logger_name", "audit"); auditLogger = LogManager.getLogger(loggerName); logLevel = Level.toLevel(settings.get(settingsPrefix + ".log4j.level", "INFO").toUpperCase()); enabled = auditLogger.isEnabled(logLevel); diff --git a/src/test/java/org/opensearch/security/auditlog/sink/SinkProviderTest.java b/src/test/java/org/opensearch/security/auditlog/sink/SinkProviderTest.java index 5e3203261f..af8204a5c7 100644 --- a/src/test/java/org/opensearch/security/auditlog/sink/SinkProviderTest.java +++ b/src/test/java/org/opensearch/security/auditlog/sink/SinkProviderTest.java @@ -88,6 +88,12 @@ public void testConfiguration() throws Exception { Assert.assertEquals("loggername", lsink.loggerName); Assert.assertEquals(Level.DEBUG, lsink.logLevel); + sink = provider.getSink("endpoint13"); + Assert.assertEquals(Log4JSink.class, sink.getClass()); + lsink = (Log4JSink) sink; + Assert.assertEquals("audit", lsink.loggerName); + Assert.assertEquals(Level.INFO, lsink.logLevel); + } @Test diff --git a/src/test/resources/auditlog/endpoints/sink/configuration_all_variants.yml b/src/test/resources/auditlog/endpoints/sink/configuration_all_variants.yml index f1c8620e88..82565ee3ec 100644 --- a/src/test/resources/auditlog/endpoints/sink/configuration_all_variants.yml +++ b/src/test/resources/auditlog/endpoints/sink/configuration_all_variants.yml @@ -45,3 +45,5 @@ plugins.security: config: log4j.logger_name: loggername log4j.level: invalid + endpoint13: + type: log4j From 03db12b9dc1b9c81272e34b1e78ecd0d0ad925d9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 16:15:34 +0000 Subject: [PATCH 151/204] Bump net.shibboleth.utilities:java-support from 8.4.0 to 8.4.1 (#4163) Bumps net.shibboleth.utilities:java-support from 8.4.0 to 8.4.1. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=net.shibboleth.utilities:java-support&package-manager=gradle&previous-version=8.4.0&new-version=8.4.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 7fcd272c32..f74d51d371 100644 --- a/build.gradle +++ b/build.gradle @@ -617,7 +617,7 @@ dependencies { testImplementation 'org.apache.camel:camel-xmlsecurity:3.22.1' //OpenSAML - implementation 'net.shibboleth.utilities:java-support:8.4.0' + implementation 'net.shibboleth.utilities:java-support:8.4.1' implementation "com.onelogin:java-saml:${one_login_java_saml}" implementation "com.onelogin:java-saml-core:${one_login_java_saml}" implementation "org.opensaml:opensaml-core:${open_saml_version}" From f2e9b3ef357cdd4b1572c7e7f23a731ba476bff7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 16:16:07 +0000 Subject: [PATCH 152/204] Bump open_saml_version from 4.3.0 to 4.3.1 (#4161) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [//]: # (dependabot-start) ⚠️ **Dependabot is rebasing this PR** ⚠️ Rebasing might not happen immediately, so don't worry if this takes some time. Note: if you make any changes to this PR yourself, they will take precedence over the rebase. --- [//]: # (dependabot-end) Bumps `open_saml_version` from 4.3.0 to 4.3.1. Updates `org.opensaml:opensaml-core` from 4.3.0 to 4.3.1 Updates `org.opensaml:opensaml-security-impl` from 4.3.0 to 4.3.1 Updates `org.opensaml:opensaml-security-api` from 4.3.0 to 4.3.1 Updates `org.opensaml:opensaml-xmlsec-api` from 4.3.0 to 4.3.1 Updates `org.opensaml:opensaml-xmlsec-impl` from 4.3.0 to 4.3.1 Updates `org.opensaml:opensaml-saml-api` from 4.3.0 to 4.3.1 Updates `org.opensaml:opensaml-saml-impl` from 4.3.0 to 4.3.1 Updates `org.opensaml:opensaml-messaging-api` from 4.3.0 to 4.3.1 Updates `org.opensaml:opensaml-profile-api` from 4.3.0 to 4.3.1 Updates `org.opensaml:opensaml-soap-api` from 4.3.0 to 4.3.1 Updates `org.opensaml:opensaml-soap-impl` from 4.3.0 to 4.3.1 Updates `org.opensaml:opensaml-storage-api` from 4.3.0 to 4.3.1 Updates `org.opensaml:opensaml-messaging-impl` from 4.3.0 to 4.3.1 Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index f74d51d371..97b4409c13 100644 --- a/build.gradle +++ b/build.gradle @@ -27,7 +27,7 @@ buildscript { common_utils_version = System.getProperty("common_utils.version", '3.0.0.0-SNAPSHOT') kafka_version = '3.7.0' apache_cxf_version = '4.0.4' - open_saml_version = '4.3.0' + open_saml_version = '4.3.1' one_login_java_saml = '2.9.0' jjwt_version = '0.12.5' guava_version = '32.1.3-jre' From bb1b90044955547c1addeec41602ee39c247b55d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 16:16:42 +0000 Subject: [PATCH 153/204] Bump net.minidev:accessors-smart from 2.5.0 to 2.5.1 (#4162) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [net.minidev:accessors-smart](https://github.com/netplex/json-smart-v2) from 2.5.0 to 2.5.1.
Release notes

Sourced from net.minidev:accessors-smart's releases.

V 2.5.1

What's Changed

New Contributors

Full Changelog: https://github.com/netplex/json-smart-v2/compare/2.5.0...2.5.1

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=net.minidev:accessors-smart&package-manager=gradle&previous-version=2.5.0&new-version=2.5.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 97b4409c13..187f0483bf 100644 --- a/build.gradle +++ b/build.gradle @@ -599,7 +599,7 @@ dependencies { implementation "org.apache.kafka:kafka-clients:${kafka_version}" - runtimeOnly 'net.minidev:accessors-smart:2.5.0' + runtimeOnly 'net.minidev:accessors-smart:2.5.1' runtimeOnly "org.apache.cxf:cxf-core:${apache_cxf_version}" implementation "org.apache.cxf:cxf-rt-rs-json-basic:${apache_cxf_version}" From c64d0aa66654e43886377c648811d07d0840af9b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 16:19:19 +0000 Subject: [PATCH 154/204] Bump org.ow2.asm:asm from 9.6 to 9.7 (#4164) Bumps org.ow2.asm:asm from 9.6 to 9.7. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.ow2.asm:asm&package-manager=gradle&previous-version=9.6&new-version=9.7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 187f0483bf..ea392a8eac 100644 --- a/build.gradle +++ b/build.gradle @@ -612,7 +612,7 @@ dependencies { compileOnly 'com.google.errorprone:error_prone_annotations:2.26.1' runtimeOnly 'com.sun.istack:istack-commons-runtime:4.2.0' runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.2' - runtimeOnly 'org.ow2.asm:asm:9.6' + runtimeOnly 'org.ow2.asm:asm:9.7' testImplementation 'org.apache.camel:camel-xmlsecurity:3.22.1' From a731e62f8a52420645473c8279081ae46bfb86b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 18:06:33 +0100 Subject: [PATCH 155/204] Bump Wandalen/wretry.action from 1.4.10 to 2.1.0 (#4165) Bumps [Wandalen/wretry.action](https://github.com/wandalen/wretry.action) from 1.4.10 to 2.1.0.
Commits
  • f062299 version 2.1.0
  • 1900814 Merge pull request #144 from dmvict/master
  • 1903bde Extend action, add option github_token to work with private actions
  • 4ca71ac version 2.0.0
  • aa38070 Merge pull request #142 from dmvict/master
  • 86e0a3a Improve Readme.md, fix some mistakes, add comments
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Wandalen/wretry.action&package-manager=github_actions&previous-version=1.4.10&new-version=2.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0595106ce7..68e966f56e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,7 +81,7 @@ jobs: working-directory: downloaded-artifacts - name: Upload Coverage with retry - uses: Wandalen/wretry.action@v1.4.10 + uses: Wandalen/wretry.action@v2.1.0 with: attempt_limit: 5 attempt_delay: 2000 From e2a06f001154d558c4c717c2a2bc54b9f57d1654 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 26 Mar 2024 15:24:51 -0400 Subject: [PATCH 156/204] Switch to built-in security transports from core (#4119) ### Description The security plugin does not need to provide the secure transports anymore but SecureSettingsFactory so the core transport modules will be able to configure those. ### Issues Resolved Closes https://github.com/opensearch-project/security/issues/4118 Is this a backport? If so, please add backport PR # and/or commits # ### Testing [Please provide details of testing done: unit testing, integration testing and manual testing] ### Check List - [ ] New functionality includes testing - [ ] New functionality has been documented - [ ] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: Andriy Redko --- .../security/OpenSearchSecurityPlugin.java | 47 ++- .../security/filter/SecurityRestFilter.java | 8 +- ...rt.java => NonSslHttpServerTransport.java} | 8 +- ...rt.java => SecureHttpServerTransport.java} | 35 +- .../ssl/OpenSearchSecureSettingsFactory.java | 74 +++++ .../ssl/OpenSearchSecuritySSLPlugin.java | 126 +++++-- .../netty/Netty4ConditionalDecompressor.java | 4 +- .../Netty4HttpRequestHeaderVerifier.java | 10 +- .../SecuritySSLNettyHttpServerTransport.java | 168 ---------- .../ssl/transport/DualModeSSLHandler.java | 89 ----- .../transport/SecuritySSLNettyTransport.java | 308 ------------------ .../ssl/OpenSearchSecuritySSLPluginTest.java | 246 ++++++++++++++ .../transport/DualModeSSLHandlerTests.java | 120 ------- .../SecuritySSLNettyTransportTests.java | 201 ------------ 14 files changed, 487 insertions(+), 957 deletions(-) rename src/main/java/org/opensearch/security/http/{SecurityNonSslHttpServerTransport.java => NonSslHttpServerTransport.java} (90%) rename src/main/java/org/opensearch/security/http/{SecurityHttpServerTransport.java => SecureHttpServerTransport.java} (74%) create mode 100644 src/main/java/org/opensearch/security/ssl/OpenSearchSecureSettingsFactory.java delete mode 100644 src/main/java/org/opensearch/security/ssl/http/netty/SecuritySSLNettyHttpServerTransport.java delete mode 100644 src/main/java/org/opensearch/security/ssl/transport/DualModeSSLHandler.java delete mode 100644 src/main/java/org/opensearch/security/ssl/transport/SecuritySSLNettyTransport.java create mode 100644 src/test/java/org/opensearch/security/ssl/OpenSearchSecuritySSLPluginTest.java delete mode 100644 src/test/java/org/opensearch/security/ssl/transport/DualModeSSLHandlerTests.java delete mode 100644 src/test/java/org/opensearch/security/ssl/transport/SecuritySSLNettyTransportTests.java diff --git a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java index 688b797e85..ac32da1d1b 100644 --- a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java +++ b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java @@ -46,6 +46,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; @@ -114,6 +115,8 @@ import org.opensearch.plugins.ExtensionAwarePlugin; import org.opensearch.plugins.IdentityPlugin; import org.opensearch.plugins.MapperPlugin; +import org.opensearch.plugins.SecureSettingsFactory; +import org.opensearch.plugins.SecureTransportSettingsProvider; import org.opensearch.repositories.RepositoriesService; import org.opensearch.rest.RestController; import org.opensearch.rest.RestHandler; @@ -150,8 +153,8 @@ import org.opensearch.security.dlic.rest.validation.PasswordValidator; import org.opensearch.security.filter.SecurityFilter; import org.opensearch.security.filter.SecurityRestFilter; -import org.opensearch.security.http.SecurityHttpServerTransport; -import org.opensearch.security.http.SecurityNonSslHttpServerTransport; +import org.opensearch.security.http.NonSslHttpServerTransport; +import org.opensearch.security.http.SecureHttpServerTransport; import org.opensearch.security.http.XFFResolver; import org.opensearch.security.identity.SecurityTokenManager; import org.opensearch.security.privileges.PrivilegesEvaluator; @@ -167,11 +170,11 @@ import org.opensearch.security.securityconf.DynamicConfigFactory; import org.opensearch.security.setting.OpensearchDynamicSetting; import org.opensearch.security.setting.TransportPassiveAuthSetting; +import org.opensearch.security.ssl.OpenSearchSecureSettingsFactory; import org.opensearch.security.ssl.OpenSearchSecuritySSLPlugin; import org.opensearch.security.ssl.SslExceptionHandler; import org.opensearch.security.ssl.http.netty.ValidatingDispatcher; import org.opensearch.security.ssl.transport.DefaultPrincipalExtractor; -import org.opensearch.security.ssl.transport.SecuritySSLNettyTransport; import org.opensearch.security.ssl.util.SSLConfigConstants; import org.opensearch.security.support.ConfigConstants; import org.opensearch.security.support.GuardedSearchOperationWrapper; @@ -199,6 +202,7 @@ import org.opensearch.transport.TransportRequestOptions; import org.opensearch.transport.TransportResponseHandler; import org.opensearch.transport.TransportService; +import org.opensearch.transport.netty4.ssl.SecureNetty4Transport; import org.opensearch.watcher.ResourceWatcherService; import static org.opensearch.security.dlic.rest.api.RestApiAdminPrivilegesEvaluator.ENDPOINTS_WITH_PERMISSIONS; @@ -858,25 +862,27 @@ public void sendRequest( } @Override - public Map> getTransports( + public Map> getSecureTransports( Settings settings, ThreadPool threadPool, PageCacheRecycler pageCacheRecycler, CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry, NetworkService networkService, + SecureTransportSettingsProvider secureTransportSettingsProvider, Tracer tracer ) { Map> transports = new HashMap>(); if (SSLConfig.isSslOnlyMode()) { - return super.getTransports( + return super.getSecureTransports( settings, threadPool, pageCacheRecycler, circuitBreakerService, namedWriteableRegistry, networkService, + secureTransportSettingsProvider, tracer ); } @@ -884,18 +890,16 @@ public Map> getTransports( if (transportSSLEnabled) { transports.put( "org.opensearch.security.ssl.http.netty.SecuritySSLNettyTransport", - () -> new SecuritySSLNettyTransport( - settings, + () -> new SecureNetty4Transport( + migrateSettings(settings), Version.CURRENT, threadPool, networkService, pageCacheRecycler, namedWriteableRegistry, circuitBreakerService, - sks, - evaluateSslExceptionHandler(), sharedGroupFactory, - SSLConfig, + secureTransportSettingsProvider, tracer ) ); @@ -904,7 +908,7 @@ public Map> getTransports( } @Override - public Map> getHttpTransports( + public Map> getSecureHttpTransports( Settings settings, ThreadPool threadPool, BigArrays bigArrays, @@ -914,11 +918,12 @@ public Map> getHttpTransports( NetworkService networkService, Dispatcher dispatcher, ClusterSettings clusterSettings, + SecureTransportSettingsProvider secureTransportSettingsProvider, Tracer tracer ) { if (SSLConfig.isSslOnlyMode()) { - return super.getHttpTransports( + return super.getSecureHttpTransports( settings, threadPool, bigArrays, @@ -928,6 +933,7 @@ public Map> getHttpTransports( networkService, dispatcher, clusterSettings, + secureTransportSettingsProvider, tracer ); } @@ -943,17 +949,16 @@ public Map> getHttpTransports( evaluateSslExceptionHandler() ); // TODO close odshst - final SecurityHttpServerTransport odshst = new SecurityHttpServerTransport( - settings, + final SecureHttpServerTransport odshst = new SecureHttpServerTransport( + migrateSettings(settings), networkService, bigArrays, threadPool, - sks, - evaluateSslExceptionHandler(), xContentRegistry, validatingDispatcher, clusterSettings, sharedGroupFactory, + secureTransportSettingsProvider, tracer, securityRestHandler ); @@ -962,8 +967,8 @@ public Map> getHttpTransports( } else if (!client) { return Collections.singletonMap( "org.opensearch.security.http.SecurityHttpServerTransport", - () -> new SecurityNonSslHttpServerTransport( - settings, + () -> new NonSslHttpServerTransport( + migrateSettings(settings), networkService, bigArrays, threadPool, @@ -971,6 +976,7 @@ public Map> getHttpTransports( dispatcher, clusterSettings, sharedGroupFactory, + secureTransportSettingsProvider, tracer, securityRestHandler ) @@ -2005,6 +2011,11 @@ public SecurityTokenManager getTokenManager() { return tokenManager; } + @Override + public Optional getSecureSettingFactory(Settings settings) { + return Optional.of(new OpenSearchSecureSettingsFactory(settings, sks, sslExceptionHandler)); + } + public static class GuiceHolder implements LifecycleComponent { private static RepositoriesService repositoriesService; diff --git a/src/main/java/org/opensearch/security/filter/SecurityRestFilter.java b/src/main/java/org/opensearch/security/filter/SecurityRestFilter.java index b649b8d71f..8ccaa3041c 100644 --- a/src/main/java/org/opensearch/security/filter/SecurityRestFilter.java +++ b/src/main/java/org/opensearch/security/filter/SecurityRestFilter.java @@ -69,10 +69,10 @@ import static org.opensearch.security.OpenSearchSecurityPlugin.LEGACY_OPENDISTRO_PREFIX; import static org.opensearch.security.OpenSearchSecurityPlugin.PLUGINS_PREFIX; -import static org.opensearch.security.http.SecurityHttpServerTransport.CONTEXT_TO_RESTORE; -import static org.opensearch.security.http.SecurityHttpServerTransport.EARLY_RESPONSE; -import static org.opensearch.security.http.SecurityHttpServerTransport.IS_AUTHENTICATED; -import static org.opensearch.security.http.SecurityHttpServerTransport.UNCONSUMED_PARAMS; +import static org.opensearch.security.http.SecureHttpServerTransport.CONTEXT_TO_RESTORE; +import static org.opensearch.security.http.SecureHttpServerTransport.EARLY_RESPONSE; +import static org.opensearch.security.http.SecureHttpServerTransport.IS_AUTHENTICATED; +import static org.opensearch.security.http.SecureHttpServerTransport.UNCONSUMED_PARAMS; public class SecurityRestFilter { diff --git a/src/main/java/org/opensearch/security/http/SecurityNonSslHttpServerTransport.java b/src/main/java/org/opensearch/security/http/NonSslHttpServerTransport.java similarity index 90% rename from src/main/java/org/opensearch/security/http/SecurityNonSslHttpServerTransport.java rename to src/main/java/org/opensearch/security/http/NonSslHttpServerTransport.java index f37ebb48e8..c97d872aca 100644 --- a/src/main/java/org/opensearch/security/http/SecurityNonSslHttpServerTransport.java +++ b/src/main/java/org/opensearch/security/http/NonSslHttpServerTransport.java @@ -33,6 +33,8 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.http.HttpHandlingSettings; import org.opensearch.http.netty4.Netty4HttpServerTransport; +import org.opensearch.http.netty4.ssl.SecureNetty4HttpServerTransport; +import org.opensearch.plugins.SecureTransportSettingsProvider; import org.opensearch.security.filter.SecurityRestFilter; import org.opensearch.security.ssl.http.netty.Netty4ConditionalDecompressor; import org.opensearch.security.ssl.http.netty.Netty4HttpRequestHeaderVerifier; @@ -44,11 +46,11 @@ import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelInboundHandlerAdapter; -public class SecurityNonSslHttpServerTransport extends Netty4HttpServerTransport { +public class NonSslHttpServerTransport extends SecureNetty4HttpServerTransport { private final ChannelInboundHandlerAdapter headerVerifier; - public SecurityNonSslHttpServerTransport( + public NonSslHttpServerTransport( final Settings settings, final NetworkService networkService, final BigArrays bigArrays, @@ -57,6 +59,7 @@ public SecurityNonSslHttpServerTransport( final Dispatcher dispatcher, final ClusterSettings clusterSettings, final SharedGroupFactory sharedGroupFactory, + final SecureTransportSettingsProvider secureTransportSettingsProvider, final Tracer tracer, final SecurityRestFilter restFilter ) { @@ -69,6 +72,7 @@ public SecurityNonSslHttpServerTransport( dispatcher, clusterSettings, sharedGroupFactory, + secureTransportSettingsProvider, tracer ); headerVerifier = new Netty4HttpRequestHeaderVerifier(restFilter, threadPool, settings); diff --git a/src/main/java/org/opensearch/security/http/SecurityHttpServerTransport.java b/src/main/java/org/opensearch/security/http/SecureHttpServerTransport.java similarity index 74% rename from src/main/java/org/opensearch/security/http/SecurityHttpServerTransport.java rename to src/main/java/org/opensearch/security/http/SecureHttpServerTransport.java index eb75f898f4..170f39ffd6 100644 --- a/src/main/java/org/opensearch/security/http/SecurityHttpServerTransport.java +++ b/src/main/java/org/opensearch/security/http/SecureHttpServerTransport.java @@ -34,19 +34,21 @@ import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.http.netty4.ssl.SecureNetty4HttpServerTransport; +import org.opensearch.plugins.SecureTransportSettingsProvider; import org.opensearch.security.filter.SecurityResponse; import org.opensearch.security.filter.SecurityRestFilter; -import org.opensearch.security.ssl.SecurityKeyStore; -import org.opensearch.security.ssl.SslExceptionHandler; -import org.opensearch.security.ssl.http.netty.SecuritySSLNettyHttpServerTransport; +import org.opensearch.security.ssl.http.netty.Netty4ConditionalDecompressor; +import org.opensearch.security.ssl.http.netty.Netty4HttpRequestHeaderVerifier; import org.opensearch.security.ssl.http.netty.ValidatingDispatcher; import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.SharedGroupFactory; +import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.util.AttributeKey; -public class SecurityHttpServerTransport extends SecuritySSLNettyHttpServerTransport { +public class SecureHttpServerTransport extends SecureNetty4HttpServerTransport { public static final AttributeKey EARLY_RESPONSE = AttributeKey.newInstance("opensearch-http-early-response"); public static final AttributeKey> UNCONSUMED_PARAMS = AttributeKey.newInstance("opensearch-http-request-consumed-params"); @@ -56,17 +58,18 @@ public class SecurityHttpServerTransport extends SecuritySSLNettyHttpServerTrans public static final AttributeKey SHOULD_DECOMPRESS = AttributeKey.newInstance("opensearch-http-should-decompress"); public static final AttributeKey IS_AUTHENTICATED = AttributeKey.newInstance("opensearch-http-is-authenticated"); - public SecurityHttpServerTransport( + private final ChannelInboundHandlerAdapter headerVerifier; + + public SecureHttpServerTransport( final Settings settings, final NetworkService networkService, final BigArrays bigArrays, final ThreadPool threadPool, - final SecurityKeyStore odsks, - final SslExceptionHandler sslExceptionHandler, final NamedXContentRegistry namedXContentRegistry, final ValidatingDispatcher dispatcher, final ClusterSettings clusterSettings, SharedGroupFactory sharedGroupFactory, + final SecureTransportSettingsProvider secureTransportSettingsProvider, Tracer tracer, SecurityRestFilter restFilter ) { @@ -75,14 +78,24 @@ public SecurityHttpServerTransport( networkService, bigArrays, threadPool, - odsks, namedXContentRegistry, dispatcher, - sslExceptionHandler, clusterSettings, sharedGroupFactory, - tracer, - restFilter + secureTransportSettingsProvider, + tracer ); + + headerVerifier = new Netty4HttpRequestHeaderVerifier(restFilter, threadPool, settings); + } + + @Override + protected ChannelInboundHandlerAdapter createHeaderVerifier() { + return headerVerifier; + } + + @Override + protected ChannelInboundHandlerAdapter createDecompressor() { + return new Netty4ConditionalDecompressor(); } } diff --git a/src/main/java/org/opensearch/security/ssl/OpenSearchSecureSettingsFactory.java b/src/main/java/org/opensearch/security/ssl/OpenSearchSecureSettingsFactory.java new file mode 100644 index 0000000000..d85f490d0c --- /dev/null +++ b/src/main/java/org/opensearch/security/ssl/OpenSearchSecureSettingsFactory.java @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.ssl; + +import java.util.Optional; +import javax.net.ssl.SSLEngine; +import javax.net.ssl.SSLException; + +import org.opensearch.common.settings.Settings; +import org.opensearch.http.HttpServerTransport; +import org.opensearch.plugins.SecureSettingsFactory; +import org.opensearch.plugins.SecureTransportSettingsProvider; +import org.opensearch.transport.TcpTransport; + +public class OpenSearchSecureSettingsFactory implements SecureSettingsFactory { + private final Settings settings; + private final SecurityKeyStore sks; + private final SslExceptionHandler sslExceptionHandler; + + public OpenSearchSecureSettingsFactory(Settings settings, SecurityKeyStore sks, SslExceptionHandler sslExceptionHandler) { + this.settings = settings; + this.sks = sks; + this.sslExceptionHandler = sslExceptionHandler; + } + + @Override + public Optional getSecureTransportSettingsProvider(Settings settings) { + return Optional.of(new SecureTransportSettingsProvider() { + @Override + public Optional buildHttpServerExceptionHandler(Settings settings, HttpServerTransport transport) { + return Optional.of(new ServerExceptionHandler() { + @Override + public void onError(Throwable t) { + sslExceptionHandler.logError(t, true); + } + }); + } + + @Override + public Optional buildServerTransportExceptionHandler(Settings settings, TcpTransport transport) { + return Optional.of(new ServerExceptionHandler() { + @Override + public void onError(Throwable t) { + sslExceptionHandler.logError(t, false); + } + }); + } + + @Override + public Optional buildSecureHttpServerEngine(Settings settings, HttpServerTransport transport) throws SSLException { + return Optional.of(sks.createHTTPSSLEngine()); + } + + @Override + public Optional buildSecureServerTransportEngine(Settings settings, TcpTransport transport) throws SSLException { + return Optional.of(sks.createServerTransportSSLEngine()); + } + + @Override + public Optional buildSecureClientTransportEngine(Settings settings, String hostname, int port) throws SSLException { + return Optional.of(sks.createClientTransportSSLEngine(hostname, port)); + } + }); + } +} diff --git a/src/main/java/org/opensearch/security/ssl/OpenSearchSecuritySSLPlugin.java b/src/main/java/org/opensearch/security/ssl/OpenSearchSecuritySSLPlugin.java index e6e4e85b33..3acbce21cf 100644 --- a/src/main/java/org/opensearch/security/ssl/OpenSearchSecuritySSLPlugin.java +++ b/src/main/java/org/opensearch/security/ssl/OpenSearchSecuritySSLPlugin.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.function.Function; import java.util.function.Supplier; @@ -62,6 +63,8 @@ import org.opensearch.http.HttpServerTransport.Dispatcher; import org.opensearch.plugins.NetworkPlugin; import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.SecureSettingsFactory; +import org.opensearch.plugins.SecureTransportSettingsProvider; import org.opensearch.plugins.SystemIndexPlugin; import org.opensearch.repositories.RepositoriesService; import org.opensearch.rest.RestController; @@ -70,20 +73,21 @@ import org.opensearch.security.DefaultObjectMapper; import org.opensearch.security.NonValidatingObjectMapper; import org.opensearch.security.filter.SecurityRestFilter; -import org.opensearch.security.ssl.http.netty.SecuritySSLNettyHttpServerTransport; +import org.opensearch.security.http.SecureHttpServerTransport; import org.opensearch.security.ssl.http.netty.ValidatingDispatcher; import org.opensearch.security.ssl.rest.SecuritySSLInfoAction; import org.opensearch.security.ssl.transport.DefaultPrincipalExtractor; import org.opensearch.security.ssl.transport.PrincipalExtractor; import org.opensearch.security.ssl.transport.SSLConfig; -import org.opensearch.security.ssl.transport.SecuritySSLNettyTransport; import org.opensearch.security.ssl.transport.SecuritySSLTransportInterceptor; import org.opensearch.security.ssl.util.SSLConfigConstants; +import org.opensearch.security.support.SecuritySettings; import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.SharedGroupFactory; import org.opensearch.transport.Transport; import org.opensearch.transport.TransportInterceptor; +import org.opensearch.transport.netty4.ssl.SecureNetty4Transport; import org.opensearch.watcher.ResourceWatcherService; import io.netty.handler.ssl.OpenSsl; @@ -91,6 +95,21 @@ //For ES5 this class has only effect when SSL only plugin is installed public class OpenSearchSecuritySSLPlugin extends Plugin implements SystemIndexPlugin, NetworkPlugin { + private static final Setting SECURITY_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION = Setting.boolSetting( + SSLConfigConstants.SECURITY_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION, + true, + Property.NodeScope, + Property.Filtered, + Property.Deprecated + ); + + private static final Setting SECURITY_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION_RESOLVE_HOST_NAME = Setting.boolSetting( + SSLConfigConstants.SECURITY_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION_RESOLVE_HOST_NAME, + true, + Property.NodeScope, + Property.Filtered, + Property.Deprecated + ); private static boolean USE_NETTY_DEFAULT_ALLOCATOR = Booleans.parseBoolean( System.getProperty("opensearch.unsafe.use_netty_default_allocator"), @@ -237,7 +256,7 @@ public Object run() { } @Override - public Map> getHttpTransports( + public Map> getSecureHttpTransports( Settings settings, ThreadPool threadPool, BigArrays bigArrays, @@ -247,6 +266,7 @@ public Map> getHttpTransports( NetworkService networkService, Dispatcher dispatcher, ClusterSettings clusterSettings, + SecureTransportSettingsProvider secureTransportSettingsProvider, Tracer tracer ) { @@ -259,17 +279,16 @@ public Map> getHttpTransports( configPath, NOOP_SSL_EXCEPTION_HANDLER ); - final SecuritySSLNettyHttpServerTransport sgsnht = new SecuritySSLNettyHttpServerTransport( - settings, + final SecureHttpServerTransport sgsnht = new SecureHttpServerTransport( + migrateSettings(settings), networkService, bigArrays, threadPool, - sks, xContentRegistry, validatingDispatcher, - NOOP_SSL_EXCEPTION_HANDLER, clusterSettings, sharedGroupFactory, + secureTransportSettingsProvider, tracer, securityRestHandler ); @@ -313,13 +332,14 @@ public List getTransportInterceptors(NamedWriteableRegistr } @Override - public Map> getTransports( + public Map> getSecureTransports( Settings settings, ThreadPool threadPool, PageCacheRecycler pageCacheRecycler, CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry, NetworkService networkService, + SecureTransportSettingsProvider secureTransportSettingsProvider, Tracer tracer ) { @@ -327,18 +347,16 @@ public Map> getTransports( if (transportSSLEnabled) { transports.put( "org.opensearch.security.ssl.http.netty.SecuritySSLNettyTransport", - () -> new SecuritySSLNettyTransport( - settings, + () -> new SecureNetty4Transport( + migrateSettings(settings), Version.CURRENT, threadPool, networkService, pageCacheRecycler, namedWriteableRegistry, circuitBreakerService, - sks, - NOOP_SSL_EXCEPTION_HANDLER, sharedGroupFactory, - SSLConfig, + secureTransportSettingsProvider, tracer ) ); @@ -436,22 +454,13 @@ public List> getSettings() { Property.Filtered ) ); - settings.add( - Setting.boolSetting( - SSLConfigConstants.SECURITY_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION, - true, - Property.NodeScope, - Property.Filtered - ) - ); - settings.add( - Setting.boolSetting( - SSLConfigConstants.SECURITY_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION_RESOLVE_HOST_NAME, - true, - Property.NodeScope, - Property.Filtered - ) - ); + if (!settings.stream().anyMatch(s -> s.getKey().equalsIgnoreCase(NetworkModule.TRANSPORT_SSL_ENFORCE_HOSTNAME_VERIFICATION_KEY))) { + settings.add(SECURITY_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION); + } + if (!settings.stream() + .anyMatch(s -> s.getKey().equalsIgnoreCase(NetworkModule.TRANSPORT_SSL_ENFORCE_HOSTNAME_VERIFICATION_RESOLVE_HOST_NAME_KEY))) { + settings.add(SECURITY_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION_RESOLVE_HOST_NAME); + } settings.add( Setting.simpleString(SSLConfigConstants.SECURITY_SSL_TRANSPORT_KEYSTORE_FILEPATH, Property.NodeScope, Property.Filtered) ); @@ -664,4 +673,63 @@ public List getSettingsFilter() { settingsFilter.add("plugins.security.*"); return settingsFilter; } + + @Override + public Optional getSecureSettingFactory(Settings settings) { + return Optional.of(new OpenSearchSecureSettingsFactory(settings, sks, NOOP_SSL_EXCEPTION_HANDLER)); + } + + protected Settings migrateSettings(Settings settings) { + final Settings.Builder builder = Settings.builder().put(settings); + + if (!NetworkModule.TRANSPORT_SSL_DUAL_MODE_ENABLED.exists(settings)) { + builder.put(NetworkModule.TRANSPORT_SSL_DUAL_MODE_ENABLED_KEY, SecuritySettings.SSL_DUAL_MODE_SETTING.get(settings)); + } else { + if (SecuritySettings.SSL_DUAL_MODE_SETTING.exists(settings)) { + throw new OpenSearchException( + "Only one of the settings [" + + NetworkModule.TRANSPORT_SSL_DUAL_MODE_ENABLED_KEY + + ", " + + SecuritySettings.SSL_DUAL_MODE_SETTING.getKey() + + " (deprecated)] could be specified but not both" + ); + } + } + + if (!NetworkModule.TRANSPORT_SSL_ENFORCE_HOSTNAME_VERIFICATION_RESOLVE_HOST_NAME.exists(settings)) { + builder.put( + NetworkModule.TRANSPORT_SSL_ENFORCE_HOSTNAME_VERIFICATION_RESOLVE_HOST_NAME_KEY, + SECURITY_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION_RESOLVE_HOST_NAME.get(settings) + ); + } else { + if (SECURITY_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION_RESOLVE_HOST_NAME.exists(settings)) { + throw new OpenSearchException( + "Only one of the settings [" + + NetworkModule.TRANSPORT_SSL_ENFORCE_HOSTNAME_VERIFICATION_RESOLVE_HOST_NAME_KEY + + ", " + + SECURITY_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION_RESOLVE_HOST_NAME.getKey() + + " (deprecated)] could be specified but not both" + ); + } + } + + if (!NetworkModule.TRANSPORT_SSL_ENFORCE_HOSTNAME_VERIFICATION.exists(settings)) { + builder.put( + NetworkModule.TRANSPORT_SSL_ENFORCE_HOSTNAME_VERIFICATION_KEY, + SECURITY_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION.get(settings) + ); + } else { + if (SECURITY_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION.exists(settings)) { + throw new OpenSearchException( + "Only one of the settings [" + + NetworkModule.TRANSPORT_SSL_ENFORCE_HOSTNAME_VERIFICATION_KEY + + ", " + + SECURITY_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION.getKey() + + " (deprecated)] could be specified but not both" + ); + } + } + + return builder.build(); + } } diff --git a/src/main/java/org/opensearch/security/ssl/http/netty/Netty4ConditionalDecompressor.java b/src/main/java/org/opensearch/security/ssl/http/netty/Netty4ConditionalDecompressor.java index f133d997f9..8b2d4bb1d2 100644 --- a/src/main/java/org/opensearch/security/ssl/http/netty/Netty4ConditionalDecompressor.java +++ b/src/main/java/org/opensearch/security/ssl/http/netty/Netty4ConditionalDecompressor.java @@ -13,8 +13,8 @@ import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.http.HttpContentDecompressor; -import static org.opensearch.security.http.SecurityHttpServerTransport.EARLY_RESPONSE; -import static org.opensearch.security.http.SecurityHttpServerTransport.SHOULD_DECOMPRESS; +import static org.opensearch.security.http.SecureHttpServerTransport.EARLY_RESPONSE; +import static org.opensearch.security.http.SecureHttpServerTransport.SHOULD_DECOMPRESS; public class Netty4ConditionalDecompressor extends HttpContentDecompressor { diff --git a/src/main/java/org/opensearch/security/ssl/http/netty/Netty4HttpRequestHeaderVerifier.java b/src/main/java/org/opensearch/security/ssl/http/netty/Netty4HttpRequestHeaderVerifier.java index e6dec0c213..052f1961e2 100644 --- a/src/main/java/org/opensearch/security/ssl/http/netty/Netty4HttpRequestHeaderVerifier.java +++ b/src/main/java/org/opensearch/security/ssl/http/netty/Netty4HttpRequestHeaderVerifier.java @@ -32,11 +32,11 @@ import io.netty.handler.codec.http.HttpRequest; import io.netty.util.ReferenceCountUtil; -import static org.opensearch.security.http.SecurityHttpServerTransport.CONTEXT_TO_RESTORE; -import static org.opensearch.security.http.SecurityHttpServerTransport.EARLY_RESPONSE; -import static org.opensearch.security.http.SecurityHttpServerTransport.IS_AUTHENTICATED; -import static org.opensearch.security.http.SecurityHttpServerTransport.SHOULD_DECOMPRESS; -import static org.opensearch.security.http.SecurityHttpServerTransport.UNCONSUMED_PARAMS; +import static org.opensearch.security.http.SecureHttpServerTransport.CONTEXT_TO_RESTORE; +import static org.opensearch.security.http.SecureHttpServerTransport.EARLY_RESPONSE; +import static org.opensearch.security.http.SecureHttpServerTransport.IS_AUTHENTICATED; +import static org.opensearch.security.http.SecureHttpServerTransport.SHOULD_DECOMPRESS; +import static org.opensearch.security.http.SecureHttpServerTransport.UNCONSUMED_PARAMS; @Sharable public class Netty4HttpRequestHeaderVerifier extends SimpleChannelInboundHandler { diff --git a/src/main/java/org/opensearch/security/ssl/http/netty/SecuritySSLNettyHttpServerTransport.java b/src/main/java/org/opensearch/security/ssl/http/netty/SecuritySSLNettyHttpServerTransport.java deleted file mode 100644 index fc2f31b2b0..0000000000 --- a/src/main/java/org/opensearch/security/ssl/http/netty/SecuritySSLNettyHttpServerTransport.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright 2015-2017 floragunn GmbH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.opensearch.security.ssl.http.netty; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.opensearch.common.network.NetworkService; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.BigArrays; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.http.HttpChannel; -import org.opensearch.http.HttpHandlingSettings; -import org.opensearch.http.netty4.Netty4HttpChannel; -import org.opensearch.http.netty4.Netty4HttpServerTransport; -import org.opensearch.security.filter.SecurityRestFilter; -import org.opensearch.security.ssl.SecurityKeyStore; -import org.opensearch.security.ssl.SslExceptionHandler; -import org.opensearch.telemetry.tracing.Tracer; -import org.opensearch.threadpool.ThreadPool; -import org.opensearch.transport.SharedGroupFactory; - -import io.netty.channel.Channel; -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.handler.codec.DecoderException; -import io.netty.handler.ssl.ApplicationProtocolNames; -import io.netty.handler.ssl.ApplicationProtocolNegotiationHandler; -import io.netty.handler.ssl.SslHandler; - -public class SecuritySSLNettyHttpServerTransport extends Netty4HttpServerTransport { - private static final Logger logger = LogManager.getLogger(SecuritySSLNettyHttpServerTransport.class); - private final SecurityKeyStore sks; - private final SslExceptionHandler errorHandler; - private final ChannelInboundHandlerAdapter headerVerifier; - - public SecuritySSLNettyHttpServerTransport( - final Settings settings, - final NetworkService networkService, - final BigArrays bigArrays, - final ThreadPool threadPool, - final SecurityKeyStore sks, - final NamedXContentRegistry namedXContentRegistry, - final ValidatingDispatcher dispatcher, - final SslExceptionHandler errorHandler, - ClusterSettings clusterSettings, - SharedGroupFactory sharedGroupFactory, - Tracer tracer, - SecurityRestFilter restFilter - ) { - super( - settings, - networkService, - bigArrays, - threadPool, - namedXContentRegistry, - dispatcher, - clusterSettings, - sharedGroupFactory, - tracer - ); - this.sks = sks; - this.errorHandler = errorHandler; - headerVerifier = new Netty4HttpRequestHeaderVerifier(restFilter, threadPool, settings); - } - - @Override - public ChannelHandler configureServerChannelHandler() { - return new SSLHttpChannelHandler(this, handlingSettings, sks); - } - - @Override - public void onException(HttpChannel channel, Exception cause0) { - Throwable cause = cause0; - - if (cause0 instanceof DecoderException && cause0 != null) { - cause = cause0.getCause(); - } - - errorHandler.logError(cause, true); - logger.error("Exception during establishing a SSL connection: " + cause, cause); - - super.onException(channel, cause0); - } - - protected class SSLHttpChannelHandler extends Netty4HttpServerTransport.HttpChannelHandler { - /** - * Application negotiation handler to select either HTTP 1.1 or HTTP 2 protocol, based - * on client/server ALPN negotiations. - */ - private class Http2OrHttpHandler extends ApplicationProtocolNegotiationHandler { - protected Http2OrHttpHandler() { - super(ApplicationProtocolNames.HTTP_1_1); - } - - @Override - protected void configurePipeline(ChannelHandlerContext ctx, String protocol) throws Exception { - if (ApplicationProtocolNames.HTTP_2.equals(protocol)) { - configureDefaultHttp2Pipeline(ctx.pipeline()); - } else if (ApplicationProtocolNames.HTTP_1_1.equals(protocol)) { - configureDefaultHttpPipeline(ctx.pipeline()); - } else { - throw new IllegalStateException("Unknown application protocol: " + protocol); - } - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - super.exceptionCaught(ctx, cause); - Netty4HttpChannel channel = ctx.channel().attr(HTTP_CHANNEL_KEY).get(); - if (channel != null) { - if (cause instanceof Error) { - onException(channel, new Exception(cause)); - } else { - onException(channel, (Exception) cause); - } - } - } - } - - protected SSLHttpChannelHandler( - Netty4HttpServerTransport transport, - final HttpHandlingSettings handlingSettings, - final SecurityKeyStore odsks - ) { - super(transport, handlingSettings); - } - - @Override - protected void initChannel(Channel ch) throws Exception { - super.initChannel(ch); - final SslHandler sslHandler = new SslHandler(SecuritySSLNettyHttpServerTransport.this.sks.createHTTPSSLEngine()); - ch.pipeline().addFirst("ssl_http", sslHandler); - } - - @Override - protected void configurePipeline(Channel ch) { - ch.pipeline().addLast(new Http2OrHttpHandler()); - } - } - - @Override - protected ChannelInboundHandlerAdapter createHeaderVerifier() { - return headerVerifier; - } - - @Override - protected ChannelInboundHandlerAdapter createDecompressor() { - return new Netty4ConditionalDecompressor(); - } -} diff --git a/src/main/java/org/opensearch/security/ssl/transport/DualModeSSLHandler.java b/src/main/java/org/opensearch/security/ssl/transport/DualModeSSLHandler.java deleted file mode 100644 index a7961f864b..0000000000 --- a/src/main/java/org/opensearch/security/ssl/transport/DualModeSSLHandler.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ -package org.opensearch.security.ssl.transport; - -import java.nio.charset.StandardCharsets; -import java.util.List; -import javax.net.ssl.SSLException; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.opensearch.security.ssl.SecurityKeyStore; -import org.opensearch.security.ssl.util.SSLConnectionTestUtil; -import org.opensearch.security.ssl.util.TLSUtil; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelPipeline; -import io.netty.handler.codec.ByteToMessageDecoder; -import io.netty.handler.ssl.SslHandler; - -/** - * Modifies the current pipeline dynamically to enable TLS - */ -public class DualModeSSLHandler extends ByteToMessageDecoder { - - private static final Logger logger = LogManager.getLogger(DualModeSSLHandler.class); - private final SecurityKeyStore securityKeyStore; - - private final SslHandler providedSSLHandler; - - public DualModeSSLHandler(SecurityKeyStore securityKeyStore) { - this(securityKeyStore, null); - } - - @VisibleForTesting - protected DualModeSSLHandler(SecurityKeyStore securityKeyStore, SslHandler providedSSLHandler) { - this.securityKeyStore = securityKeyStore; - this.providedSSLHandler = providedSSLHandler; - } - - @Override - protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) throws Exception { - // Will use the first six bytes to detect a protocol. - if (in.readableBytes() < 6) { - return; - } - int offset = in.readerIndex(); - if (in.getCharSequence(offset, 6, StandardCharsets.UTF_8).equals(SSLConnectionTestUtil.DUAL_MODE_CLIENT_HELLO_MSG)) { - logger.debug("Received DualSSL Client Hello message"); - ByteBuf responseBuffer = Unpooled.buffer(6); - responseBuffer.writeCharSequence(SSLConnectionTestUtil.DUAL_MODE_SERVER_HELLO_MSG, StandardCharsets.UTF_8); - ctx.writeAndFlush(responseBuffer).addListener(ChannelFutureListener.CLOSE); - return; - } - - if (TLSUtil.isTLS(in)) { - logger.debug("Identified request as SSL request"); - enableSsl(ctx); - } else { - logger.debug("Identified request as non SSL request, running in HTTP mode as dual mode is enabled"); - ctx.pipeline().remove(this); - } - } - - private void enableSsl(ChannelHandlerContext ctx) throws SSLException { - SslHandler sslHandler; - if (providedSSLHandler != null) { - sslHandler = providedSSLHandler; - } else { - sslHandler = new SslHandler(securityKeyStore.createServerTransportSSLEngine()); - } - ChannelPipeline p = ctx.pipeline(); - p.addAfter("port_unification_handler", "ssl_server", sslHandler); - p.remove(this); - logger.debug("Removed port unification handler and added SSL handler as incoming request is SSL"); - } -} diff --git a/src/main/java/org/opensearch/security/ssl/transport/SecuritySSLNettyTransport.java b/src/main/java/org/opensearch/security/ssl/transport/SecuritySSLNettyTransport.java deleted file mode 100644 index 5be3424528..0000000000 --- a/src/main/java/org/opensearch/security/ssl/transport/SecuritySSLNettyTransport.java +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Copyright 2015-2017 floragunn GmbH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.security.ssl.transport; - -import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.security.AccessController; -import java.security.PrivilegedAction; -import javax.net.ssl.SSLEngine; -import javax.net.ssl.SSLException; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.opensearch.ExceptionsHelper; -import org.opensearch.OpenSearchSecurityException; -import org.opensearch.Version; -import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.network.NetworkService; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.PageCacheRecycler; -import org.opensearch.core.common.io.stream.NamedWriteableRegistry; -import org.opensearch.core.indices.breaker.CircuitBreakerService; -import org.opensearch.security.ssl.SecurityKeyStore; -import org.opensearch.security.ssl.SslExceptionHandler; -import org.opensearch.security.ssl.util.SSLConfigConstants; -import org.opensearch.security.ssl.util.SSLConnectionTestResult; -import org.opensearch.security.ssl.util.SSLConnectionTestUtil; -import org.opensearch.telemetry.tracing.Tracer; -import org.opensearch.threadpool.ThreadPool; -import org.opensearch.transport.SharedGroupFactory; -import org.opensearch.transport.TcpChannel; -import org.opensearch.transport.netty4.Netty4Transport; - -import io.netty.channel.Channel; -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelOutboundHandlerAdapter; -import io.netty.channel.ChannelPromise; -import io.netty.handler.codec.DecoderException; -import io.netty.handler.ssl.SslHandler; - -public class SecuritySSLNettyTransport extends Netty4Transport { - - private static final Logger logger = LogManager.getLogger(SecuritySSLNettyTransport.class); - private final SecurityKeyStore ossks; - private final SslExceptionHandler errorHandler; - private final SSLConfig SSLConfig; - - public SecuritySSLNettyTransport( - final Settings settings, - final Version version, - final ThreadPool threadPool, - final NetworkService networkService, - final PageCacheRecycler pageCacheRecycler, - final NamedWriteableRegistry namedWriteableRegistry, - final CircuitBreakerService circuitBreakerService, - final SecurityKeyStore ossks, - final SslExceptionHandler errorHandler, - SharedGroupFactory sharedGroupFactory, - final SSLConfig SSLConfig, - final Tracer tracer - ) { - super( - settings, - version, - threadPool, - networkService, - pageCacheRecycler, - namedWriteableRegistry, - circuitBreakerService, - sharedGroupFactory, - tracer - ); - - this.ossks = ossks; - this.errorHandler = errorHandler; - this.SSLConfig = SSLConfig; - } - - // This allows for testing log messages - Logger getLogger() { - return logger; - } - - @Override - public void onException(TcpChannel channel, Exception e) { - - Throwable cause = e; - - if (e instanceof DecoderException && e != null) { - cause = e.getCause(); - } - - errorHandler.logError(cause, false); - getLogger().error("Exception during establishing a SSL connection: " + cause, cause); - - if (channel == null || !channel.isOpen()) { - throw new OpenSearchSecurityException("The provided TCP channel is invalid.", e); - } - super.onException(channel, e); - } - - @Override - protected ChannelHandler getServerChannelInitializer(String name) { - return new SSLServerChannelInitializer(name); - } - - @Override - protected ChannelHandler getClientChannelInitializer(DiscoveryNode node) { - return new SSLClientChannelInitializer(node); - } - - protected class SSLServerChannelInitializer extends Netty4Transport.ServerChannelInitializer { - - public SSLServerChannelInitializer(String name) { - super(name); - } - - @Override - protected void initChannel(Channel ch) throws Exception { - super.initChannel(ch); - - boolean dualModeEnabled = SSLConfig.isDualModeEnabled(); - if (dualModeEnabled) { - logger.info("SSL Dual mode enabled, using port unification handler"); - final ChannelHandler portUnificationHandler = new DualModeSSLHandler(ossks); - ch.pipeline().addFirst("port_unification_handler", portUnificationHandler); - } else { - final SslHandler sslHandler = new SslHandler(ossks.createServerTransportSSLEngine()); - ch.pipeline().addFirst("ssl_server", sslHandler); - } - } - - @Override - public final void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - if (cause instanceof DecoderException && cause != null) { - cause = cause.getCause(); - } - - errorHandler.logError(cause, false); - getLogger().error("Exception during establishing a SSL connection: " + cause, cause); - - super.exceptionCaught(ctx, cause); - } - } - - protected static class ClientSSLHandler extends ChannelOutboundHandlerAdapter { - private final Logger log = LogManager.getLogger(this.getClass()); - private final SecurityKeyStore sks; - private final boolean hostnameVerificationEnabled; - private final boolean hostnameVerificationResovleHostName; - private final SslExceptionHandler errorHandler; - - private ClientSSLHandler( - final SecurityKeyStore sks, - final boolean hostnameVerificationEnabled, - final boolean hostnameVerificationResovleHostName, - final SslExceptionHandler errorHandler - ) { - this.sks = sks; - this.hostnameVerificationEnabled = hostnameVerificationEnabled; - this.hostnameVerificationResovleHostName = hostnameVerificationResovleHostName; - this.errorHandler = errorHandler; - } - - @Override - public final void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - if (cause instanceof DecoderException && cause != null) { - cause = cause.getCause(); - } - - errorHandler.logError(cause, false); - logger.error("Exception during establishing a SSL connection: " + cause, cause); - - super.exceptionCaught(ctx, cause); - } - - @Override - public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, SocketAddress localAddress, ChannelPromise promise) - throws Exception { - SSLEngine engine = null; - try { - if (hostnameVerificationEnabled) { - final InetSocketAddress inetSocketAddress = (InetSocketAddress) remoteAddress; - String hostname = null; - if (hostnameVerificationResovleHostName) { - hostname = inetSocketAddress.getHostName(); - } else { - hostname = inetSocketAddress.getHostString(); - } - - if (log.isDebugEnabled()) { - log.debug( - "Hostname of peer is {} ({}/{}) with hostnameVerificationResovleHostName: {}", - hostname, - inetSocketAddress.getHostName(), - inetSocketAddress.getHostString(), - hostnameVerificationResovleHostName - ); - } - - engine = sks.createClientTransportSSLEngine(hostname, inetSocketAddress.getPort()); - } else { - engine = sks.createClientTransportSSLEngine(null, -1); - } - } catch (final SSLException e) { - throw ExceptionsHelper.convertToOpenSearchException(e); - } - final SslHandler sslHandler = new SslHandler(engine); - ctx.pipeline().replace(this, "ssl_client", sslHandler); - super.connect(ctx, remoteAddress, localAddress, promise); - } - } - - protected class SSLClientChannelInitializer extends Netty4Transport.ClientChannelInitializer { - private final boolean hostnameVerificationEnabled; - private final boolean hostnameVerificationResovleHostName; - private final DiscoveryNode node; - private SSLConnectionTestResult connectionTestResult; - - @SuppressWarnings("removal") - public SSLClientChannelInitializer(DiscoveryNode node) { - this.node = node; - hostnameVerificationEnabled = settings.getAsBoolean( - SSLConfigConstants.SECURITY_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION, - true - ); - hostnameVerificationResovleHostName = settings.getAsBoolean( - SSLConfigConstants.SECURITY_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION_RESOLVE_HOST_NAME, - true - ); - - connectionTestResult = SSLConnectionTestResult.SSL_AVAILABLE; - if (SSLConfig.isDualModeEnabled()) { - SSLConnectionTestUtil sslConnectionTestUtil = new SSLConnectionTestUtil( - node.getAddress().getAddress(), - node.getAddress().getPort() - ); - connectionTestResult = AccessController.doPrivileged( - (PrivilegedAction) sslConnectionTestUtil::testConnection - ); - } - } - - @Override - protected void initChannel(Channel ch) throws Exception { - super.initChannel(ch); - - if (connectionTestResult == SSLConnectionTestResult.OPENSEARCH_PING_FAILED) { - logger.error( - "SSL dual mode is enabled but dual mode handshake and OpenSearch ping has failed during client connection setup, closing channel" - ); - ch.close(); - return; - } - - if (connectionTestResult == SSLConnectionTestResult.SSL_AVAILABLE) { - logger.debug("Connection to {} needs to be ssl, adding ssl handler to the client channel ", node.getHostName()); - ch.pipeline() - .addFirst( - "client_ssl_handler", - new ClientSSLHandler(ossks, hostnameVerificationEnabled, hostnameVerificationResovleHostName, errorHandler) - ); - } else { - logger.debug("Connection to {} needs to be non ssl", node.getHostName()); - } - } - - @Override - public final void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - if (cause instanceof DecoderException && cause != null) { - cause = cause.getCause(); - } - - errorHandler.logError(cause, false); - getLogger().error("Exception during establishing a SSL connection: " + cause, cause); - - super.exceptionCaught(ctx, cause); - } - } -} diff --git a/src/test/java/org/opensearch/security/ssl/OpenSearchSecuritySSLPluginTest.java b/src/test/java/org/opensearch/security/ssl/OpenSearchSecuritySSLPluginTest.java new file mode 100644 index 0000000000..03488fe17c --- /dev/null +++ b/src/test/java/org/opensearch/security/ssl/OpenSearchSecuritySSLPluginTest.java @@ -0,0 +1,246 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.security.ssl; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; +import javax.net.ssl.SSLEngine; +import javax.net.ssl.SSLException; + +import org.junit.Before; +import org.junit.Test; + +import org.opensearch.OpenSearchException; +import org.opensearch.common.collect.Tuple; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.http.HttpServerTransport; +import org.opensearch.plugins.SecureTransportSettingsProvider; +import org.opensearch.security.ssl.util.SSLConfigConstants; +import org.opensearch.security.support.SecuritySettings; +import org.opensearch.security.test.AbstractSecurityUnitTest; +import org.opensearch.security.test.helper.file.FileHelper; +import org.opensearch.telemetry.tracing.noop.NoopTracer; +import org.opensearch.transport.TcpTransport; +import org.opensearch.transport.Transport; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.collection.IsMapContaining.hasKey; +import static org.junit.Assert.assertThrows; + +public class OpenSearchSecuritySSLPluginTest extends AbstractSecurityUnitTest { + private Settings settings; + private SecureTransportSettingsProvider secureTransportSettingsProvider; + private ClusterSettings clusterSettings; + + @Before + public void setUp() { + settings = Settings.builder() + .put( + SSLConfigConstants.SECURITY_SSL_TRANSPORT_KEYSTORE_FILEPATH, + FileHelper.getAbsoluteFilePathFromClassPath("ssl/kirk-keystore.jks") + ) + .put( + SSLConfigConstants.SECURITY_SSL_HTTP_PEMTRUSTEDCAS_FILEPATH, + FileHelper.getAbsoluteFilePathFromClassPath("ssl/root-ca.pem") + ) + .put( + SSLConfigConstants.SECURITY_SSL_TRANSPORT_TRUSTSTORE_FILEPATH, + FileHelper.getAbsoluteFilePathFromClassPath("ssl/truststore.jks") + ) + .put( + SSLConfigConstants.SECURITY_SSL_TRANSPORT_KEYSTORE_FILEPATH, + FileHelper.getAbsoluteFilePathFromClassPath("ssl/kirk-keystore.jks") + ) + .put( + SSLConfigConstants.SECURITY_SSL_HTTP_KEYSTORE_FILEPATH, + FileHelper.getAbsoluteFilePathFromClassPath("ssl/node-0-keystore.jks") + ) + .put(SSLConfigConstants.SECURITY_SSL_HTTP_ENABLED, true) + .put(OpenSearchSecuritySSLPlugin.CLIENT_TYPE, "node") + .build(); + + secureTransportSettingsProvider = new SecureTransportSettingsProvider() { + @Override + public Optional buildHttpServerExceptionHandler(Settings settings, HttpServerTransport transport) { + return Optional.empty(); + } + + @Override + public Optional buildServerTransportExceptionHandler(Settings settings, TcpTransport transport) { + return Optional.empty(); + } + + @Override + public Optional buildSecureHttpServerEngine(Settings settings, HttpServerTransport transport) throws SSLException { + return Optional.empty(); + } + + @Override + public Optional buildSecureServerTransportEngine(Settings settings, TcpTransport transport) throws SSLException { + return Optional.empty(); + } + + @Override + public Optional buildSecureClientTransportEngine(Settings settings, String hostname, int port) throws SSLException { + return Optional.empty(); + } + }; + + clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + } + + @Test + public void testRegisterSecureHttpTransport() throws IOException { + try (OpenSearchSecuritySSLPlugin plugin = new OpenSearchSecuritySSLPlugin(settings, null, false)) { + final Map> transports = plugin.getSecureHttpTransports( + settings, + MOCK_POOL, + null, + null, + null, + null, + null, + null, + clusterSettings, + secureTransportSettingsProvider, + NoopTracer.INSTANCE + ); + assertThat(transports, hasKey("org.opensearch.security.ssl.http.netty.SecuritySSLNettyHttpServerTransport")); + } + } + + @Test + public void testRegisterSecureTransport() throws IOException { + try (OpenSearchSecuritySSLPlugin plugin = new OpenSearchSecuritySSLPlugin(settings, null, false)) { + final Map> transports = plugin.getSecureTransports( + settings, + MOCK_POOL, + null, + null, + null, + null, + secureTransportSettingsProvider, + NoopTracer.INSTANCE + ); + assertThat(transports, hasKey("org.opensearch.security.ssl.http.netty.SecuritySSLNettyTransport")); + } + } + + @Test + public void testRegisterSecureTransportWithDeprecatedSecuirtyPluginSettings() throws IOException { + final Settings deprecated = Settings.builder() + .put(settings) + .put(SecuritySettings.SSL_DUAL_MODE_SETTING.getKey(), true) + .put(SSLConfigConstants.SECURITY_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION_RESOLVE_HOST_NAME, false) + .put(SSLConfigConstants.SECURITY_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION, false) + .build(); + + try (OpenSearchSecuritySSLPlugin plugin = new OpenSearchSecuritySSLPlugin(deprecated, null, false)) { + final Map> transports = plugin.getSecureTransports( + deprecated, + MOCK_POOL, + null, + null, + null, + null, + secureTransportSettingsProvider, + NoopTracer.INSTANCE + ); + assertThat(transports, hasKey("org.opensearch.security.ssl.http.netty.SecuritySSLNettyTransport")); + assertThat(transports.get("org.opensearch.security.ssl.http.netty.SecuritySSLNettyTransport").get(), not(nullValue())); + } + } + + @Test + public void testRegisterSecureTransportWithNetworkModuleSettings() throws IOException { + final Settings migrated = Settings.builder() + .put(settings) + .put(NetworkModule.TRANSPORT_SSL_DUAL_MODE_ENABLED_KEY, true) + .put(NetworkModule.TRANSPORT_SSL_ENFORCE_HOSTNAME_VERIFICATION_RESOLVE_HOST_NAME_KEY, false) + .put(NetworkModule.TRANSPORT_SSL_ENFORCE_HOSTNAME_VERIFICATION_KEY, false) + .build(); + + try (OpenSearchSecuritySSLPlugin plugin = new OpenSearchSecuritySSLPlugin(migrated, null, false)) { + final Map> transports = plugin.getSecureTransports( + migrated, + MOCK_POOL, + null, + null, + null, + null, + secureTransportSettingsProvider, + NoopTracer.INSTANCE + ); + assertThat(transports, hasKey("org.opensearch.security.ssl.http.netty.SecuritySSLNettyTransport")); + assertThat(transports.get("org.opensearch.security.ssl.http.netty.SecuritySSLNettyTransport").get(), not(nullValue())); + } + } + + @Test + public void testRegisterSecureTransportWithDuplicateSettings() throws IOException { + final Collection> duplicates = List.of( + Tuple.tuple(SecuritySettings.SSL_DUAL_MODE_SETTING.getKey(), NetworkModule.TRANSPORT_SSL_DUAL_MODE_ENABLED_KEY), + Tuple.tuple( + SSLConfigConstants.SECURITY_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION_RESOLVE_HOST_NAME, + NetworkModule.TRANSPORT_SSL_ENFORCE_HOSTNAME_VERIFICATION_RESOLVE_HOST_NAME_KEY + ), + Tuple.tuple( + SSLConfigConstants.SECURITY_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION, + NetworkModule.TRANSPORT_SSL_ENFORCE_HOSTNAME_VERIFICATION_KEY + ) + ); + + for (final Tuple duplicate : duplicates) { + final Settings migrated = Settings.builder() + .put(settings) + .put(duplicate.v1(), true) + .put(NetworkModule.TRANSPORT_SSL_DUAL_MODE_ENABLED_KEY, true) + .put(NetworkModule.TRANSPORT_SSL_ENFORCE_HOSTNAME_VERIFICATION_RESOLVE_HOST_NAME_KEY, false) + .put(NetworkModule.TRANSPORT_SSL_ENFORCE_HOSTNAME_VERIFICATION_KEY, false) + .build(); + + try (OpenSearchSecuritySSLPlugin plugin = new OpenSearchSecuritySSLPlugin(migrated, null, false)) { + final Map> transports = plugin.getSecureTransports( + migrated, + MOCK_POOL, + null, + null, + null, + null, + secureTransportSettingsProvider, + NoopTracer.INSTANCE + ); + assertThat(transports, hasKey("org.opensearch.security.ssl.http.netty.SecuritySSLNettyTransport")); + final OpenSearchException ex = assertThrows( + OpenSearchException.class, + transports.get("org.opensearch.security.ssl.http.netty.SecuritySSLNettyTransport")::get + ); + assertThat( + ex.getMessage(), + containsString( + "Only one of the settings [" + + duplicate.v2() + + ", " + + duplicate.v1() + + " (deprecated)] could be specified but not both" + ) + ); + } + } + } +} diff --git a/src/test/java/org/opensearch/security/ssl/transport/DualModeSSLHandlerTests.java b/src/test/java/org/opensearch/security/ssl/transport/DualModeSSLHandlerTests.java deleted file mode 100644 index e71e77d414..0000000000 --- a/src/test/java/org/opensearch/security/ssl/transport/DualModeSSLHandlerTests.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ -package org.opensearch.security.ssl.transport; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import org.opensearch.security.ssl.SecurityKeyStore; -import org.opensearch.security.ssl.util.SSLConnectionTestUtil; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufAllocator; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelPipeline; -import io.netty.handler.ssl.SslHandler; -import org.mockito.ArgumentCaptor; -import org.mockito.Mockito; - -import static org.opensearch.transport.NettyAllocator.getAllocator; - -public class DualModeSSLHandlerTests { - - public static final int TLS_MAJOR_VERSION = 3; - public static final int TLS_MINOR_VERSION = 0; - private static final ByteBufAllocator ALLOCATOR = getAllocator(); - - private SecurityKeyStore securityKeyStore; - private ChannelPipeline pipeline; - private ChannelHandlerContext ctx; - private SslHandler sslHandler; - - @Before - public void setup() { - pipeline = Mockito.mock(ChannelPipeline.class); - ctx = Mockito.mock(ChannelHandlerContext.class); - Mockito.when(ctx.pipeline()).thenReturn(pipeline); - - securityKeyStore = Mockito.mock(SecurityKeyStore.class); - sslHandler = Mockito.mock(SslHandler.class); - } - - @Test - public void testInvalidMessage() throws Exception { - DualModeSSLHandler handler = new DualModeSSLHandler(securityKeyStore); - - handler.decode(ctx, ALLOCATOR.buffer(4), null); - // ensure pipeline is not fetched and manipulated - Mockito.verify(ctx, Mockito.times(0)).pipeline(); - } - - @Test - public void testValidTLSMessage() throws Exception { - DualModeSSLHandler handler = new DualModeSSLHandler(securityKeyStore, sslHandler); - - ByteBuf buffer = ALLOCATOR.buffer(6); - buffer.writeByte(20); - buffer.writeByte(TLS_MAJOR_VERSION); - buffer.writeByte(TLS_MINOR_VERSION); - buffer.writeByte(100); - buffer.writeByte(0); - buffer.writeByte(0); - - handler.decode(ctx, buffer, null); - // ensure ssl handler is added - Mockito.verify(ctx, Mockito.times(1)).pipeline(); - Mockito.verify(pipeline, Mockito.times(1)).addAfter("port_unification_handler", "ssl_server", sslHandler); - Mockito.verify(pipeline, Mockito.times(1)).remove(handler); - } - - @Test - public void testNonTLSMessage() throws Exception { - DualModeSSLHandler handler = new DualModeSSLHandler(securityKeyStore, sslHandler); - - ByteBuf buffer = ALLOCATOR.buffer(6); - - for (int i = 0; i < 6; i++) { - buffer.writeByte(1); - } - - handler.decode(ctx, buffer, null); - // ensure ssl handler is added - Mockito.verify(ctx, Mockito.times(1)).pipeline(); - Mockito.verify(pipeline, Mockito.times(0)).addAfter("port_unification_handler", "ssl_server", sslHandler); - Mockito.verify(pipeline, Mockito.times(1)).remove(handler); - } - - @Test - public void testDualModeClientHelloMessage() throws Exception { - ChannelFuture channelFuture = Mockito.mock(ChannelFuture.class); - Mockito.when(ctx.writeAndFlush(Mockito.any())).thenReturn(channelFuture); - Mockito.when(channelFuture.addListener(Mockito.any())).thenReturn(channelFuture); - - ByteBuf buffer = ALLOCATOR.buffer(6); - buffer.writeCharSequence(SSLConnectionTestUtil.DUAL_MODE_CLIENT_HELLO_MSG, StandardCharsets.UTF_8); - - DualModeSSLHandler handler = new DualModeSSLHandler(securityKeyStore, sslHandler); - List decodedObjs = new ArrayList<>(); - handler.decode(ctx, buffer, decodedObjs); - - ArgumentCaptor serverHelloReplyBuffer = ArgumentCaptor.forClass(ByteBuf.class); - Mockito.verify(ctx, Mockito.times(1)).writeAndFlush(serverHelloReplyBuffer.capture()); - - String actualReply = serverHelloReplyBuffer.getValue().getCharSequence(0, 6, StandardCharsets.UTF_8).toString(); - Assert.assertEquals(SSLConnectionTestUtil.DUAL_MODE_SERVER_HELLO_MSG, actualReply); - } -} diff --git a/src/test/java/org/opensearch/security/ssl/transport/SecuritySSLNettyTransportTests.java b/src/test/java/org/opensearch/security/ssl/transport/SecuritySSLNettyTransportTests.java deleted file mode 100644 index 32e0f48fac..0000000000 --- a/src/test/java/org/opensearch/security/ssl/transport/SecuritySSLNettyTransportTests.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.security.ssl.transport; - -import java.util.Collections; - -import org.apache.logging.log4j.Logger; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; - -import org.opensearch.OpenSearchSecurityException; -import org.opensearch.Version; -import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.network.NetworkService; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.PageCacheRecycler; -import org.opensearch.core.common.io.stream.NamedWriteableRegistry; -import org.opensearch.core.indices.breaker.CircuitBreakerService; -import org.opensearch.security.ssl.SecurityKeyStore; -import org.opensearch.security.ssl.SslExceptionHandler; -import org.opensearch.security.ssl.transport.SecuritySSLNettyTransport.SSLClientChannelInitializer; -import org.opensearch.security.ssl.transport.SecuritySSLNettyTransport.SSLServerChannelInitializer; -import org.opensearch.telemetry.tracing.Tracer; -import org.opensearch.threadpool.ThreadPool; -import org.opensearch.transport.FakeTcpChannel; -import org.opensearch.transport.SharedGroupFactory; -import org.opensearch.transport.TcpChannel; - -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.DecoderException; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; -import static org.junit.Assert.assertThrows; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class SecuritySSLNettyTransportTests { - - @Mock - private Version version; - @Mock - private ThreadPool threadPool; - @Mock - private PageCacheRecycler pageCacheRecycler; - @Mock - private NamedWriteableRegistry namedWriteableRegistry; - @Mock - private CircuitBreakerService circuitBreakerService; - @Mock - private Tracer trace; - @Mock - private SecurityKeyStore ossks; - @Mock - private SslExceptionHandler sslExceptionHandler; - @Mock - private DiscoveryNode discoveryNode; - - // This initializes all the above mocks - @Rule - public MockitoRule rule = MockitoJUnit.rule(); - - private NetworkService networkService; - private SharedGroupFactory sharedGroupFactory; - private Logger mockLogger; - private SSLConfig sslConfig; - private SecuritySSLNettyTransport securitySSLNettyTransport; - Throwable testCause = new Throwable("Test Cause"); - - @Before - public void setup() { - - networkService = new NetworkService(Collections.emptyList()); - sharedGroupFactory = new SharedGroupFactory(Settings.EMPTY); - - sslConfig = new SSLConfig(Settings.EMPTY); - mockLogger = mock(Logger.class); - - securitySSLNettyTransport = spy( - new SecuritySSLNettyTransport( - Settings.EMPTY, - version, - threadPool, - networkService, - pageCacheRecycler, - namedWriteableRegistry, - circuitBreakerService, - ossks, - sslExceptionHandler, - sharedGroupFactory, - sslConfig, - trace - ) - ); - } - - @Test - public void OnException_withNullChannelShouldThrowException() { - - OpenSearchSecurityException exception = new OpenSearchSecurityException("The provided TCP channel is invalid"); - assertThrows(OpenSearchSecurityException.class, () -> securitySSLNettyTransport.onException(null, exception)); - } - - @Test - public void OnException_withClosedChannelShouldThrowException() { - - TcpChannel channel = new FakeTcpChannel(); - channel.close(); - OpenSearchSecurityException exception = new OpenSearchSecurityException("The provided TCP channel is invalid"); - assertThrows(OpenSearchSecurityException.class, () -> securitySSLNettyTransport.onException(channel, exception)); - } - - @Test - public void OnException_withNullExceptionShouldSucceed() { - - TcpChannel channel = new FakeTcpChannel(); - securitySSLNettyTransport.onException(channel, null); - verify(securitySSLNettyTransport, times(1)).onException(channel, null); - channel.close(); - } - - @Test - public void OnException_withDecoderExceptionShouldGetCause() { - - when(securitySSLNettyTransport.getLogger()).thenReturn(mockLogger); - DecoderException exception = new DecoderException("Test Exception", testCause); - TcpChannel channel = new FakeTcpChannel(); - securitySSLNettyTransport.onException(channel, exception); - verify(mockLogger, times(1)).error("Exception during establishing a SSL connection: " + exception.getCause(), exception.getCause()); - } - - @Test - public void getServerChannelInitializer_shouldReturnValidServerChannel() { - - ChannelHandler channelHandler = securitySSLNettyTransport.getServerChannelInitializer("test-server-channel"); - assertThat(channelHandler, is(notNullValue())); - assertThat(channelHandler, is(instanceOf(SSLServerChannelInitializer.class))); - } - - @Test - public void getClientChannelInitializer_shouldReturnValidClientChannel() { - ChannelHandler channelHandler = securitySSLNettyTransport.getClientChannelInitializer(discoveryNode); - assertThat(channelHandler, is(notNullValue())); - assertThat(channelHandler, is(instanceOf(SSLClientChannelInitializer.class))); - } - - @Test - public void exceptionWithServerChannelHandlerContext_nonNullDecoderExceptionShouldGetCause() throws Exception { - when(securitySSLNettyTransport.getLogger()).thenReturn(mockLogger); - Throwable exception = new DecoderException("Test Exception", testCause); - ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); - securitySSLNettyTransport.getServerChannelInitializer(discoveryNode.getName()).exceptionCaught(ctx, exception); - verify(mockLogger, times(1)).error("Exception during establishing a SSL connection: " + exception.getCause(), exception.getCause()); - } - - @Test - public void exceptionWithServerChannelHandlerContext_nonNullCauseOnlyShouldNotGetCause() throws Exception { - when(securitySSLNettyTransport.getLogger()).thenReturn(mockLogger); - Throwable exception = new OpenSearchSecurityException("Test Exception", testCause); - ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); - securitySSLNettyTransport.getServerChannelInitializer(discoveryNode.getName()).exceptionCaught(ctx, exception); - verify(mockLogger, times(1)).error("Exception during establishing a SSL connection: " + exception, exception); - } - - @Test - public void exceptionWithClientChannelHandlerContext_nonNullDecoderExceptionShouldGetCause() throws Exception { - when(securitySSLNettyTransport.getLogger()).thenReturn(mockLogger); - Throwable exception = new DecoderException("Test Exception", testCause); - ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); - securitySSLNettyTransport.getClientChannelInitializer(discoveryNode).exceptionCaught(ctx, exception); - verify(mockLogger, times(1)).error("Exception during establishing a SSL connection: " + exception.getCause(), exception.getCause()); - } - - @Test - public void exceptionWithClientChannelHandlerContext_nonNullCauseOnlyShouldNotGetCause() throws Exception { - when(securitySSLNettyTransport.getLogger()).thenReturn(mockLogger); - Throwable exception = new OpenSearchSecurityException("Test Exception", testCause); - ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); - securitySSLNettyTransport.getClientChannelInitializer(discoveryNode).exceptionCaught(ctx, exception); - verify(mockLogger, times(1)).error("Exception during establishing a SSL connection: " + exception, exception); - } -} From a16bb6272b6146c0d8ed9ba6755bd73641ccbce7 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 26 Mar 2024 17:20:43 -0400 Subject: [PATCH 157/204] Add simple roles mapping integ test to test mapping of backend role to role (#4172) Signed-off-by: Craig Perkins --- .../security/http/RolesMappingTests.java | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 src/integrationTest/java/org/opensearch/security/http/RolesMappingTests.java diff --git a/src/integrationTest/java/org/opensearch/security/http/RolesMappingTests.java b/src/integrationTest/java/org/opensearch/security/http/RolesMappingTests.java new file mode 100644 index 0000000000..3907be4153 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/http/RolesMappingTests.java @@ -0,0 +1,83 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.security.http; + +import java.util.List; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.test.framework.RolesMapping; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; + +import static org.apache.http.HttpStatus.SC_OK; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; + +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class RolesMappingTests { + static final TestSecurityConfig.User USER_A = new TestSecurityConfig.User("userA").password("s3cret").backendRoles("mapsToRoleA"); + static final TestSecurityConfig.User USER_B = new TestSecurityConfig.User("userB").password("P@ssw0rd").backendRoles("mapsToRoleB"); + + private static final TestSecurityConfig.Role ROLE_A = new TestSecurityConfig.Role("roleA").clusterPermissions("cluster_all"); + + private static final TestSecurityConfig.Role ROLE_B = new TestSecurityConfig.Role("roleB").clusterPermissions("cluster_all"); + + public static final TestSecurityConfig.AuthcDomain AUTHC_DOMAIN = new TestSecurityConfig.AuthcDomain("basic", 0) + .httpAuthenticatorWithChallenge("basic") + .backend("internal"); + + @ClassRule + public static final LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.SINGLENODE) + .anonymousAuth(false) + .authc(AUTHC_DOMAIN) + .roles(ROLE_A, ROLE_B) + .rolesMapping(new RolesMapping(ROLE_A).backendRoles("mapsToRoleA"), new RolesMapping(ROLE_B).backendRoles("mapsToRoleB")) + .users(USER_A, USER_B) + .build(); + + @Test + public void testBackendRoleToRoleMapping() { + try (TestRestClient client = cluster.getRestClient(USER_A)) { + + TestRestClient.HttpResponse response = client.getAuthInfo(); + + assertThat(response, is(notNullValue())); + List roles = response.getTextArrayFromJsonBody("/roles"); + List backendRoles = response.getTextArrayFromJsonBody("/backend_roles"); + assertThat(roles, contains(ROLE_A.getName())); + assertThat(roles, not(contains(ROLE_B.getName()))); + assertThat(backendRoles, contains("mapsToRoleA")); + response.assertStatusCode(SC_OK); + } + + try (TestRestClient client = cluster.getRestClient(USER_B)) { + + TestRestClient.HttpResponse response = client.getAuthInfo(); + + assertThat(response, is(notNullValue())); + List roles = response.getTextArrayFromJsonBody("/roles"); + List backendRoles = response.getTextArrayFromJsonBody("/backend_roles"); + assertThat(roles, contains(ROLE_B.getName())); + assertThat(roles, not(contains(ROLE_A.getName()))); + assertThat(backendRoles, contains("mapsToRoleB")); + response.assertStatusCode(SC_OK); + } + } +} From 551d026552579774dc4cabd26831ba0829e95cf1 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 26 Mar 2024 18:40:26 -0400 Subject: [PATCH 158/204] Update file to modify in the README when adding a system index (#4139) ### Description Updates the README to the current location of the default system indexes. * Category Documentation ### Check List - [ ] New functionality includes testing - [ ] New functionality has been documented - [ ] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). Signed-off-by: Craig Perkins --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index fe698a12d9..0d5d81e109 100644 --- a/README.md +++ b/README.md @@ -129,8 +129,7 @@ plugins.security.system_indices.indices: [".plugins-ml-model", ".plugins-ml-task The demo configuration can be modified in the following files to add a new system index to the demo configuration: -- https://github.com/opensearch-project/security/blob/main/tools/install_demo_configuration.sh -- https://github.com/opensearch-project/security/blob/main/tools/install_demo_configuration.bat +- https://github.com/opensearch-project/security/blob/main/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java ## Contributing From dc8bd61a8df01b32805ddde0b6f6c6010eeca5cf Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Wed, 27 Mar 2024 08:32:10 -0500 Subject: [PATCH 159/204] Remove Pom task dependenices rewrite (#4178) ### Description The underlying issue associated with Pom validation was fixed in OpenSearch [1], this causes the 'hack' that was used to unblock pom generation in security to start causing problems in the distribution builds. - [1] https://github.com/opensearch-project/OpenSearch/pull/12807 ### Issues Resolved - Fixes https://github.com/opensearch-project/security/issues/4160 ### Testing Added new CI test that executes the same job used by the distribution build and checks the expected artifacts exist ### Check List - [X] New functionality includes testing - [ ] ~New functionality has been documented~ - [X] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). Signed-off-by: Peter Nied --- .github/workflows/ci.yml | 2 ++ build.gradle | 5 ----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68e966f56e..d1873fba2a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -246,6 +246,8 @@ jobs: - run: ./gradlew clean assemble -Dbuild.version_qualifier=${{ env.TEST_QUALIFIER }} && test -s ./build/distributions/opensearch-security-${{ env.SECURITY_PLUGIN_VERSION_ONLY_NUMBER }}-${{ env.TEST_QUALIFIER }}-SNAPSHOT.zip + - run: ./gradlew clean publishPluginZipPublicationToZipStagingRepository && test -s ./build/distributions/opensearch-security-${{ env.SECURITY_PLUGIN_VERSION }}.zip && test -s ./build/distributions/opensearch-security-${{ env.SECURITY_PLUGIN_VERSION }}.pom + - name: List files in the build directory if there was an error run: ls -al ./build/distributions/ if: failure() diff --git a/build.gradle b/build.gradle index ea392a8eac..aa99828d5c 100644 --- a/build.gradle +++ b/build.gradle @@ -407,11 +407,6 @@ opensearchplugin { // This requires an additional Jar not published as part of build-tools loggerUsageCheck.enabled = false -// No need to validate pom, as we do not upload to maven/sonatype -tasks.matching {it.path in [":validateMavenPom", ":validateNebulaPom", ":validatePluginZipPom"]}.all { task -> - task.dependsOn ':generatePomFileForNebulaPublication', ':generatePomFileForPluginZipPublication', ':generatePomFileForMavenPublication' -} - publishing { publications { pluginZip(MavenPublication) { publication -> From b0d26ddbfd584a76cb7eb48ee36c461fd0e9e19b Mon Sep 17 00:00:00 2001 From: Andrey Pleskach Date: Wed, 27 Mar 2024 17:09:22 +0100 Subject: [PATCH 160/204] Fix cluster default initialization Part 1 (#4002) Signed-off-by: Andrey Pleskach --- build.gradle | 4 +- ...=> AbstractDefaultConfigurationTests.java} | 88 +-- ...ultConfigurationMultiNodeClusterTests.java | 39 ++ ...nMultiNodeClusterUseClusterStateTests.java | 42 ++ ...ltConfigurationSingleNodeClusterTests.java | 44 ++ ...SingleNodeClusterUseClusterStateTests.java | 42 ++ .../SecurityConfigurationBootstrapTests.java | 3 +- .../security/OpenSearchSecurityPlugin.java | 33 +- .../ConfigurationRepository.java | 221 ++++++-- .../impl/SecurityDynamicConfiguration.java | 5 + .../security/state/SecurityConfig.java | 124 +++++ .../security/state/SecurityMetadata.java | 128 +++++ .../security/support/ConfigConstants.java | 5 + .../security/support/ConfigHelper.java | 1 + .../support/SecurityIndexHandler.java | 233 ++++++++ .../security/support/YamlConfigReader.java | 95 ++++ .../ConfigurationRepositoryTest.java | 190 ++++++- ...SecurityMetadataSerializationTestCase.java | 154 ++++++ .../security/support/ConfigReaderTest.java | 63 +++ .../support/SecurityIndexHandlerTest.java | 510 ++++++++++++++++++ 20 files changed, 1930 insertions(+), 94 deletions(-) rename src/integrationTest/java/org/opensearch/security/{DefaultConfigurationTests.java => AbstractDefaultConfigurationTests.java} (69%) create mode 100644 src/integrationTest/java/org/opensearch/security/DefaultConfigurationMultiNodeClusterTests.java create mode 100644 src/integrationTest/java/org/opensearch/security/DefaultConfigurationMultiNodeClusterUseClusterStateTests.java create mode 100644 src/integrationTest/java/org/opensearch/security/DefaultConfigurationSingleNodeClusterTests.java create mode 100644 src/integrationTest/java/org/opensearch/security/DefaultConfigurationSingleNodeClusterUseClusterStateTests.java create mode 100644 src/main/java/org/opensearch/security/state/SecurityConfig.java create mode 100644 src/main/java/org/opensearch/security/state/SecurityMetadata.java create mode 100644 src/main/java/org/opensearch/security/support/SecurityIndexHandler.java create mode 100644 src/main/java/org/opensearch/security/support/YamlConfigReader.java create mode 100644 src/test/java/org/opensearch/security/state/SecurityMetadataSerializationTestCase.java create mode 100644 src/test/java/org/opensearch/security/support/ConfigReaderTest.java create mode 100644 src/test/java/org/opensearch/security/support/SecurityIndexHandlerTest.java diff --git a/build.gradle b/build.gradle index aa99828d5c..607649d082 100644 --- a/build.gradle +++ b/build.gradle @@ -613,6 +613,7 @@ dependencies { //OpenSAML implementation 'net.shibboleth.utilities:java-support:8.4.1' + runtimeOnly "io.dropwizard.metrics:metrics-core:4.2.15" implementation "com.onelogin:java-saml:${one_login_java_saml}" implementation "com.onelogin:java-saml-core:${one_login_java_saml}" implementation "org.opensaml:opensaml-core:${open_saml_version}" @@ -638,7 +639,6 @@ dependencies { runtimeOnly 'com.google.j2objc:j2objc-annotations:2.8' compileOnly 'com.google.code.findbugs:jsr305:3.0.2' runtimeOnly 'org.lz4:lz4-java:1.8.0' - runtimeOnly 'io.dropwizard.metrics:metrics-core:4.2.25' runtimeOnly 'org.slf4j:slf4j-api:1.7.36' runtimeOnly "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}" runtimeOnly 'org.xerial.snappy:snappy-java:1.1.10.5' @@ -699,12 +699,12 @@ dependencies { exclude(group:'org.springframework', module: 'spring-jcl' ) } testRuntimeOnly 'org.scala-lang:scala-library:2.13.13' - testRuntimeOnly 'com.yammer.metrics:metrics-core:2.2.0' testRuntimeOnly 'com.typesafe.scala-logging:scala-logging_3:3.9.5' testRuntimeOnly('org.apache.zookeeper:zookeeper:3.9.2') { exclude(group:'ch.qos.logback', module: 'logback-classic' ) exclude(group:'ch.qos.logback', module: 'logback-core' ) } + testRuntimeOnly 'com.yammer.metrics:metrics-core:2.2.0' testRuntimeOnly "org.apache.kafka:kafka-metadata:${kafka_version}" testRuntimeOnly "org.apache.kafka:kafka-storage:${kafka_version}" diff --git a/src/integrationTest/java/org/opensearch/security/DefaultConfigurationTests.java b/src/integrationTest/java/org/opensearch/security/AbstractDefaultConfigurationTests.java similarity index 69% rename from src/integrationTest/java/org/opensearch/security/DefaultConfigurationTests.java rename to src/integrationTest/java/org/opensearch/security/AbstractDefaultConfigurationTests.java index eb028c74e4..5387b3e516 100644 --- a/src/integrationTest/java/org/opensearch/security/DefaultConfigurationTests.java +++ b/src/integrationTest/java/org/opensearch/security/AbstractDefaultConfigurationTests.java @@ -1,12 +1,12 @@ /* -* Copyright OpenSearch Contributors -* SPDX-License-Identifier: Apache-2.0 -* -* The OpenSearch Contributors require contributions made to -* this file be licensed under the Apache-2.0 license or a -* compatible open source license. -* -*/ + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ package org.opensearch.security; import java.io.IOException; @@ -19,17 +19,16 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import com.fasterxml.jackson.databind.JsonNode; import org.apache.commons.io.FileUtils; +import org.apache.http.HttpStatus; import org.awaitility.Awaitility; import org.junit.AfterClass; -import org.junit.ClassRule; import org.junit.Test; import org.junit.runner.RunWith; -import org.opensearch.test.framework.TestSecurityConfig.User; -import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.security.state.SecurityMetadata; +import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; -import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.aMapWithSize; @@ -37,29 +36,22 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.not; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; @RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) @ThreadLeakScope(ThreadLeakScope.Scope.NONE) -public class DefaultConfigurationTests { - - private final static Path configurationFolder = ConfigurationFiles.createConfigurationDirectory(); - private static final User ADMIN_USER = new User("admin"); - private static final User NEW_USER = new User("new-user"); - private static final User LIMITED_USER = new User("limited-user"); - - @ClassRule - public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.SINGLENODE) - .nodeSettings( - Map.of( - "plugins.security.allow_default_init_securityindex", - true, - "plugins.security.restapi.roles_enabled", - List.of("user_admin__all_access") - ) - ) - .defaultConfigurationInitDirectory(configurationFolder.toString()) - .loadConfigurationIntoIndex(false) - .build(); +public abstract class AbstractDefaultConfigurationTests { + public final static Path configurationFolder = ConfigurationFiles.createConfigurationDirectory(); + private static final TestSecurityConfig.User ADMIN_USER = new TestSecurityConfig.User("admin"); + private static final TestSecurityConfig.User NEW_USER = new TestSecurityConfig.User("new-user"); + private static final TestSecurityConfig.User LIMITED_USER = new TestSecurityConfig.User("limited-user"); + + private final LocalCluster cluster; + + protected AbstractDefaultConfigurationTests(LocalCluster cluster) { + this.cluster = cluster; + } @AfterClass public static void cleanConfigurationDirectory() throws IOException { @@ -73,18 +65,43 @@ public void shouldLoadDefaultConfiguration() { } try (TestRestClient client = cluster.getRestClient(ADMIN_USER)) { client.confirmCorrectCredentials(ADMIN_USER.getName()); - HttpResponse response = client.get("_plugins/_security/api/internalusers"); - response.assertStatusCode(200); + TestRestClient.HttpResponse response = client.get("_plugins/_security/api/internalusers"); + response.assertStatusCode(HttpStatus.SC_OK); Map users = response.getBodyAs(Map.class); assertThat( + response.getBody(), users, allOf(aMapWithSize(3), hasKey(ADMIN_USER.getName()), hasKey(NEW_USER.getName()), hasKey(LIMITED_USER.getName())) ); } } + void assertClusterState(final TestRestClient client) { + if (cluster.node().settings().getAsBoolean("plugins.security.allow_default_init_securityindex.use_cluster_state", false)) { + final TestRestClient.HttpResponse response = client.get("_cluster/state"); + response.assertStatusCode(HttpStatus.SC_OK); + final var clusterState = response.getBodyAs(Map.class); + assertTrue(response.getBody(), clusterState.containsKey(SecurityMetadata.TYPE)); + @SuppressWarnings("unchecked") + final var securityClusterState = (Map) clusterState.get(SecurityMetadata.TYPE); + @SuppressWarnings("unchecked") + final var securityConfiguration = (Map) ((Map) clusterState.get(SecurityMetadata.TYPE)).get( + "configuration" + ); + assertTrue(response.getBody(), securityClusterState.containsKey("created")); + assertNotNull(response.getBody(), securityClusterState.get("created")); + + for (final var k : securityConfiguration.keySet()) { + @SuppressWarnings("unchecked") + final var sc = (Map) securityConfiguration.get(k); + assertTrue(response.getBody(), sc.containsKey("hash")); + assertTrue(response.getBody(), sc.containsKey("last_modified")); + } + } + } + @Test - public void securityRolesUgrade() throws Exception { + public void securityRolesUpgrade() throws Exception { try (var client = cluster.getRestClient(ADMIN_USER)) { // Setup: Make sure the config is ready before starting modifications Awaitility.await().alias("Load default configuration").until(() -> client.getAuthInfo().getStatusCode(), equalTo(200)); @@ -159,4 +176,5 @@ private Set extractFieldNames(final JsonNode json) { json.fieldNames().forEachRemaining(set::add); return set; } + } diff --git a/src/integrationTest/java/org/opensearch/security/DefaultConfigurationMultiNodeClusterTests.java b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationMultiNodeClusterTests.java new file mode 100644 index 0000000000..704e2c7255 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationMultiNodeClusterTests.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.security; + +import java.util.List; +import java.util.Map; + +import org.junit.ClassRule; + +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; + +public class DefaultConfigurationMultiNodeClusterTests extends AbstractDefaultConfigurationTests { + + @ClassRule + public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS) + .nodeSettings( + Map.of( + "plugins.security.allow_default_init_securityindex", + true, + "plugins.security.restapi.roles_enabled", + List.of("user_admin__all_access") + ) + ) + .defaultConfigurationInitDirectory(configurationFolder.toString()) + .loadConfigurationIntoIndex(false) + .build(); + + public DefaultConfigurationMultiNodeClusterTests() { + super(cluster); + } +} diff --git a/src/integrationTest/java/org/opensearch/security/DefaultConfigurationMultiNodeClusterUseClusterStateTests.java b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationMultiNodeClusterUseClusterStateTests.java new file mode 100644 index 0000000000..8abffac9cf --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationMultiNodeClusterUseClusterStateTests.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.security; + +import java.util.List; +import java.util.Map; + +import org.junit.ClassRule; + +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; + +public class DefaultConfigurationMultiNodeClusterUseClusterStateTests extends AbstractDefaultConfigurationTests { + + @ClassRule + public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS) + .nodeSettings( + Map.of( + "plugins.security.allow_default_init_securityindex", + true, + "plugins.security.allow_default_init_securityindex.use_cluster_state", + true, + "plugins.security.restapi.roles_enabled", + List.of("user_admin__all_access") + ) + ) + .defaultConfigurationInitDirectory(configurationFolder.toString()) + .loadConfigurationIntoIndex(false) + .build(); + + public DefaultConfigurationMultiNodeClusterUseClusterStateTests() { + super(cluster); + } + +} diff --git a/src/integrationTest/java/org/opensearch/security/DefaultConfigurationSingleNodeClusterTests.java b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationSingleNodeClusterTests.java new file mode 100644 index 0000000000..362245db5e --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationSingleNodeClusterTests.java @@ -0,0 +1,44 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security; + +import java.util.List; +import java.util.Map; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.junit.ClassRule; +import org.junit.runner.RunWith; + +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; + +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class DefaultConfigurationSingleNodeClusterTests extends AbstractDefaultConfigurationTests { + + @ClassRule + public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.SINGLENODE) + .nodeSettings( + Map.of( + "plugins.security.allow_default_init_securityindex", + true, + "plugins.security.restapi.roles_enabled", + List.of("user_admin__all_access") + ) + ) + .defaultConfigurationInitDirectory(configurationFolder.toString()) + .loadConfigurationIntoIndex(false) + .build(); + + public DefaultConfigurationSingleNodeClusterTests() { + super(cluster); + } + +} diff --git a/src/integrationTest/java/org/opensearch/security/DefaultConfigurationSingleNodeClusterUseClusterStateTests.java b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationSingleNodeClusterUseClusterStateTests.java new file mode 100644 index 0000000000..e05005e912 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/DefaultConfigurationSingleNodeClusterUseClusterStateTests.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.security; + +import java.util.List; +import java.util.Map; + +import org.junit.ClassRule; + +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; + +public class DefaultConfigurationSingleNodeClusterUseClusterStateTests extends AbstractDefaultConfigurationTests { + + @ClassRule + public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.SINGLENODE) + .nodeSettings( + Map.of( + "plugins.security.allow_default_init_securityindex", + true, + "plugins.security.allow_default_init_securityindex.use_cluster_state", + true, + "plugins.security.restapi.roles_enabled", + List.of("user_admin__all_access") + ) + ) + .defaultConfigurationInitDirectory(configurationFolder.toString()) + .loadConfigurationIntoIndex(false) + .build(); + + public DefaultConfigurationSingleNodeClusterUseClusterStateTests() { + super(cluster); + } + +} diff --git a/src/integrationTest/java/org/opensearch/security/SecurityConfigurationBootstrapTests.java b/src/integrationTest/java/org/opensearch/security/SecurityConfigurationBootstrapTests.java index 5b83e0d6d0..e6af5d58bb 100644 --- a/src/integrationTest/java/org/opensearch/security/SecurityConfigurationBootstrapTests.java +++ b/src/integrationTest/java/org/opensearch/security/SecurityConfigurationBootstrapTests.java @@ -124,6 +124,7 @@ public void shouldStillLoadSecurityConfigDuringBootstrapAndActiveConfigUpdateReq .put("action_groups.yml", CType.ACTIONGROUPS) .put("config.yml", CType.CONFIG) .put("roles.yml", CType.ROLES) + .put("roles_mapping.yml", CType.ROLESMAPPING) .put("tenants.yml", CType.TENANTS) .build(); @@ -146,7 +147,7 @@ public void shouldStillLoadSecurityConfigDuringBootstrapAndActiveConfigUpdateReq // After the configuration has been loaded, the rest clients should be able to connect successfully cluster.triggerConfigurationReloadForCTypes( internalNodeClient, - List.of(CType.ACTIONGROUPS, CType.CONFIG, CType.ROLES, CType.TENANTS), + List.of(CType.ACTIONGROUPS, CType.CONFIG, CType.ROLES, CType.ROLESMAPPING, CType.TENANTS), true ); try (final TestRestClient freshClient = cluster.getRestClient(USER_ADMIN)) { diff --git a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java index ac32da1d1b..a59d1f531d 100644 --- a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java +++ b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java @@ -73,6 +73,8 @@ import org.opensearch.action.search.SearchScrollAction; import org.opensearch.action.support.ActionFilter; import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.NamedDiff; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; @@ -176,6 +178,7 @@ import org.opensearch.security.ssl.http.netty.ValidatingDispatcher; import org.opensearch.security.ssl.transport.DefaultPrincipalExtractor; import org.opensearch.security.ssl.util.SSLConfigConstants; +import org.opensearch.security.state.SecurityMetadata; import org.opensearch.security.support.ConfigConstants; import org.opensearch.security.support.GuardedSearchOperationWrapper; import org.opensearch.security.support.HeaderHelper; @@ -208,6 +211,8 @@ import static org.opensearch.security.dlic.rest.api.RestApiAdminPrivilegesEvaluator.ENDPOINTS_WITH_PERMISSIONS; import static org.opensearch.security.dlic.rest.api.RestApiAdminPrivilegesEvaluator.SECURITY_CONFIG_UPDATE; import static org.opensearch.security.setting.DeprecatedSettings.checkForDeprecatedSetting; +import static org.opensearch.security.support.ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX; +import static org.opensearch.security.support.ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_USE_CLUSTER_STATE; import static org.opensearch.security.support.ConfigConstants.SECURITY_UNSUPPORTED_RESTAPI_ALLOW_SECURITYCONFIG_MODIFICATION; // CS-ENFORCE-SINGLE @@ -288,6 +293,10 @@ private static boolean isDisabled(final Settings settings) { return settings.getAsBoolean(ConfigConstants.SECURITY_DISABLED, false); } + private static boolean useClusterStateToInitSecurityConfig(final Settings settings) { + return settings.getAsBoolean(SECURITY_ALLOW_DEFAULT_INIT_USE_CLUSTER_STATE, false); + } + /** * SSL Cert Reload will be enabled only if security is not disabled and not in we are not using sslOnly mode. * @param settings Elastic configuration settings @@ -1172,11 +1181,23 @@ public Collection createComponents( components.add(si); components.add(dcf); components.add(userService); - + final var allowDefaultInit = settings.getAsBoolean(SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, false); + final var useClusterState = useClusterStateToInitSecurityConfig(settings); + if (!SSLConfig.isSslOnlyMode() && !isDisabled(settings) && allowDefaultInit && useClusterState) { + clusterService.addListener(cr); + } return components; } + @Override + public List getNamedWriteables() { + return List.of( + new NamedWriteableRegistry.Entry(ClusterState.Custom.class, SecurityMetadata.TYPE, SecurityMetadata::new), + new NamedWriteableRegistry.Entry(NamedDiff.class, SecurityMetadata.TYPE, SecurityMetadata::readDiffFrom) + ); + } + @Override public Settings additionalSettings() { @@ -1317,9 +1338,8 @@ public List> getSettings() { settings.add( Setting.boolSetting(ConfigConstants.SECURITY_ALLOW_UNSAFE_DEMOCERTIFICATES, false, Property.NodeScope, Property.Filtered) ); - settings.add( - Setting.boolSetting(ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, false, Property.NodeScope, Property.Filtered) - ); + settings.add(Setting.boolSetting(SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, false, Property.NodeScope, Property.Filtered)); + settings.add(Setting.boolSetting(SECURITY_ALLOW_DEFAULT_INIT_USE_CLUSTER_STATE, false, Property.NodeScope, Property.Filtered)); settings.add( Setting.boolSetting( ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, @@ -1915,11 +1935,10 @@ public List getSettingsFilter() { @Override public void onNodeStarted(DiscoveryNode localNode) { - log.info("Node started"); - if (!SSLConfig.isSslOnlyMode() && !client && !disabled) { + this.localNode.set(localNode); + if (!SSLConfig.isSslOnlyMode() && !client && !disabled && !useClusterStateToInitSecurityConfig(settings)) { cr.initOnNodeStart(); } - this.localNode.set(localNode); final Set securityModules = ReflectionHelper.getModulesLoaded(); log.info("{} OpenSearch Security modules loaded so far: {}", securityModules.size(), securityModules); } diff --git a/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java b/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java index 353286fc4a..44ba77428f 100644 --- a/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java +++ b/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java @@ -31,6 +31,7 @@ import java.security.AccessController; import java.security.PrivilegedAction; import java.text.SimpleDateFormat; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -38,12 +39,16 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Supplier; +import java.util.stream.Collectors; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -58,13 +63,19 @@ import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.ClusterStateListener; +import org.opensearch.cluster.ClusterStateUpdateTask; import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.Priority; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.concurrent.ThreadContext.StoredContext; +import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.Strings; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.MediaTypeRegistry; @@ -75,12 +86,16 @@ import org.opensearch.security.securityconf.impl.CType; import org.opensearch.security.securityconf.impl.SecurityDynamicConfiguration; import org.opensearch.security.ssl.util.ExceptionUtils; +import org.opensearch.security.state.SecurityMetadata; import org.opensearch.security.support.ConfigConstants; import org.opensearch.security.support.ConfigHelper; +import org.opensearch.security.support.SecurityIndexHandler; import org.opensearch.security.support.SecurityUtils; import org.opensearch.threadpool.ThreadPool; -public class ConfigurationRepository { +import static org.opensearch.security.support.ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_USE_CLUSTER_STATE; + +public class ConfigurationRepository implements ClusterStateListener { private static final Logger LOGGER = LogManager.getLogger(ConfigurationRepository.class); private final String securityIndex; @@ -96,20 +111,27 @@ public class ConfigurationRepository { private DynamicConfigFactory dynamicConfigFactory; public static final int DEFAULT_CONFIG_VERSION = 2; private final CompletableFuture initalizeConfigTask = new CompletableFuture<>(); + private final boolean acceptInvalid; - private ConfigurationRepository( - Settings settings, + private final AtomicBoolean auditHotReloadingEnabled = new AtomicBoolean(false); + + private final AtomicBoolean initializationInProcess = new AtomicBoolean(false); + + private final SecurityIndexHandler securityIndexHandler; + + // visible for testing + protected ConfigurationRepository( + final String securityIndex, + final Settings settings, final Path configPath, - ThreadPool threadPool, - Client client, - ClusterService clusterService, - AuditLog auditLog + final ThreadPool threadPool, + final Client client, + final ClusterService clusterService, + final AuditLog auditLog, + final SecurityIndexHandler securityIndexHandler ) { - this.securityIndex = settings.get( - ConfigConstants.SECURITY_CONFIG_INDEX_NAME, - ConfigConstants.OPENDISTRO_SECURITY_DEFAULT_CONFIG_INDEX - ); + this.securityIndex = securityIndex; this.settings = settings; this.configPath = configPath; this.client = client; @@ -119,8 +141,38 @@ private ConfigurationRepository( this.configurationChangedListener = new ArrayList<>(); this.acceptInvalid = settings.getAsBoolean(ConfigConstants.SECURITY_UNSUPPORTED_ACCEPT_INVALID_CONFIG, false); cl = new ConfigurationLoaderSecurity7(client, threadPool, settings, clusterService); - configCache = CacheBuilder.newBuilder().build(); + this.securityIndexHandler = securityIndexHandler; + } + + private Path resolveConfigDir() { + return Optional.ofNullable(System.getProperty("security.default_init.dir")) + .map(Path::of) + .orElseGet(() -> new Environment(settings, configPath).configDir().resolve("opensearch-security/")); + } + + @Override + public void clusterChanged(final ClusterChangedEvent event) { + final SecurityMetadata securityMetadata = event.state().custom(SecurityMetadata.TYPE); + // init and upload sec index on the manager node only as soon as + // creation of index and upload config are done a new cluster state will be created. + // in case of failures it repeats attempt after restart + if (nodeSelectedAsManager(event)) { + if (securityMetadata == null) { + initSecurityIndex(event); + } + } + // executes reload of cache on each node on the cluster, + // since sec initialization has been finished + if (securityMetadata != null) { + executeConfigurationInitialization(securityMetadata); + } + } + + private boolean nodeSelectedAsManager(final ClusterChangedEvent event) { + boolean wasClusterManager = event.previousState().nodes().isLocalNodeElectedClusterManager(); + boolean isClusterManager = event.localNodeClusterManager(); + return !wasClusterManager && isClusterManager; } public String getConfigDirectory() { @@ -236,7 +288,7 @@ private void initalizeClusterConfiguration(final boolean installDefaultConfig) { } catch (Exception e) { LOGGER.debug("Unable to load configuration due to {}", String.valueOf(ExceptionUtils.getRootCause(e))); try { - Thread.sleep(3000); + TimeUnit.MILLISECONDS.sleep(3000); } catch (InterruptedException e1) { Thread.currentThread().interrupt(); LOGGER.debug("Thread was interrupted so we cancel initialization"); @@ -244,27 +296,7 @@ private void initalizeClusterConfiguration(final boolean installDefaultConfig) { } } } - - final Set deprecatedAuditKeysInSettings = AuditConfig.getDeprecatedKeys(settings); - if (!deprecatedAuditKeysInSettings.isEmpty()) { - LOGGER.warn( - "Following keys {} are deprecated in opensearch settings. They will be removed in plugin v2.0.0.0", - deprecatedAuditKeysInSettings - ); - } - final boolean isAuditConfigDocPresentInIndex = cl.isAuditConfigDocPresentInIndex(); - if (isAuditConfigDocPresentInIndex) { - if (!deprecatedAuditKeysInSettings.isEmpty()) { - LOGGER.warn("Audit configuration settings found in both index and opensearch settings (deprecated)"); - } - LOGGER.info("Hot-reloading of audit configuration is enabled"); - } else { - LOGGER.info( - "Hot-reloading of audit configuration is disabled. Using configuration with defaults from opensearch settings. Populate the configuration in index using audit.yml or securityadmin to enable it." - ); - auditLog.setConfig(AuditConfig.from(settings)); - } - + setupAuditConfigurationIfAny(cl.isAuditConfigDocPresentInIndex()); LOGGER.info("Node '{}' initialized", clusterService.localNode().getName()); } catch (Exception e) { @@ -272,6 +304,27 @@ private void initalizeClusterConfiguration(final boolean installDefaultConfig) { } } + private void setupAuditConfigurationIfAny(final boolean auditConfigDocPresent) { + final Set deprecatedAuditKeysInSettings = AuditConfig.getDeprecatedKeys(settings); + if (!deprecatedAuditKeysInSettings.isEmpty()) { + LOGGER.warn( + "Following keys {} are deprecated in opensearch settings. They will be removed in plugin v2.0.0.0", + deprecatedAuditKeysInSettings + ); + } + if (auditConfigDocPresent) { + if (!deprecatedAuditKeysInSettings.isEmpty()) { + LOGGER.warn("Audit configuration settings found in both index and opensearch settings (deprecated)"); + } + LOGGER.info("Hot-reloading of audit configuration is enabled"); + } else { + LOGGER.info( + "Hot-reloading of audit configuration is disabled. Using configuration with defaults from opensearch settings. Populate the configuration in index using audit.yml or securityadmin to enable it." + ); + auditLog.setConfig(AuditConfig.from(settings)); + } + } + private boolean createSecurityIndexIfAbsent() { try { final Map indexSettings = ImmutableMap.of("index.number_of_shards", 1, "index.auto_expand_replicas", "0-all"); @@ -304,7 +357,7 @@ private void waitForSecurityIndexToBeAtLeastYellow() { response == null ? "no response" : (response.isTimedOut() ? "timeout" : "other, maybe red cluster") ); try { - Thread.sleep(500); + TimeUnit.MILLISECONDS.sleep(500); } catch (InterruptedException e) { // ignore Thread.currentThread().interrupt(); @@ -317,6 +370,69 @@ private void waitForSecurityIndexToBeAtLeastYellow() { } } + void initSecurityIndex(final ClusterChangedEvent event) { + if (!event.state().metadata().hasIndex(securityIndex)) { + securityIndexHandler.createIndex( + ActionListener.wrap(r -> uploadDefaultConfiguration0(), e -> LOGGER.error("Couldn't create index {}", securityIndex, e)) + ); + } else { + // in case index was created and cluster state has not been changed (e.g. restart of the node or something) + // just upload default configuration + uploadDefaultConfiguration0(); + } + } + + private void uploadDefaultConfiguration0() { + securityIndexHandler.uploadDefaultConfiguration( + resolveConfigDir(), + ActionListener.wrap( + configuration -> clusterService.submitStateUpdateTask( + "init-security-configuration", + new ClusterStateUpdateTask(Priority.IMMEDIATE) { + @Override + public ClusterState execute(ClusterState clusterState) throws Exception { + return ClusterState.builder(clusterState) + .putCustom(SecurityMetadata.TYPE, new SecurityMetadata(Instant.now(), configuration)) + .build(); + } + + @Override + public void onFailure(String s, Exception e) { + LOGGER.error(s, e); + } + } + ), + e -> LOGGER.error("Couldn't upload default configuration", e) + ) + ); + } + + Future executeConfigurationInitialization(final SecurityMetadata securityMetadata) { + if (!initalizeConfigTask.isDone()) { + if (initializationInProcess.compareAndSet(false, true)) { + return threadPool.generic().submit(() -> { + securityIndexHandler.loadConfiguration(securityMetadata.configuration(), ActionListener.wrap(cTypeConfigs -> { + notifyConfigurationListeners(cTypeConfigs); + final var auditConfigDocPresent = cTypeConfigs.containsKey(CType.AUDIT) && cTypeConfigs.get(CType.AUDIT).notEmpty(); + setupAuditConfigurationIfAny(auditConfigDocPresent); + auditHotReloadingEnabled.getAndSet(auditConfigDocPresent); + initalizeConfigTask.complete(null); + LOGGER.info( + "Security configuration initialized. Applied hashes: {}", + securityMetadata.configuration() + .stream() + .map(c -> String.format("%s:%s", c.type().toLCString(), c.hash())) + .collect(Collectors.toList()) + ); + }, e -> LOGGER.error("Couldn't reload security configuration", e))); + return null; + }); + } + } + return CompletableFuture.completedFuture(null); + } + + @Deprecated public CompletableFuture initOnNodeStart() { final boolean installDefaultConfig = settings.getAsBoolean(ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, false); @@ -333,13 +449,15 @@ public CompletableFuture initOnNodeStart() { return startInitialization.get(); } else if (settings.getAsBoolean(ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, true)) { LOGGER.info( - "Will not attempt to create index {} and default configs if they are absent. Use securityadmin to initialize cluster", + "Will not attempt to create index {} and default configs if they are absent." + + " Use securityadmin to initialize cluster", securityIndex ); return startInitialization.get(); } else { LOGGER.info( - "Will not attempt to create index {} and default configs if they are absent. Will not perform background initialization", + "Will not attempt to create index {} and default configs if they are absent. " + + "Will not perform background initialization", securityIndex ); initalizeConfigTask.complete(null); @@ -352,7 +470,11 @@ public CompletableFuture initOnNodeStart() { } public boolean isAuditHotReloadingEnabled() { - return cl.isAuditConfigDocPresentInIndex(); + if (settings.getAsBoolean(SECURITY_ALLOW_DEFAULT_INIT_USE_CLUSTER_STATE, false)) { + return auditHotReloadingEnabled.get(); + } else { + return cl.isAuditConfigDocPresentInIndex(); + } } public static ConfigurationRepository create( @@ -363,15 +485,20 @@ public static ConfigurationRepository create( ClusterService clusterService, AuditLog auditLog ) { - final ConfigurationRepository repository = new ConfigurationRepository( + final var securityIndex = settings.get( + ConfigConstants.SECURITY_CONFIG_INDEX_NAME, + ConfigConstants.OPENDISTRO_SECURITY_DEFAULT_CONFIG_INDEX + ); + return new ConfigurationRepository( + securityIndex, settings, configPath, threadPool, client, clusterService, - auditLog + auditLog, + new SecurityIndexHandler(securityIndex, settings, client) ); - return repository; } public void setDynamicConfigFactory(DynamicConfigFactory dynamicConfigFactory) { @@ -403,6 +530,10 @@ private boolean reloadConfiguration(final Collection configTypes, final b LOGGER.warn("Unable to reload configuration, initalization thread has not yet completed."); return false; } + return loadConfigurationWithLock(configTypes); + } + + private boolean loadConfigurationWithLock(Collection configTypes) { try { if (LOCK.tryLock(60, TimeUnit.SECONDS)) { try { @@ -422,8 +553,12 @@ private boolean reloadConfiguration(final Collection configTypes, final b private void reloadConfiguration0(Collection configTypes, boolean acceptInvalid) { final Map> loaded = getConfigurationsFromIndex(configTypes, false, acceptInvalid); - configCache.putAll(loaded); - notifyAboutChanges(loaded); + notifyConfigurationListeners(loaded); + } + + private void notifyConfigurationListeners(final Map> configuration) { + configCache.putAll(configuration); + notifyAboutChanges(configuration); } public synchronized void subscribeOnChange(ConfigurationChangeListener listener) { diff --git a/src/main/java/org/opensearch/security/securityconf/impl/SecurityDynamicConfiguration.java b/src/main/java/org/opensearch/security/securityconf/impl/SecurityDynamicConfiguration.java index 90508840e7..83553f2de7 100644 --- a/src/main/java/org/opensearch/security/securityconf/impl/SecurityDynamicConfiguration.java +++ b/src/main/java/org/opensearch/security/securityconf/impl/SecurityDynamicConfiguration.java @@ -68,6 +68,11 @@ public static SecurityDynamicConfiguration empty() { return new SecurityDynamicConfiguration(); } + @JsonIgnore + public boolean notEmpty() { + return !centries.isEmpty(); + } + public static SecurityDynamicConfiguration fromJson(String json, CType ctype, int version, long seqNo, long primaryTerm) throws IOException { return fromJson(json, ctype, version, seqNo, primaryTerm, false); diff --git a/src/main/java/org/opensearch/security/state/SecurityConfig.java b/src/main/java/org/opensearch/security/state/SecurityConfig.java new file mode 100644 index 0000000000..f8de098365 --- /dev/null +++ b/src/main/java/org/opensearch/security/state/SecurityConfig.java @@ -0,0 +1,124 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ +package org.opensearch.security.state; + +import java.io.IOException; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.security.securityconf.impl.CType; + +import static java.time.format.DateTimeFormatter.ISO_INSTANT; + +public class SecurityConfig implements Writeable, ToXContent { + + private final CType type; + + private final Instant lastModified; + + private final String hash; + + public SecurityConfig(final CType type, final String hash, final Instant lastModified) { + this.type = type; + this.hash = hash; + this.lastModified = lastModified; + } + + public SecurityConfig(final StreamInput in) throws IOException { + this.type = in.readEnum(CType.class); + this.hash = in.readString(); + this.lastModified = in.readOptionalInstant(); + } + + public Optional lastModified() { + return Optional.ofNullable(lastModified); + } + + public CType type() { + return type; + } + + public String hash() { + return hash; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeEnum(type); + out.writeString(hash); + out.writeOptionalInstant(lastModified); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder xContentBuilder, final Params params) throws IOException { + xContentBuilder.startObject(type.toLCString()).field("hash", hash); + if (lastModified != null) { + xContentBuilder.field("last_modified", ISO_INSTANT.format(lastModified)); + } else { + xContentBuilder.nullField("last_modified"); + } + return xContentBuilder.endObject(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SecurityConfig that = (SecurityConfig) o; + return type == that.type && Objects.equals(lastModified, that.lastModified) && Objects.equals(hash, that.hash); + } + + @Override + public int hashCode() { + return Objects.hash(type, lastModified, hash); + } + + public final static class Builder { + + private final CType type; + + private Instant lastModified; + + private String hash; + + Builder(final SecurityConfig securityConfig) { + this.type = securityConfig.type; + this.lastModified = securityConfig.lastModified; + this.hash = securityConfig.hash; + } + + public Builder withHash(final String hash) { + this.hash = hash; + return this; + } + + public Builder withLastModified(final Instant lastModified) { + this.lastModified = lastModified; + return this; + } + + public SecurityConfig build() { + return new SecurityConfig(type, hash, lastModified); + } + + } + + public static SecurityConfig.Builder from(final SecurityConfig securityConfig) { + return new SecurityConfig.Builder(securityConfig); + } + +} diff --git a/src/main/java/org/opensearch/security/state/SecurityMetadata.java b/src/main/java/org/opensearch/security/state/SecurityMetadata.java new file mode 100644 index 0000000000..f8e2e043fd --- /dev/null +++ b/src/main/java/org/opensearch/security/state/SecurityMetadata.java @@ -0,0 +1,128 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.security.state; + +import java.io.IOException; +import java.time.Instant; +import java.util.Comparator; +import java.util.Objects; +import java.util.Set; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; + +import org.opensearch.Version; +import org.opensearch.cluster.AbstractNamedDiffable; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.NamedDiff; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.XContentBuilder; + +import static java.time.format.DateTimeFormatter.ISO_INSTANT; + +public final class SecurityMetadata extends AbstractNamedDiffable implements ClusterState.Custom { + + public final static String TYPE = "security"; + + private final Instant created; + + private final Set configuration; + + public SecurityMetadata(final Instant created, final Set configuration) { + this.created = created; + this.configuration = configuration; + } + + public SecurityMetadata(StreamInput in) throws IOException { + this.created = in.readInstant(); + this.configuration = in.readSet(SecurityConfig::new); + } + + public Instant created() { + return created; + } + + public Set configuration() { + return configuration; + } + + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT.minimumCompatibilityVersion(); + } + + @Override + public String getWriteableName() { + return TYPE; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInstant(created); + out.writeCollection(configuration); + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.field("created", ISO_INSTANT.format(created)); + xContentBuilder.startObject("configuration"); + for (final var securityConfig : configuration) { + securityConfig.toXContent(xContentBuilder, EMPTY_PARAMS); + } + return xContentBuilder.endObject(); + } + + public static NamedDiff readDiffFrom(StreamInput in) throws IOException { + return readDiffFrom(ClusterState.Custom.class, TYPE, in); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SecurityMetadata that = (SecurityMetadata) o; + return Objects.equals(created, that.created) && Objects.equals(configuration, that.configuration); + } + + @Override + public int hashCode() { + return Objects.hash(created, configuration); + } + + public final static class Builder { + + private final Instant created; + + private final ImmutableSet.Builder configuration = new ImmutableSortedSet.Builder<>( + Comparator.comparing(SecurityConfig::type) + ); + + Builder(SecurityMetadata oldMetadata) { + this.created = oldMetadata.created; + this.configuration.addAll(oldMetadata.configuration); + } + + public Builder withSecurityConfig(final SecurityConfig securityConfig) { + this.configuration.add(securityConfig); + return this; + } + + public SecurityMetadata build() { + return new SecurityMetadata(created, configuration.build()); + } + + } + + public static SecurityMetadata.Builder from(final SecurityMetadata securityMetadata) { + return new SecurityMetadata.Builder(securityMetadata); + } + +} diff --git a/src/main/java/org/opensearch/security/support/ConfigConstants.java b/src/main/java/org/opensearch/security/support/ConfigConstants.java index 3060e1b2dc..5169d02d20 100644 --- a/src/main/java/org/opensearch/security/support/ConfigConstants.java +++ b/src/main/java/org/opensearch/security/support/ConfigConstants.java @@ -220,9 +220,14 @@ public class ConfigConstants { public static final String SECURITY_NODES_DN = "plugins.security.nodes_dn"; public static final String SECURITY_NODES_DN_DYNAMIC_CONFIG_ENABLED = "plugins.security.nodes_dn_dynamic_config_enabled"; public static final String SECURITY_DISABLED = "plugins.security.disabled"; + public static final String SECURITY_CACHE_TTL_MINUTES = "plugins.security.cache.ttl_minutes"; public static final String SECURITY_ALLOW_UNSAFE_DEMOCERTIFICATES = "plugins.security.allow_unsafe_democertificates"; public static final String SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX = "plugins.security.allow_default_init_securityindex"; + + public static final String SECURITY_ALLOW_DEFAULT_INIT_USE_CLUSTER_STATE = + "plugins.security.allow_default_init_securityindex.use_cluster_state"; + public static final String SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST = "plugins.security.background_init_if_securityindex_not_exist"; diff --git a/src/main/java/org/opensearch/security/support/ConfigHelper.java b/src/main/java/org/opensearch/security/support/ConfigHelper.java index 4f310f6af7..e8526478f2 100644 --- a/src/main/java/org/opensearch/security/support/ConfigHelper.java +++ b/src/main/java/org/opensearch/security/support/ConfigHelper.java @@ -57,6 +57,7 @@ import static org.opensearch.core.xcontent.DeprecationHandler.THROW_UNSUPPORTED_OPERATION; +@Deprecated public class ConfigHelper { private static final Logger LOGGER = LogManager.getLogger(ConfigHelper.class); diff --git a/src/main/java/org/opensearch/security/support/SecurityIndexHandler.java b/src/main/java/org/opensearch/security/support/SecurityIndexHandler.java new file mode 100644 index 0000000000..1ed8a99614 --- /dev/null +++ b/src/main/java/org/opensearch/security/support/SecurityIndexHandler.java @@ -0,0 +1,233 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.security.support; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.hash.Hashing; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.get.MultiGetRequest; +import org.opensearch.action.get.MultiGetResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.security.DefaultObjectMapper; +import org.opensearch.security.securityconf.impl.CType; +import org.opensearch.security.securityconf.impl.SecurityDynamicConfiguration; +import org.opensearch.security.state.SecurityConfig; + +import static org.opensearch.core.xcontent.DeprecationHandler.THROW_UNSUPPORTED_OPERATION; +import static org.opensearch.security.configuration.ConfigurationRepository.DEFAULT_CONFIG_VERSION; +import static org.opensearch.security.support.YamlConfigReader.emptyJsonConfigFor; +import static org.opensearch.security.support.YamlConfigReader.yamlContentFor; + +public class SecurityIndexHandler { + + private final static int MINIMUM_HASH_BITS = 128; + + private static final Logger LOGGER = LogManager.getLogger(SecurityIndexHandler.class); + + private final Settings settings; + + private final Client client; + + private final String indexName; + + public SecurityIndexHandler(final String indexName, final Settings settings, final Client client) { + this.indexName = indexName; + this.settings = settings; + this.client = client; + } + + public final static Map INDEX_SETTINGS = Map.of("index.number_of_shards", 1, "index.auto_expand_replicas", "0-all"); + + public void createIndex(ActionListener listener) { + try (final ThreadContext.StoredContext threadContext = client.threadPool().getThreadContext().stashContext()) { + client.admin() + .indices() + .create( + new CreateIndexRequest(indexName).settings(INDEX_SETTINGS).waitForActiveShards(1), + ActionListener.runBefore(ActionListener.wrap(r -> { + if (r.isAcknowledged()) { + listener.onResponse(true); + } else listener.onFailure(new SecurityException("Couldn't create security index " + indexName)); + }, listener::onFailure), threadContext::restore) + ); + } + } + + public void uploadDefaultConfiguration(final Path configDir, final ActionListener> listener) { + try (final ThreadContext.StoredContext threadContext = client.threadPool().getThreadContext().stashContext()) { + AccessController.doPrivileged((PrivilegedAction) () -> { + try { + LOGGER.info("Uploading default security configuration from {}", configDir.toAbsolutePath()); + final var bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + final var configuration = new ImmutableSortedSet.Builder<>(Comparator.comparing(SecurityConfig::type)); + for (final var cType : CType.values()) { + final var fileExists = Files.exists(cType.configFile(configDir)); + // Audit config is not packaged by default and while list is deprecated + if ((cType == CType.AUDIT || cType == CType.WHITELIST) && !fileExists) continue; + if (cType == CType.WHITELIST) { + LOGGER.warn( + "WHITELIST configuration type is deprecated and will be replaced with ALLOWLIST in the next major version" + ); + } + final var yamlContent = yamlContentFor(cType, configDir); + final var hash = Hashing.goodFastHash(MINIMUM_HASH_BITS).hashBytes(yamlContent.toBytesRef().bytes); + configuration.add(new SecurityConfig(cType, hash.toString(), null)); + bulkRequest.add( + new IndexRequest(indexName).id(cType.toLCString()) + .opType(DocWriteRequest.OpType.INDEX) + .source(cType.toLCString(), yamlContent) + ); + } + client.bulk(bulkRequest, ActionListener.runBefore(ActionListener.wrap(r -> { + if (r.hasFailures()) { + listener.onFailure(new SecurityException(r.buildFailureMessage())); + return; + } + listener.onResponse(configuration.build()); + }, listener::onFailure), threadContext::restore)); + } catch (final IOException ioe) { + listener.onFailure(new SecurityException(ioe)); + } + return null; + }); + } + } + + public void loadConfiguration( + final Set configuration, + final ActionListener>> listener + ) { + try (final ThreadContext.StoredContext threadContext = client.threadPool().getThreadContext().stashContext()) { + client.threadPool().getThreadContext().putHeader(ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER, "true"); + final var configurationTypes = configuration.stream().map(SecurityConfig::type).collect(Collectors.toUnmodifiableList()); + client.multiGet(newMultiGetRequest(configurationTypes), ActionListener.runBefore(ActionListener.wrap(r -> { + final var cTypeConfigsBuilder = ImmutableMap.>builderWithExpectedSize( + configuration.size() + ); + var hasFailures = false; + for (final var item : r.getResponses()) { + if (item.isFailed()) { + listener.onFailure(new SecurityException(multiGetFailureMessage(item.getId(), item.getFailure()))); + hasFailures = true; + break; + } + final var cType = CType.fromString(item.getId()); + final var cTypeResponse = item.getResponse(); + if (cTypeResponse.isExists() && !cTypeResponse.isSourceEmpty()) { + final var config = buildDynamicConfiguration( + cType, + cTypeResponse.getSourceAsBytesRef(), + cTypeResponse.getSeqNo(), + cTypeResponse.getPrimaryTerm() + ); + if (config.getVersion() != DEFAULT_CONFIG_VERSION) { + listener.onFailure( + new SecurityException("Version " + config.getVersion() + " is not supported for " + cType.name()) + ); + hasFailures = true; + break; + } + cTypeConfigsBuilder.put(cType, config); + } else { + if (!cType.emptyIfMissing()) { + listener.onFailure(new SecurityException("Missing required configuration for type: " + cType)); + hasFailures = true; + break; + } + cTypeConfigsBuilder.put( + cType, + SecurityDynamicConfiguration.fromJson( + emptyJsonConfigFor(cType), + cType, + DEFAULT_CONFIG_VERSION, + cTypeResponse.getSeqNo(), + cTypeResponse.getPrimaryTerm() + ) + ); + } + } + if (!hasFailures) { + listener.onResponse(cTypeConfigsBuilder.build()); + } + }, listener::onFailure), threadContext::restore)); + } + } + + private MultiGetRequest newMultiGetRequest(final List configurationTypes) { + final var request = new MultiGetRequest().realtime(true).refresh(true); + for (final var cType : configurationTypes) { + request.add(indexName, cType.toLCString()); + } + return request; + } + + private SecurityDynamicConfiguration buildDynamicConfiguration( + final CType cType, + final BytesReference bytesRef, + final long seqNo, + final long primaryTerm + ) { + try { + final var source = SecurityUtils.replaceEnvVars(configTypeSource(bytesRef.streamInput()), settings); + final var jsonNode = DefaultObjectMapper.readTree(source); + var version = 1; + if (jsonNode.has("_meta")) { + if (jsonNode.get("_meta").has("config_version")) { + version = jsonNode.get("_meta").get("config_version").asInt(); + } + } + return SecurityDynamicConfiguration.fromJson(source, cType, version, seqNo, primaryTerm); + } catch (IOException e) { + throw new SecurityException("Couldn't parse content for " + cType, e); + } + } + + private String configTypeSource(final InputStream inputStream) throws IOException { + final var jsonContent = XContentType.JSON.xContent(); + try (final var parser = jsonContent.createParser(NamedXContentRegistry.EMPTY, THROW_UNSUPPORTED_OPERATION, inputStream)) { + parser.nextToken(); + parser.nextToken(); + parser.nextToken(); + return new String(parser.binaryValue(), StandardCharsets.UTF_8); + } + } + + private String multiGetFailureMessage(final String cTypeId, final MultiGetResponse.Failure failure) { + return String.format("Failure %s retrieving configuration for %s (index=%s)", failure, cTypeId, indexName); + } + +} diff --git a/src/main/java/org/opensearch/security/support/YamlConfigReader.java b/src/main/java/org/opensearch/security/support/YamlConfigReader.java new file mode 100644 index 0000000000..237e5b5bfb --- /dev/null +++ b/src/main/java/org/opensearch/security/support/YamlConfigReader.java @@ -0,0 +1,95 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.security.support; + +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.security.DefaultObjectMapper; +import org.opensearch.security.securityconf.impl.CType; +import org.opensearch.security.securityconf.impl.Meta; +import org.opensearch.security.securityconf.impl.SecurityDynamicConfiguration; + +import static org.opensearch.core.xcontent.DeprecationHandler.THROW_UNSUPPORTED_OPERATION; +import static org.opensearch.security.configuration.ConfigurationRepository.DEFAULT_CONFIG_VERSION; + +/** + * Read YAML security config files + */ +public final class YamlConfigReader { + + private static final Logger LOGGER = LogManager.getLogger(YamlConfigReader.class); + + public static BytesReference yamlContentFor(final CType cType, final Path configDir) throws IOException { + final var yamlXContent = XContentType.YAML.xContent(); + try ( + final var r = newReader(cType, configDir); + final var parser = yamlXContent.createParser(NamedXContentRegistry.EMPTY, THROW_UNSUPPORTED_OPERATION, r) + ) { + parser.nextToken(); + try (final var xContentBuilder = XContentFactory.jsonBuilder()) { + xContentBuilder.copyCurrentStructure(parser); + final var bytesRef = BytesReference.bytes(xContentBuilder); + validateYamlContent(cType, bytesRef.streamInput()); + return bytesRef; + } + } + } + + public static Reader newReader(final CType cType, final Path configDir) throws IOException { + final var cTypeFile = cType.configFile(configDir); + final var fileExists = Files.exists(cTypeFile); + if (!fileExists && !cType.emptyIfMissing()) { + throw new IOException("Couldn't find configuration file " + cTypeFile.getFileName()); + } + if (fileExists) { + LOGGER.info("Reading {} configuration from {}", cType, cTypeFile.getFileName()); + return new FileReader(cTypeFile.toFile(), StandardCharsets.UTF_8); + } else { + LOGGER.info("Reading empty {} configuration", cType); + return new StringReader(emptyYamlConfigFor(cType)); + } + } + + private static SecurityDynamicConfiguration emptyConfigFor(final CType cType) { + final var emptyConfiguration = SecurityDynamicConfiguration.empty(); + emptyConfiguration.setCType(cType); + emptyConfiguration.set_meta(new Meta()); + emptyConfiguration.get_meta().setConfig_version(DEFAULT_CONFIG_VERSION); + emptyConfiguration.get_meta().setType(cType.toLCString()); + return emptyConfiguration; + } + + public static String emptyJsonConfigFor(final CType cType) throws IOException { + return DefaultObjectMapper.writeValueAsString(emptyConfigFor(cType), false); + } + + public static String emptyYamlConfigFor(final CType cType) throws IOException { + return DefaultObjectMapper.YAML_MAPPER.writeValueAsString(emptyConfigFor(cType)); + } + + private static void validateYamlContent(final CType cType, final InputStream in) throws IOException { + SecurityDynamicConfiguration.fromNode(DefaultObjectMapper.YAML_MAPPER.readTree(in), cType, DEFAULT_CONFIG_VERSION, -1, -1); + } + +} diff --git a/src/test/java/org/opensearch/security/configuration/ConfigurationRepositoryTest.java b/src/test/java/org/opensearch/security/configuration/ConfigurationRepositoryTest.java index 5ce1873405..30cbbe6a01 100644 --- a/src/test/java/org/opensearch/security/configuration/ConfigurationRepositoryTest.java +++ b/src/test/java/org/opensearch/security/configuration/ConfigurationRepositoryTest.java @@ -13,22 +13,37 @@ import java.io.IOException; import java.nio.file.Path; +import java.time.Instant; +import java.util.Map; +import java.util.Set; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.ClusterStateUpdateTask; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.Priority; import org.opensearch.common.settings.Settings; +import org.opensearch.core.action.ActionListener; import org.opensearch.security.auditlog.AuditLog; import org.opensearch.security.securityconf.impl.CType; import org.opensearch.security.securityconf.impl.SecurityDynamicConfiguration; +import org.opensearch.security.state.SecurityConfig; +import org.opensearch.security.state.SecurityMetadata; import org.opensearch.security.support.ConfigConstants; +import org.opensearch.security.support.SecurityIndexHandler; import org.opensearch.security.transport.SecurityInterceptorTests; import org.opensearch.threadpool.ThreadPool; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; -import org.mockito.MockitoAnnotations; +import org.mockito.junit.MockitoJUnitRunner; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; @@ -36,7 +51,22 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; +import static org.opensearch.security.support.ConfigConstants.OPENDISTRO_SECURITY_DEFAULT_CONFIG_INDEX; +import static org.opensearch.security.support.ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; +@RunWith(MockitoJUnitRunner.class) public class ConfigurationRepositoryTest { @Mock @@ -50,21 +80,44 @@ public class ConfigurationRepositoryTest { private ThreadPool threadPool; + @Mock + private SecurityIndexHandler securityIndexHandler; + + @Mock + private ClusterChangedEvent event; + @Before public void setUp() { - MockitoAnnotations.openMocks(this); - Settings settings = Settings.builder() .put("node.name", SecurityInterceptorTests.class.getSimpleName()) .put("request.headers.default", "1") .build(); threadPool = new ThreadPool(settings); + + final var previousState = mock(ClusterState.class); + final var previousDiscoveryNodes = mock(DiscoveryNodes.class); + when(previousState.nodes()).thenReturn(previousDiscoveryNodes); + when(event.previousState()).thenReturn(previousState); + + final var newState = mock(ClusterState.class); + when(event.state()).thenReturn(newState); + when(event.state().metadata()).thenReturn(mock(Metadata.class)); + + when(event.state().custom(SecurityMetadata.TYPE)).thenReturn(null); } private ConfigurationRepository createConfigurationRepository(Settings settings) { - - return ConfigurationRepository.create(settings, path, threadPool, localClient, clusterService, auditLog); + return new ConfigurationRepository( + settings.get(ConfigConstants.SECURITY_CONFIG_INDEX_NAME, ConfigConstants.OPENDISTRO_SECURITY_DEFAULT_CONFIG_INDEX), + settings, + path, + threadPool, + localClient, + clusterService, + auditLog, + securityIndexHandler + ); } @Test @@ -77,7 +130,7 @@ public void create_shouldReturnConfigurationRepository() { @Test public void initOnNodeStart_withSecurityIndexCreationEnabledShouldSetInstallDefaultConfigTrue() { - Settings settings = Settings.builder().put(ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, true).build(); + Settings settings = Settings.builder().put(SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, true).build(); ConfigurationRepository configRepository = createConfigurationRepository(settings); @@ -111,4 +164,129 @@ public void getConfiguration_withInvalidConfigurationShouldReturnNewEmptyConfigu assertThat(config.getSeqNo(), is(equalTo(emptyConfig.getSeqNo()))); assertThat(config, is(not(equalTo(emptyConfig)))); } + + @Test + public void testClusterChanged_shouldInitSecurityIndexIfNoSecurityData() { + when(event.previousState().nodes().isLocalNodeElectedClusterManager()).thenReturn(false); + when(event.localNodeClusterManager()).thenReturn(true); + + final var configurationRepository = mock(ConfigurationRepository.class); + doCallRealMethod().when(configurationRepository).clusterChanged(any()); + configurationRepository.clusterChanged(event); + + verify(configurationRepository).initSecurityIndex(any()); + } + + @Test + public void testClusterChanged_shouldExecuteInitialization() { + when(event.state().custom(SecurityMetadata.TYPE)).thenReturn(new SecurityMetadata(Instant.now(), Set.of())); + + final var configurationRepository = mock(ConfigurationRepository.class); + doCallRealMethod().when(configurationRepository).clusterChanged(any()); + configurationRepository.clusterChanged(event); + + verify(configurationRepository).executeConfigurationInitialization(any()); + } + + @Test + public void testClusterChanged_shouldNotExecuteInitialization() { + final var configurationRepository = mock(ConfigurationRepository.class); + doCallRealMethod().when(configurationRepository).clusterChanged(any()); + configurationRepository.clusterChanged(event); + + verify(configurationRepository, never()).executeConfigurationInitialization(any()); + } + + @Test + public void testInitSecurityIndex_shouldCreateIndexAndUploadConfiguration() throws Exception { + System.setProperty("security.default_init.dir", Path.of(".").toString()); + ConfigurationRepository configRepository = createConfigurationRepository(Settings.EMPTY); + + doAnswer(invocation -> { + @SuppressWarnings("unchecked") + final var listener = (ActionListener) invocation.getArgument(0); + listener.onResponse(true); + return null; + }).when(securityIndexHandler).createIndex(any()); + doAnswer(invocation -> { + @SuppressWarnings("unchecked") + final var listener = (ActionListener>) invocation.getArgument(1); + listener.onResponse(Set.of(new SecurityConfig(CType.CONFIG, "aaa", null))); + return null; + }).when(securityIndexHandler).uploadDefaultConfiguration(any(), any()); + when(event.state().metadata().hasIndex(OPENDISTRO_SECURITY_DEFAULT_CONFIG_INDEX)).thenReturn(false); + configRepository.initSecurityIndex(event); + + final var clusterStateUpdateTaskCaptor = ArgumentCaptor.forClass(ClusterStateUpdateTask.class); + verify(securityIndexHandler).createIndex(any()); + verify(securityIndexHandler).uploadDefaultConfiguration(any(), any()); + verify(clusterService).submitStateUpdateTask(anyString(), clusterStateUpdateTaskCaptor.capture()); + verifyNoMoreInteractions(clusterService, securityIndexHandler); + + assertClusterState(clusterStateUpdateTaskCaptor); + } + + @Test + public void testInitSecurityIndex_shouldUploadConfigIfIndexCreated() throws Exception { + System.setProperty("security.default_init.dir", Path.of(".").toString()); + + doAnswer(invocation -> { + @SuppressWarnings("unchecked") + final var listener = (ActionListener>) invocation.getArgument(1); + listener.onResponse(Set.of(new SecurityConfig(CType.CONFIG, "aaa", null))); + return null; + }).when(securityIndexHandler).uploadDefaultConfiguration(any(), any()); + + when(event.state().metadata().hasIndex(OPENDISTRO_SECURITY_DEFAULT_CONFIG_INDEX)).thenReturn(true); + + ConfigurationRepository configRepository = createConfigurationRepository(Settings.EMPTY); + configRepository.initSecurityIndex(event); + + final var clusterStateUpdateTaskCaptor = ArgumentCaptor.forClass(ClusterStateUpdateTask.class); + + verify(event.state().metadata()).hasIndex(OPENDISTRO_SECURITY_DEFAULT_CONFIG_INDEX); + verify(clusterService).submitStateUpdateTask(anyString(), clusterStateUpdateTaskCaptor.capture()); + verify(securityIndexHandler, never()).createIndex(any()); + verify(securityIndexHandler).uploadDefaultConfiguration(any(), any()); + verifyNoMoreInteractions(securityIndexHandler, clusterService); + + assertClusterState(clusterStateUpdateTaskCaptor); + } + + @Test + public void testExecuteConfigurationInitialization_executeInitializationOnlyOnce() throws Exception { + doAnswer(invocation -> { + @SuppressWarnings("unchecked") + final var listener = (ActionListener>>) invocation.getArgument(1); + listener.onResponse(Map.of()); + return null; + }).when(securityIndexHandler).loadConfiguration(any(), any()); + + ConfigurationRepository configRepository = createConfigurationRepository(Settings.EMPTY); + configRepository.executeConfigurationInitialization( + new SecurityMetadata(Instant.now(), Set.of(new SecurityConfig(CType.CONFIG, "aaa", null))) + ).get(); + + verify(securityIndexHandler).loadConfiguration(any(), any()); + verifyNoMoreInteractions(securityIndexHandler); + + reset(securityIndexHandler); + + configRepository.executeConfigurationInitialization( + new SecurityMetadata(Instant.now(), Set.of(new SecurityConfig(CType.CONFIG, "aaa", null))) + ).get(); + + verify(securityIndexHandler, never()).loadConfiguration(any(), any()); + verifyNoMoreInteractions(securityIndexHandler); + } + + void assertClusterState(final ArgumentCaptor clusterStateUpdateTaskCaptor) throws Exception { + final var initializedStateUpdate = clusterStateUpdateTaskCaptor.getValue(); + assertEquals(Priority.IMMEDIATE, initializedStateUpdate.priority()); + var clusterState = initializedStateUpdate.execute(ClusterState.EMPTY_STATE); + SecurityMetadata securityMetadata = clusterState.custom(SecurityMetadata.TYPE); + assertNotNull(securityMetadata.created()); + assertNotNull(securityMetadata.configuration()); + } + } diff --git a/src/test/java/org/opensearch/security/state/SecurityMetadataSerializationTestCase.java b/src/test/java/org/opensearch/security/state/SecurityMetadataSerializationTestCase.java new file mode 100644 index 0000000000..c52f37cf54 --- /dev/null +++ b/src/test/java/org/opensearch/security/state/SecurityMetadataSerializationTestCase.java @@ -0,0 +1,154 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ +package org.opensearch.security.state; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +import com.carrotsearch.randomizedtesting.RandomizedContext; +import com.carrotsearch.randomizedtesting.RandomizedRunner; +import com.carrotsearch.randomizedtesting.RandomizedTest; +import com.google.common.collect.ImmutableSortedSet; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.Version; +import org.opensearch.cluster.ClusterState; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.security.securityconf.impl.CType; +import org.opensearch.test.DiffableTestUtils; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; + +@RunWith(RandomizedRunner.class) +public class SecurityMetadataSerializationTestCase extends RandomizedTest { + + protected ClusterState.Custom createTestInstance() { + final var configuration = new ImmutableSortedSet.Builder<>(Comparator.comparing(SecurityConfig::type)); + for (final var c : CType.values()) { + configuration.add(new SecurityConfig(c, randomAsciiAlphanumOfLength(128), null)); + } + return new SecurityMetadata(randomInstant(), configuration.build()); + } + + protected ClusterState.Custom makeTestChanges(ClusterState.Custom custom) { + final var securityMetadata = (SecurityMetadata) custom; + + if (randomBoolean()) { + final var configuration = securityMetadata.configuration(); + int leaveElements = randomIntBetween(0, configuration.size() - 1); + final var randomConfigs = randomSubsetOf(leaveElements, configuration); + final var securityMetadataBuilder = SecurityMetadata.from(securityMetadata); + for (final var config : randomConfigs) { + securityMetadataBuilder.withSecurityConfig( + SecurityConfig.from(config).withLastModified(randomInstant()).withHash(randomAsciiAlphanumOfLength(128)).build() + ); + } + return securityMetadataBuilder.build(); + } + + return securityMetadata; + } + + public static List randomSubsetOf(int size, Collection collection) { + if (size > collection.size()) { + throw new IllegalArgumentException( + "Can't pick " + size + " random objects from a collection of " + collection.size() + " objects" + ); + } + List tempList = new ArrayList<>(collection); + Collections.shuffle(tempList, RandomizedContext.current().getRandom()); + return tempList.subList(0, size); + } + + protected Instant randomInstant() { + return Instant.ofEpochSecond(randomLongBetween(0L, 3000000000L), randomLongBetween(0L, 999999999L)); + } + + @Test + public void testSerialization() throws IOException { + for (int runs = 0; runs < 20; runs++) { + ClusterState.Custom testInstance = createTestInstance(); + assertSerialization(testInstance); + } + } + + void assertSerialization(ClusterState.Custom testInstance) throws IOException { + assertSerialization(testInstance, Version.CURRENT); + } + + void assertSerialization(ClusterState.Custom testInstance, Version version) throws IOException { + ClusterState.Custom deserializedInstance = copyInstance(testInstance, version); + assertEqualInstances(testInstance, deserializedInstance); + } + + void assertEqualInstances(ClusterState.Custom expectedInstance, ClusterState.Custom newInstance) { + assertNotSame(newInstance, expectedInstance); + assertEquals(expectedInstance, newInstance); + assertEquals(expectedInstance.hashCode(), newInstance.hashCode()); + } + + @Test + public void testDiffableSerialization() throws IOException { + DiffableTestUtils.testDiffableSerialization( + this::createTestInstance, + this::makeTestChanges, + getNamedWriteableRegistry(), + SecurityMetadata::new, + SecurityMetadata::readDiffFrom + ); + } + + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(Collections.emptyList()); + } + + protected final ClusterState.Custom copyInstance(ClusterState.Custom instance, Version version) throws IOException { + return copyWriteable(instance, getNamedWriteableRegistry(), SecurityMetadata::new, version); + } + + public static T copyWriteable( + T original, + NamedWriteableRegistry namedWriteableRegistry, + Writeable.Reader reader, + Version version + ) throws IOException { + return copyInstance(original, namedWriteableRegistry, (out, value) -> value.writeTo(out), reader, version); + } + + protected static T copyInstance( + T original, + NamedWriteableRegistry namedWriteableRegistry, + Writeable.Writer writer, + Writeable.Reader reader, + Version version + ) throws IOException { + try (BytesStreamOutput output = new BytesStreamOutput()) { + output.setVersion(version); + writer.write(output, original); + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { + in.setVersion(version); + return reader.read(in); + } + } + } + +} diff --git a/src/test/java/org/opensearch/security/support/ConfigReaderTest.java b/src/test/java/org/opensearch/security/support/ConfigReaderTest.java new file mode 100644 index 0000000000..189b92ff68 --- /dev/null +++ b/src/test/java/org/opensearch/security/support/ConfigReaderTest.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.security.support; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; + +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import org.opensearch.security.DefaultObjectMapper; +import org.opensearch.security.securityconf.impl.CType; + +import static org.opensearch.security.configuration.ConfigurationRepository.DEFAULT_CONFIG_VERSION; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public class ConfigReaderTest { + + @ClassRule + public static TemporaryFolder folder = new TemporaryFolder(); + + private static File configDir; + + @BeforeClass + public static void createConfigFile() throws IOException { + configDir = folder.newFolder("config"); + } + + @Test + public void testThrowsIOExceptionForMandatoryCTypes() { + for (final var cType : CType.REQUIRED_CONFIG_FILES) { + assertThrows(IOException.class, () -> YamlConfigReader.newReader(cType, configDir.toPath())); + } + } + + @Test + public void testCreateReaderForNonMandatoryCTypes() throws IOException { + final var yamlMapper = DefaultObjectMapper.YAML_MAPPER; + for (final var cType : CType.NOT_REQUIRED_CONFIG_FILES) { + try (final var reader = new BufferedReader(YamlConfigReader.newReader(cType, configDir.toPath()))) { + final var emptyYaml = yamlMapper.readTree(reader); + assertTrue(emptyYaml.has("_meta")); + + final var meta = emptyYaml.get("_meta"); + assertEquals(cType.toLCString(), meta.get("type").asText()); + assertEquals(DEFAULT_CONFIG_VERSION, meta.get("config_version").asInt()); + } + } + } + +} diff --git a/src/test/java/org/opensearch/security/support/SecurityIndexHandlerTest.java b/src/test/java/org/opensearch/security/support/SecurityIndexHandlerTest.java new file mode 100644 index 0000000000..170f0a9853 --- /dev/null +++ b/src/test/java/org/opensearch/security/support/SecurityIndexHandlerTest.java @@ -0,0 +1,510 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.security.support; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.Set; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; + +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.bulk.BulkItemResponse; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.get.GetResponse; +import org.opensearch.action.get.MultiGetItemResponse; +import org.opensearch.action.get.MultiGetRequest; +import org.opensearch.action.get.MultiGetResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.support.ActiveShardCount; +import org.opensearch.client.AdminClient; +import org.opensearch.client.Client; +import org.opensearch.client.IndicesAdminClient; +import org.opensearch.common.CheckedSupplier; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.index.get.GetResult; +import org.opensearch.security.DefaultObjectMapper; +import org.opensearch.security.securityconf.impl.CType; +import org.opensearch.security.securityconf.impl.SecurityDynamicConfiguration; +import org.opensearch.security.state.SecurityConfig; +import org.opensearch.threadpool.ThreadPool; + +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.opensearch.security.configuration.ConfigurationRepository.DEFAULT_CONFIG_VERSION; +import static org.opensearch.security.support.YamlConfigReader.emptyYamlConfigFor; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.isA; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class SecurityIndexHandlerTest { + + final static String INDEX_NAME = "some_index"; + + final static String CONFIG_YAML = "_meta: \n" + + " type: \"config\"\n" + + " config_version: 2\n" + + "config:\n" + + " dynamic:\n" + + " http:\n" + + " anonymous_auth_enabled: false\n"; + + final static String USERS_YAML = "_meta:\n" + + " type: \"internalusers\"\n" + + " config_version: 2\n" + + "admin:\n" + + " hash: \"$2y$12$erlkZeSv7eRMa1vs3UgDl.xoqu1P9GY94Toj1BwdvJiq7eKTOjQjS\"\n" + + " reserved: true\n" + + " backend_roles:\n" + + " - \"admin\"\n" + + " description: \"Some admin user\"\n"; + + final static String ROLES_YAML = "_meta:\n" + " type: \"roles\"\n" + " config_version: 2\n" + "some_role:\n" + " reserved: true\n"; + + final static String ROLES_MAPPING_YAML = "_meta:\n" + + " type: \"rolesmapping\"\n" + + " config_version: 2\n" + + "all_access: \n" + + " reserved: false\n"; + + static final Map> YAML = Map.of( + CType.ACTIONGROUPS, + () -> emptyYamlConfigFor(CType.ACTIONGROUPS), + CType.ALLOWLIST, + () -> emptyYamlConfigFor(CType.ALLOWLIST), + CType.AUDIT, + () -> emptyYamlConfigFor(CType.AUDIT), + CType.CONFIG, + () -> CONFIG_YAML, + CType.INTERNALUSERS, + () -> USERS_YAML, + CType.NODESDN, + () -> emptyYamlConfigFor(CType.NODESDN), + CType.ROLES, + () -> ROLES_YAML, + CType.ROLESMAPPING, + () -> ROLES_MAPPING_YAML, + CType.TENANTS, + () -> emptyYamlConfigFor(CType.TENANTS), + CType.WHITELIST, + () -> emptyYamlConfigFor(CType.WHITELIST) + ); + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Mock + private Client client; + + @Mock + private ThreadPool threadPool; + + @Mock + private IndicesAdminClient indicesAdminClient; + + private Path configFolder; + + private ThreadContext threadContext; + + private SecurityIndexHandler securityIndexHandler; + + @Before + public void setupClient() throws IOException { + when(client.admin()).thenReturn(mock(AdminClient.class)); + when(client.admin().indices()).thenReturn(indicesAdminClient); + when(client.threadPool()).thenReturn(threadPool); + threadContext = new ThreadContext(Settings.EMPTY); + when(client.threadPool()).thenReturn(threadPool); + when(threadPool.getThreadContext()).thenReturn(threadContext); + configFolder = temporaryFolder.newFolder("config").toPath(); + securityIndexHandler = new SecurityIndexHandler(INDEX_NAME, Settings.EMPTY, client); + } + + @Test + public void testCreateIndex_shouldCreateIndex() { + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(1); + actionListener.onResponse(new CreateIndexResponse(true, true, "some_index")); + return null; + }).when(indicesAdminClient).create(any(), any()); + + securityIndexHandler.createIndex(ActionListener.wrap(Assert::assertTrue, Assert::assertNull)); + + final var requestCaptor = ArgumentCaptor.forClass(CreateIndexRequest.class); + + verify(indicesAdminClient).create(requestCaptor.capture(), any()); + + final var createRequest = requestCaptor.getValue(); + assertEquals(INDEX_NAME, createRequest.index()); + for (final var setting : SecurityIndexHandler.INDEX_SETTINGS.entrySet()) + assertEquals(setting.getValue().toString(), createRequest.settings().get(setting.getKey())); + + assertEquals(ActiveShardCount.ONE, createRequest.waitForActiveShards()); + } + + @Test + public void testCreateIndex_shouldReturnSecurityExceptionIfItCanNotCreateIndex() { + + final var listener = spy(ActionListener.wrap(r -> fail("Unexpected behave"), e -> { + assertEquals(SecurityException.class, e.getClass()); + assertEquals("Couldn't create security index " + INDEX_NAME, e.getMessage()); + })); + + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(1); + actionListener.onResponse(new CreateIndexResponse(false, false, "some_index")); + return null; + }).when(indicesAdminClient).create(any(), any()); + + securityIndexHandler.createIndex(listener); + + verify(indicesAdminClient).create(isA(CreateIndexRequest.class), any()); + verify(listener).onFailure(any()); + } + + @Test + public void testUploadDefaultConfiguration_shouldFailIfRequiredConfigFilesAreMissing() { + final var listener = spy(ActionListener.>wrap(r -> fail("Unexpected behave"), e -> { + assertEquals(SecurityException.class, e.getClass()); + assertThat(e.getMessage(), containsString("Couldn't find configuration file")); + })); + securityIndexHandler.uploadDefaultConfiguration(configFolder, listener); + + verify(listener).onFailure(any()); + } + + @Test + public void testUploadDefaultConfiguration_shouldFailIfBulkHasFailures() throws IOException { + final var failedBulkResponse = new BulkResponse( + new BulkItemResponse[] { + new BulkItemResponse(1, DocWriteRequest.OpType.CREATE, new BulkItemResponse.Failure("a", "b", new Exception())) }, + 100L + ); + final var listener = spy(ActionListener.>wrap(r -> fail("Unexpected behave"), e -> { + assertEquals(SecurityException.class, e.getClass()); + assertEquals(e.getMessage(), failedBulkResponse.buildFailureMessage()); + })); + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(1); + actionListener.onResponse(failedBulkResponse); + return null; + }).when(client).bulk(any(BulkRequest.class), any()); + for (final var c : CType.REQUIRED_CONFIG_FILES) { + try (final var io = Files.newBufferedWriter(c.configFile(configFolder))) { + io.write(YAML.get(c).get()); + io.flush(); + } + } + securityIndexHandler.uploadDefaultConfiguration(configFolder, listener); + verify(listener).onFailure(any()); + } + + @Test + public void testUploadDefaultConfiguration_shouldCreateSetOfSecurityConfigs() throws IOException { + + final var listener = spy(ActionListener.>wrap(configuration -> { + for (final var sc : configuration) { + assertTrue(sc.lastModified().isEmpty()); + assertNotNull(sc.hash()); + } + }, e -> fail("Unexpected behave"))); + + for (final var c : CType.REQUIRED_CONFIG_FILES) { + try (final var io = Files.newBufferedWriter(c.configFile(configFolder))) { + final var source = YAML.get(c).get(); + io.write(source); + io.flush(); + } + } + + final var bulkRequestCaptor = ArgumentCaptor.forClass(BulkRequest.class); + + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(1); + final var r = mock(BulkResponse.class); + when(r.hasFailures()).thenReturn(false); + actionListener.onResponse(r); + return null; + }).when(client).bulk(bulkRequestCaptor.capture(), any()); + securityIndexHandler.uploadDefaultConfiguration(configFolder, listener); + + final var bulkRequest = bulkRequestCaptor.getValue(); + for (final var r : bulkRequest.requests()) { + final var indexRequest = (IndexRequest) r; + assertEquals(INDEX_NAME, r.index()); + assertEquals(DocWriteRequest.OpType.INDEX, indexRequest.opType()); + } + verify(listener).onResponse(any()); + } + + @Test + public void testUploadDefaultConfiguration_shouldSkipAudit() throws IOException { + final var listener = spy( + ActionListener.>wrap( + configuration -> assertFalse(configuration.stream().anyMatch(sc -> sc.type() == CType.AUDIT)), + e -> fail("Unexpected behave") + ) + ); + + for (final var c : CType.REQUIRED_CONFIG_FILES) { + if (c == CType.AUDIT) continue; + try (final var io = Files.newBufferedWriter(c.configFile(configFolder))) { + final var source = YAML.get(c).get(); + io.write(source); + io.flush(); + } + } + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(1); + final var r = mock(BulkResponse.class); + when(r.hasFailures()).thenReturn(false); + actionListener.onResponse(r); + return null; + }).when(client).bulk(any(BulkRequest.class), any()); + + securityIndexHandler.uploadDefaultConfiguration(configFolder, listener); + verify(listener).onResponse(any()); + } + + @Test + public void testUploadDefaultConfiguration_shouldSkipWhitelist() throws IOException { + final var listener = spy( + ActionListener.>wrap( + configuration -> assertFalse(configuration.stream().anyMatch(sc -> sc.type() == CType.WHITELIST)), + e -> fail("Unexpected behave") + ) + ); + + for (final var c : CType.REQUIRED_CONFIG_FILES) { + if (c == CType.WHITELIST) continue; + try (final var io = Files.newBufferedWriter(c.configFile(configFolder))) { + final var source = YAML.get(c).get(); + io.write(source); + io.flush(); + } + } + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(1); + final var r = mock(BulkResponse.class); + when(r.hasFailures()).thenReturn(false); + actionListener.onResponse(r); + return null; + }).when(client).bulk(any(BulkRequest.class), any()); + + securityIndexHandler.uploadDefaultConfiguration(configFolder, listener); + verify(listener).onResponse(any()); + } + + @Test + public void testLoadConfiguration_shouldFailIfResponseHasFailures() { + final var listener = spy( + ActionListener.>>wrap( + r -> fail("Unexpected behave"), + e -> assertEquals(SecurityException.class, e.getClass()) + ) + ); + + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(1); + final var r = mock(MultiGetResponse.class); + final var mr = mock(MultiGetItemResponse.class); + when(mr.isFailed()).thenReturn(true); + when(mr.getFailure()).thenReturn(new MultiGetResponse.Failure("a", "id", new Exception())); + when(r.getResponses()).thenReturn(new MultiGetItemResponse[] { mr }); + actionListener.onResponse(r); + return null; + }).when(client).multiGet(any(MultiGetRequest.class), any()); + + securityIndexHandler.loadConfiguration(configuration(), listener); + verify(listener).onFailure(any()); + } + + @Test + public void testLoadConfiguration_shouldFailIfNoRequiredConfigInResponse() { + final var listener = spy( + ActionListener.>>wrap( + r -> fail("Unexpected behave"), + e -> assertEquals("Missing required configuration for type: CONFIG", e.getMessage()) + ) + ); + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(1); + final var getResult = mock(GetResult.class); + final var r = new MultiGetResponse(new MultiGetItemResponse[] { new MultiGetItemResponse(new GetResponse(getResult), null) }); + when(getResult.getId()).thenReturn(CType.CONFIG.toLCString()); + when(getResult.isExists()).thenReturn(false); + actionListener.onResponse(r); + return null; + }).when(client).multiGet(any(MultiGetRequest.class), any()); + + securityIndexHandler.loadConfiguration(configuration(), listener); + + verify(listener).onFailure(any()); + } + + @Test + public void testLoadConfiguration_shouldFailForUnsupportedVersion() { + final var listener = spy( + ActionListener.>>wrap( + r -> fail("Unexpected behave"), + e -> assertEquals("Version 1 is not supported for CONFIG", e.getMessage()) + ) + ); + doAnswer(invocation -> { + + final var objectMapper = DefaultObjectMapper.objectMapper; + + ActionListener actionListener = invocation.getArgument(1); + final var getResult = mock(GetResult.class); + final var r = new MultiGetResponse(new MultiGetItemResponse[] { new MultiGetItemResponse(new GetResponse(getResult), null) }); + when(getResult.getId()).thenReturn(CType.CONFIG.toLCString()); + when(getResult.isExists()).thenReturn(true); + + final var oldVersionJson = objectMapper.createObjectNode() + .set("opendistro_security", objectMapper.createObjectNode().set("dynamic", objectMapper.createObjectNode())) + .toString() + .getBytes(StandardCharsets.UTF_8); + final var configResponse = objectMapper.createObjectNode().put(CType.CONFIG.toLCString(), oldVersionJson); + final var source = objectMapper.writeValueAsBytes(configResponse); + when(getResult.sourceRef()).thenReturn(new BytesArray(source, 0, source.length)); + actionListener.onResponse(r); + return null; + }).when(client).multiGet(any(MultiGetRequest.class), any()); + securityIndexHandler.loadConfiguration(configuration(), listener); + + verify(listener).onFailure(any()); + } + + @Test + public void testLoadConfiguration_shouldFailForUnparseableConfig() { + final var listener = spy( + ActionListener.>>wrap( + r -> fail("Unexpected behave"), + e -> assertEquals("Couldn't parse content for CONFIG", e.getMessage()) + ) + ); + doAnswer(invocation -> { + + final var objectMapper = DefaultObjectMapper.objectMapper; + + ActionListener actionListener = invocation.getArgument(1); + final var getResult = mock(GetResult.class); + final var r = new MultiGetResponse(new MultiGetItemResponse[] { new MultiGetItemResponse(new GetResponse(getResult), null) }); + when(getResult.getId()).thenReturn(CType.CONFIG.toLCString()); + when(getResult.isExists()).thenReturn(true); + + final var configResponse = objectMapper.createObjectNode() + .put( + CType.CONFIG.toLCString(), + objectMapper.createObjectNode() + .set("_meta", objectMapper.createObjectNode().put("type", CType.CONFIG.toLCString())) + .toString() + .getBytes(StandardCharsets.UTF_8) + ); + final var source = objectMapper.writeValueAsBytes(configResponse); + when(getResult.sourceRef()).thenReturn(new BytesArray(source, 0, source.length)); + actionListener.onResponse(r); + return null; + }).when(client).multiGet(any(MultiGetRequest.class), any()); + securityIndexHandler.loadConfiguration(configuration(), listener); + + verify(listener).onFailure(any()); + } + + @Test + public void testLoadConfiguration_shouldBuildSecurityConfig() { + final var listener = spy(ActionListener.>>wrap(config -> { + assertEquals(CType.values().length, config.keySet().size()); + for (final var c : CType.values()) { + assertTrue(c.toLCString(), config.containsKey(c)); + } + }, e -> fail("Unexpected behave"))); + doAnswer(invocation -> { + final var objectMapper = DefaultObjectMapper.objectMapper; + ActionListener actionListener = invocation.getArgument(1); + + final var responses = new MultiGetItemResponse[CType.values().length]; + var counter = 0; + for (final var c : CType.values()) { + final var getResult = mock(GetResult.class); + if (!c.emptyIfMissing()) { + when(getResult.getId()).thenReturn(c.toLCString()); + when(getResult.isExists()).thenReturn(true); + + final var minimumRequiredConfig = minimumRequiredConfig(c); + if (c == CType.CONFIG) minimumRequiredConfig.set( + "config", + objectMapper.createObjectNode().set("dynamic", objectMapper.createObjectNode()) + ); + + final var source = objectMapper.writeValueAsBytes( + objectMapper.createObjectNode() + .put(c.toLCString(), minimumRequiredConfig.toString().getBytes(StandardCharsets.UTF_8)) + ); + + when(getResult.sourceRef()).thenReturn(new BytesArray(source, 0, source.length)); + + responses[counter] = new MultiGetItemResponse(new GetResponse(getResult), null); + } else { + when(getResult.getId()).thenReturn(c.toLCString()); + when(getResult.isExists()).thenReturn(false); + responses[counter] = new MultiGetItemResponse(new GetResponse(getResult), null); + } + counter++; + } + actionListener.onResponse(new MultiGetResponse(responses)); + return null; + }).when(client).multiGet(any(MultiGetRequest.class), any()); + securityIndexHandler.loadConfiguration(configuration(), listener); + + verify(listener).onResponse(any()); + } + + private ObjectNode minimumRequiredConfig(final CType cType) { + final var objectMapper = DefaultObjectMapper.objectMapper; + return objectMapper.createObjectNode() + .set("_meta", objectMapper.createObjectNode().put("type", cType.toLCString()).put("config_version", DEFAULT_CONFIG_VERSION)); + } + + private Set configuration() { + return Set.of(new SecurityConfig(CType.CONFIG, "aaa", null), new SecurityConfig(CType.AUDIT, "bbb", null)); + } + +} From 864d8be791a274658ada0c7b8a1a5fbe389fbbc7 Mon Sep 17 00:00:00 2001 From: Ashish Agrawal Date: Wed, 27 Mar 2024 13:50:46 -0700 Subject: [PATCH 161/204] Add new stop words system index (#4180) Signed-off-by: Ashish Agrawal --- .../security/tools/democonfig/SecuritySettingsConfigurer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java index e66215ac9a..72f0247e53 100644 --- a/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java +++ b/src/main/java/org/opensearch/security/tools/democonfig/SecuritySettingsConfigurer.java @@ -58,6 +58,7 @@ public class SecuritySettingsConfigurer { ".plugins-ml-conversation-interactions", ".plugins-ml-memory-meta", ".plugins-ml-memory-message", + ".plugins-ml-stop-words", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", From aa56d26af83e4c8db8be3501c4c68eb72c250620 Mon Sep 17 00:00:00 2001 From: Andrey Pleskach Date: Thu, 28 Mar 2024 00:55:53 +0100 Subject: [PATCH 162/204] Extract route paths prefixes into constants (#4173) Signed-off-by: Andrey Pleskach --- .../http/saml/AuthTokenProcessorHandler.java | 23 +++++++++++-------- .../onbehalf/CreateOnBehalfOfTokenAction.java | 3 ++- .../security/dlic/rest/support/Utils.java | 14 +++++++++-- .../security/rest/DashboardsInfoAction.java | 8 +++++-- .../security/rest/SecurityHealthAction.java | 6 +++-- .../security/rest/SecurityInfoAction.java | 6 +++-- .../security/rest/TenantInfoAction.java | 6 +++-- 7 files changed, 46 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/amazon/dlic/auth/http/saml/AuthTokenProcessorHandler.java b/src/main/java/com/amazon/dlic/auth/http/saml/AuthTokenProcessorHandler.java index 32e01b9e2f..6abe934925 100644 --- a/src/main/java/com/amazon/dlic/auth/http/saml/AuthTokenProcessorHandler.java +++ b/src/main/java/com/amazon/dlic/auth/http/saml/AuthTokenProcessorHandler.java @@ -137,7 +137,9 @@ private AuthTokenProcessorAction.Response handleImpl( String samlResponseBase64, String samlRequestId, String acsEndpoint, - Saml2Settings saml2Settings + Saml2Settings saml2Settings, + String requestPath // the parameter will be removed in the future as soon as we will read of legacy paths aka + // /_opendistro/_security/... ) { if (token_log.isDebugEnabled()) { try { @@ -156,7 +158,7 @@ private AuthTokenProcessorAction.Response handleImpl( final SamlResponse samlResponse = new SamlResponse(saml2Settings, acsEndpoint, samlResponseBase64); if (!samlResponse.isValid(samlRequestId)) { - log.warn("Error while validating SAML response in /_opendistro/_security/api/authtoken"); + log.warn("Error while validating SAML response in {}", requestPath); return null; } @@ -178,17 +180,14 @@ private Optional handleLowLevel(RestRequest restRequest) throw if (restRequest.getMediaType() != XContentType.JSON) { throw new OpenSearchSecurityException( - "/_opendistro/_security/api/authtoken expects content with type application/json", + restRequest.path() + " expects content with type application/json", RestStatus.UNSUPPORTED_MEDIA_TYPE ); } if (restRequest.method() != Method.POST) { - throw new OpenSearchSecurityException( - "/_opendistro/_security/api/authtoken expects POST requests", - RestStatus.METHOD_NOT_ALLOWED - ); + throw new OpenSearchSecurityException(restRequest.path() + " expects POST requests", RestStatus.METHOD_NOT_ALLOWED); } Saml2Settings saml2Settings = this.saml2SettingsProvider.getCached(); @@ -218,7 +217,13 @@ private Optional handleLowLevel(RestRequest restRequest) throw acsEndpoint = getAbsoluteAcsEndpoint(((ObjectNode) jsonRoot).get("acsEndpoint").textValue()); } - AuthTokenProcessorAction.Response responseBody = this.handleImpl(samlResponseBase64, samlRequestId, acsEndpoint, saml2Settings); + AuthTokenProcessorAction.Response responseBody = this.handleImpl( + samlResponseBase64, + samlRequestId, + acsEndpoint, + saml2Settings, + restRequest.path() + ); if (responseBody == null) { return Optional.empty(); @@ -228,7 +233,7 @@ private Optional handleLowLevel(RestRequest restRequest) throw return Optional.of(new SecurityResponse(HttpStatus.SC_OK, null, responseBodyString, XContentType.JSON.mediaType())); } catch (JsonProcessingException e) { - log.warn("Error while parsing JSON for /_opendistro/_security/api/authtoken", e); + log.warn("Error while parsing JSON for {}", restRequest.path(), e); return Optional.of(new SecurityResponse(HttpStatus.SC_BAD_REQUEST, "JSON could not be parsed")); } } diff --git a/src/main/java/org/opensearch/security/action/onbehalf/CreateOnBehalfOfTokenAction.java b/src/main/java/org/opensearch/security/action/onbehalf/CreateOnBehalfOfTokenAction.java index 02b88bbd5c..2e88418acf 100644 --- a/src/main/java/org/opensearch/security/action/onbehalf/CreateOnBehalfOfTokenAction.java +++ b/src/main/java/org/opensearch/security/action/onbehalf/CreateOnBehalfOfTokenAction.java @@ -32,13 +32,14 @@ import org.opensearch.security.identity.SecurityTokenManager; import static org.opensearch.rest.RestRequest.Method.POST; +import static org.opensearch.security.dlic.rest.support.Utils.PLUGIN_API_ROUTE_PREFIX; import static org.opensearch.security.dlic.rest.support.Utils.addRoutesPrefix; public class CreateOnBehalfOfTokenAction extends BaseRestHandler { private static final List routes = addRoutesPrefix( ImmutableList.of(new NamedRoute.Builder().method(POST).path("/generateonbehalfoftoken").uniqueName("security:obo/create").build()), - "/_plugins/_security/api" + PLUGIN_API_ROUTE_PREFIX ); public static final long OBO_DEFAULT_EXPIRY_SECONDS = 5 * 60; diff --git a/src/main/java/org/opensearch/security/dlic/rest/support/Utils.java b/src/main/java/org/opensearch/security/dlic/rest/support/Utils.java index 74b7cd415a..ee68a629c6 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/support/Utils.java +++ b/src/main/java/org/opensearch/security/dlic/rest/support/Utils.java @@ -55,9 +55,19 @@ import org.opensearch.security.user.User; import static org.opensearch.core.xcontent.DeprecationHandler.THROW_UNSUPPORTED_OPERATION; +import static org.opensearch.security.OpenSearchSecurityPlugin.LEGACY_OPENDISTRO_PREFIX; +import static org.opensearch.security.OpenSearchSecurityPlugin.PLUGINS_PREFIX; public class Utils { + public final static String PLUGIN_ROUTE_PREFIX = "/" + PLUGINS_PREFIX; + + public final static String LEGACY_PLUGIN_ROUTE_PREFIX = "/" + LEGACY_OPENDISTRO_PREFIX; + + public final static String PLUGIN_API_ROUTE_PREFIX = PLUGIN_ROUTE_PREFIX + "/api"; + + public final static String LEGACY_PLUGIN_API_ROUTE_PREFIX = LEGACY_PLUGIN_ROUTE_PREFIX + "/api"; + private static final ObjectMapper internalMapper = new ObjectMapper(); public static Map convertJsonToxToStructuredMap(ToXContent jsonContent) { @@ -217,7 +227,7 @@ public static Set generateFieldResourcePaths(final Set fields, f *Total number of routes is expanded as twice as the number of routes passed in */ public static List addRoutesPrefix(List routes) { - return addRoutesPrefix(routes, "/_opendistro/_security/api", "/_plugins/_security/api"); + return addRoutesPrefix(routes, LEGACY_PLUGIN_API_ROUTE_PREFIX, PLUGIN_API_ROUTE_PREFIX); } /** @@ -248,7 +258,7 @@ public static List addRoutesPrefix(List routes, final String... pr *Total number of routes is expanded as twice as the number of routes passed in */ public static List addDeprecatedRoutesPrefix(List deprecatedRoutes) { - return addDeprecatedRoutesPrefix(deprecatedRoutes, "/_opendistro/_security/api", "/_plugins/_security/api"); + return addDeprecatedRoutesPrefix(deprecatedRoutes, LEGACY_PLUGIN_API_ROUTE_PREFIX, PLUGIN_API_ROUTE_PREFIX); } /** diff --git a/src/main/java/org/opensearch/security/rest/DashboardsInfoAction.java b/src/main/java/org/opensearch/security/rest/DashboardsInfoAction.java index 070648ed92..3401ac71e8 100644 --- a/src/main/java/org/opensearch/security/rest/DashboardsInfoAction.java +++ b/src/main/java/org/opensearch/security/rest/DashboardsInfoAction.java @@ -50,15 +50,19 @@ import static org.opensearch.rest.RestRequest.Method.GET; import static org.opensearch.rest.RestRequest.Method.POST; +import static org.opensearch.security.dlic.rest.support.Utils.LEGACY_PLUGIN_ROUTE_PREFIX; +import static org.opensearch.security.dlic.rest.support.Utils.PLUGIN_ROUTE_PREFIX; import static org.opensearch.security.dlic.rest.support.Utils.addRoutesPrefix; public class DashboardsInfoAction extends BaseRestHandler { private static final List routes = ImmutableList.builder() .addAll( - addRoutesPrefix(ImmutableList.of(new Route(GET, "/dashboardsinfo"), new Route(POST, "/dashboardsinfo")), "/_plugins/_security") + addRoutesPrefix(ImmutableList.of(new Route(GET, "/dashboardsinfo"), new Route(POST, "/dashboardsinfo")), PLUGIN_ROUTE_PREFIX) + ) + .addAll( + addRoutesPrefix(ImmutableList.of(new Route(GET, "/kibanainfo"), new Route(POST, "/kibanainfo")), LEGACY_PLUGIN_ROUTE_PREFIX) ) - .addAll(addRoutesPrefix(ImmutableList.of(new Route(GET, "/kibanainfo"), new Route(POST, "/kibanainfo")), "/_opendistro/_security")) .build(); private final Logger log = LogManager.getLogger(this.getClass()); diff --git a/src/main/java/org/opensearch/security/rest/SecurityHealthAction.java b/src/main/java/org/opensearch/security/rest/SecurityHealthAction.java index 1b7e788dae..3c57773417 100644 --- a/src/main/java/org/opensearch/security/rest/SecurityHealthAction.java +++ b/src/main/java/org/opensearch/security/rest/SecurityHealthAction.java @@ -44,13 +44,15 @@ import static org.opensearch.rest.RestRequest.Method.GET; import static org.opensearch.rest.RestRequest.Method.POST; +import static org.opensearch.security.dlic.rest.support.Utils.LEGACY_PLUGIN_ROUTE_PREFIX; +import static org.opensearch.security.dlic.rest.support.Utils.PLUGIN_ROUTE_PREFIX; import static org.opensearch.security.dlic.rest.support.Utils.addRoutesPrefix; public class SecurityHealthAction extends BaseRestHandler { private static final List routes = addRoutesPrefix( ImmutableList.of(new Route(GET, "/health"), new Route(POST, "/health")), - "/_opendistro/_security", - "/_plugins/_security" + LEGACY_PLUGIN_ROUTE_PREFIX, + PLUGIN_ROUTE_PREFIX ); private final BackendRegistry registry; diff --git a/src/main/java/org/opensearch/security/rest/SecurityInfoAction.java b/src/main/java/org/opensearch/security/rest/SecurityInfoAction.java index 469c7f81b4..f6cf7f82ee 100644 --- a/src/main/java/org/opensearch/security/rest/SecurityInfoAction.java +++ b/src/main/java/org/opensearch/security/rest/SecurityInfoAction.java @@ -57,13 +57,15 @@ import static org.opensearch.rest.RestRequest.Method.GET; import static org.opensearch.rest.RestRequest.Method.POST; +import static org.opensearch.security.dlic.rest.support.Utils.LEGACY_PLUGIN_ROUTE_PREFIX; +import static org.opensearch.security.dlic.rest.support.Utils.PLUGIN_ROUTE_PREFIX; import static org.opensearch.security.dlic.rest.support.Utils.addRoutesPrefix; public class SecurityInfoAction extends BaseRestHandler { private static final List routes = addRoutesPrefix( ImmutableList.of(new Route(GET, "/authinfo"), new Route(POST, "/authinfo")), - "/_opendistro/_security", - "/_plugins/_security" + LEGACY_PLUGIN_ROUTE_PREFIX, + PLUGIN_ROUTE_PREFIX ); private final Logger log = LogManager.getLogger(this.getClass()); diff --git a/src/main/java/org/opensearch/security/rest/TenantInfoAction.java b/src/main/java/org/opensearch/security/rest/TenantInfoAction.java index 1b0bdd7f8e..bd911463d4 100644 --- a/src/main/java/org/opensearch/security/rest/TenantInfoAction.java +++ b/src/main/java/org/opensearch/security/rest/TenantInfoAction.java @@ -61,13 +61,15 @@ import static org.opensearch.rest.RestRequest.Method.GET; import static org.opensearch.rest.RestRequest.Method.POST; +import static org.opensearch.security.dlic.rest.support.Utils.LEGACY_PLUGIN_ROUTE_PREFIX; +import static org.opensearch.security.dlic.rest.support.Utils.PLUGIN_ROUTE_PREFIX; import static org.opensearch.security.dlic.rest.support.Utils.addRoutesPrefix; public class TenantInfoAction extends BaseRestHandler { private static final List routes = addRoutesPrefix( ImmutableList.of(new Route(GET, "/tenantinfo"), new Route(POST, "/tenantinfo")), - "/_opendistro/_security", - "/_plugins/_security" + LEGACY_PLUGIN_ROUTE_PREFIX, + PLUGIN_ROUTE_PREFIX ); private final Logger log = LogManager.getLogger(this.getClass()); From a870b40919b1a693ec810b50eda783d334b7a684 Mon Sep 17 00:00:00 2001 From: Andrey Pleskach Date: Thu, 28 Mar 2024 14:25:13 +0100 Subject: [PATCH 163/204] Misc changes for tests (#4171) Signed-off-by: Andrey Pleskach --- .../http/AnonymousAuthenticationTest.java | 7 +- .../opensearch/security/http/AsyncTests.java | 3 +- .../http/CertificateAuthenticationTest.java | 4 +- .../http/CommonProxyAuthenticationTests.java | 13 +- .../security/http/LdapAuthenticationTest.java | 3 +- .../http/LdapTlsAuthenticationTest.java | 6 +- .../http/OnBehalfOfJwtAuthenticationTest.java | 3 +- .../security/http/RolesMappingTests.java | 6 +- .../test/framework/RolesMapping.java | 108 ------- .../test/framework/TestSecurityConfig.java | 279 +++++++++++++++++- .../test/framework/cluster/LocalCluster.java | 3 +- .../cluster/LocalOpenSearchCluster.java | 16 +- .../framework/cluster/TestRestClient.java | 8 + 13 files changed, 311 insertions(+), 148 deletions(-) delete mode 100644 src/integrationTest/java/org/opensearch/test/framework/RolesMapping.java diff --git a/src/integrationTest/java/org/opensearch/security/http/AnonymousAuthenticationTest.java b/src/integrationTest/java/org/opensearch/security/http/AnonymousAuthenticationTest.java index b1c13aeedc..99f96a388e 100644 --- a/src/integrationTest/java/org/opensearch/security/http/AnonymousAuthenticationTest.java +++ b/src/integrationTest/java/org/opensearch/security/http/AnonymousAuthenticationTest.java @@ -17,7 +17,6 @@ import org.junit.Test; import org.junit.runner.RunWith; -import org.opensearch.test.framework.RolesMapping; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.cluster.ClusterManager; import org.opensearch.test.framework.cluster.LocalCluster; @@ -45,9 +44,9 @@ public class AnonymousAuthenticationTest { /** * Maps {@link #ANONYMOUS_USER_CUSTOM_ROLE} to {@link #DEFAULT_ANONYMOUS_USER_BACKEND_ROLE_NAME} */ - private static final RolesMapping ANONYMOUS_USER_CUSTOM_ROLE_MAPPING = new RolesMapping(ANONYMOUS_USER_CUSTOM_ROLE).backendRoles( - DEFAULT_ANONYMOUS_USER_BACKEND_ROLE_NAME - ); + private static final TestSecurityConfig.RoleMapping ANONYMOUS_USER_CUSTOM_ROLE_MAPPING = new TestSecurityConfig.RoleMapping( + ANONYMOUS_USER_CUSTOM_ROLE.getName() + ).backendRoles(DEFAULT_ANONYMOUS_USER_BACKEND_ROLE_NAME); /** * User who is stored in the internal user database and can authenticate diff --git a/src/integrationTest/java/org/opensearch/security/http/AsyncTests.java b/src/integrationTest/java/org/opensearch/security/http/AsyncTests.java index 8103d50e74..514d2c45d1 100644 --- a/src/integrationTest/java/org/opensearch/security/http/AsyncTests.java +++ b/src/integrationTest/java/org/opensearch/security/http/AsyncTests.java @@ -26,7 +26,6 @@ import org.opensearch.security.IndexOperationsHelper; import org.opensearch.security.support.ConfigConstants; import org.opensearch.test.framework.AsyncActions; -import org.opensearch.test.framework.RolesMapping; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; @@ -44,7 +43,7 @@ public class AsyncTests { public static LocalCluster cluster = new LocalCluster.Builder().singleNode() .authc(AUTHC_HTTPBASIC_INTERNAL) .users(ADMIN_USER) - .rolesMapping(new RolesMapping(ALL_ACCESS).backendRoles("admin")) + .rolesMapping(new TestSecurityConfig.RoleMapping(ALL_ACCESS.getName()).backendRoles("admin")) .anonymousAuth(false) .nodeSettings(Map.of(ConfigConstants.SECURITY_RESTAPI_ROLES_ENABLED, List.of(ALL_ACCESS.getName()))) .build(); diff --git a/src/integrationTest/java/org/opensearch/security/http/CertificateAuthenticationTest.java b/src/integrationTest/java/org/opensearch/security/http/CertificateAuthenticationTest.java index 975ce25efb..18ae232a91 100644 --- a/src/integrationTest/java/org/opensearch/security/http/CertificateAuthenticationTest.java +++ b/src/integrationTest/java/org/opensearch/security/http/CertificateAuthenticationTest.java @@ -17,7 +17,7 @@ import org.junit.Test; import org.junit.runner.RunWith; -import org.opensearch.test.framework.RolesMapping; +import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain; import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.HttpAuthenticator; import org.opensearch.test.framework.TestSecurityConfig.Role; @@ -69,7 +69,7 @@ public class CertificateAuthenticationTest { .authc(AUTHC_HTTPBASIC_INTERNAL) .roles(ROLE_ALL_INDEX_SEARCH) .users(USER_ADMIN) - .rolesMapping(new RolesMapping(ROLE_ALL_INDEX_SEARCH).backendRoles(BACKEND_ROLE_BRIDGE)) + .rolesMapping(new TestSecurityConfig.RoleMapping(ROLE_ALL_INDEX_SEARCH.getName()).backendRoles(BACKEND_ROLE_BRIDGE)) .build(); private static final TestCertificates TEST_CERTIFICATES = cluster.getTestCertificates(); diff --git a/src/integrationTest/java/org/opensearch/security/http/CommonProxyAuthenticationTests.java b/src/integrationTest/java/org/opensearch/security/http/CommonProxyAuthenticationTests.java index d98acf8895..48ed08ac22 100644 --- a/src/integrationTest/java/org/opensearch/security/http/CommonProxyAuthenticationTests.java +++ b/src/integrationTest/java/org/opensearch/security/http/CommonProxyAuthenticationTests.java @@ -13,7 +13,6 @@ import java.net.InetAddress; import java.util.List; -import org.opensearch.test.framework.RolesMapping; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; @@ -84,13 +83,13 @@ abstract class CommonProxyAuthenticationTests { .indexPermissions("indices:data/read/search") .on(PERSONAL_INDEX_NAME_PATTERN); - protected static final RolesMapping ROLES_MAPPING_CAPTAIN = new RolesMapping(ROLE_PERSONAL_INDEX_SEARCH).backendRoles( - BACKEND_ROLE_CAPTAIN - ); + protected static final TestSecurityConfig.RoleMapping ROLES_MAPPING_CAPTAIN = new TestSecurityConfig.RoleMapping( + ROLE_PERSONAL_INDEX_SEARCH.getName() + ).backendRoles(BACKEND_ROLE_CAPTAIN); - protected static final RolesMapping ROLES_MAPPING_FIRST_MATE = new RolesMapping(ROLE_ALL_INDEX_SEARCH).backendRoles( - BACKEND_ROLE_FIRST_MATE - ); + protected static final TestSecurityConfig.RoleMapping ROLES_MAPPING_FIRST_MATE = new TestSecurityConfig.RoleMapping( + ROLE_ALL_INDEX_SEARCH.getName() + ).backendRoles(BACKEND_ROLE_FIRST_MATE); protected abstract LocalCluster getCluster(); diff --git a/src/integrationTest/java/org/opensearch/security/http/LdapAuthenticationTest.java b/src/integrationTest/java/org/opensearch/security/http/LdapAuthenticationTest.java index 7339808d8c..090762af21 100644 --- a/src/integrationTest/java/org/opensearch/security/http/LdapAuthenticationTest.java +++ b/src/integrationTest/java/org/opensearch/security/http/LdapAuthenticationTest.java @@ -27,7 +27,6 @@ import org.opensearch.test.framework.AuthzDomain; import org.opensearch.test.framework.LdapAuthenticationConfigBuilder; import org.opensearch.test.framework.LdapAuthorizationConfigBuilder; -import org.opensearch.test.framework.RolesMapping; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain; import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AuthenticationBackend; @@ -118,7 +117,7 @@ public class LdapAuthenticationTest { ) .authc(AUTHC_HTTPBASIC_INTERNAL) .users(ADMIN_USER) - .rolesMapping(new RolesMapping(ALL_ACCESS).backendRoles(CN_GROUP_ADMIN)) + .rolesMapping(new TestSecurityConfig.RoleMapping(ALL_ACCESS.getName()).backendRoles(CN_GROUP_ADMIN)) .authz( new AuthzDomain("ldap_roles").httpEnabled(true) .authorizationBackend( diff --git a/src/integrationTest/java/org/opensearch/security/http/LdapTlsAuthenticationTest.java b/src/integrationTest/java/org/opensearch/security/http/LdapTlsAuthenticationTest.java index 32265f4b81..e5c1012bdc 100644 --- a/src/integrationTest/java/org/opensearch/security/http/LdapTlsAuthenticationTest.java +++ b/src/integrationTest/java/org/opensearch/security/http/LdapTlsAuthenticationTest.java @@ -30,7 +30,7 @@ import org.opensearch.test.framework.AuthzDomain; import org.opensearch.test.framework.LdapAuthenticationConfigBuilder; import org.opensearch.test.framework.LdapAuthorizationConfigBuilder; -import org.opensearch.test.framework.RolesMapping; +import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain; import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AuthenticationBackend; import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.HttpAuthenticator; @@ -151,8 +151,8 @@ public class LdapTlsAuthenticationTest { .users(ADMIN_USER) .roles(ROLE_INDEX_ADMINISTRATOR, ROLE_PERSONAL_INDEX_ACCESS) .rolesMapping( - new RolesMapping(ROLE_INDEX_ADMINISTRATOR).backendRoles(CN_GROUP_ADMIN), - new RolesMapping(ROLE_PERSONAL_INDEX_ACCESS).backendRoles(CN_GROUP_CREW) + new TestSecurityConfig.RoleMapping(ROLE_INDEX_ADMINISTRATOR.getName()).backendRoles(CN_GROUP_ADMIN), + new TestSecurityConfig.RoleMapping(ROLE_PERSONAL_INDEX_ACCESS.getName()).backendRoles(CN_GROUP_CREW) ) .authz( new AuthzDomain("ldap_roles").httpEnabled(true) diff --git a/src/integrationTest/java/org/opensearch/security/http/OnBehalfOfJwtAuthenticationTest.java b/src/integrationTest/java/org/opensearch/security/http/OnBehalfOfJwtAuthenticationTest.java index 7210c53ad4..1dbb10b1f8 100644 --- a/src/integrationTest/java/org/opensearch/security/http/OnBehalfOfJwtAuthenticationTest.java +++ b/src/integrationTest/java/org/opensearch/security/http/OnBehalfOfJwtAuthenticationTest.java @@ -33,7 +33,6 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.security.authtoken.jwt.EncryptionDecryptionUtil; import org.opensearch.test.framework.OnBehalfOfConfig; -import org.opensearch.test.framework.RolesMapping; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.cluster.ClusterManager; import org.opensearch.test.framework.cluster.LocalCluster; @@ -139,7 +138,7 @@ private static OnBehalfOfConfig defaultOnBehalfOfConfig() { ) ) .authc(AUTHC_HTTPBASIC_INTERNAL) - .rolesMapping(new RolesMapping(HOST_MAPPING_ROLE).hostIPs(HOST_MAPPING_IP)) + .rolesMapping(new TestSecurityConfig.RoleMapping(HOST_MAPPING_ROLE.getName()).hosts(HOST_MAPPING_IP)) .onBehalfOf(defaultOnBehalfOfConfig()) .build(); diff --git a/src/integrationTest/java/org/opensearch/security/http/RolesMappingTests.java b/src/integrationTest/java/org/opensearch/security/http/RolesMappingTests.java index 3907be4153..828790e47b 100644 --- a/src/integrationTest/java/org/opensearch/security/http/RolesMappingTests.java +++ b/src/integrationTest/java/org/opensearch/security/http/RolesMappingTests.java @@ -16,7 +16,6 @@ import org.junit.Test; import org.junit.runner.RunWith; -import org.opensearch.test.framework.RolesMapping; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.cluster.ClusterManager; import org.opensearch.test.framework.cluster.LocalCluster; @@ -48,7 +47,10 @@ public class RolesMappingTests { .anonymousAuth(false) .authc(AUTHC_DOMAIN) .roles(ROLE_A, ROLE_B) - .rolesMapping(new RolesMapping(ROLE_A).backendRoles("mapsToRoleA"), new RolesMapping(ROLE_B).backendRoles("mapsToRoleB")) + .rolesMapping( + new TestSecurityConfig.RoleMapping(ROLE_A.getName()).backendRoles("mapsToRoleA"), + new TestSecurityConfig.RoleMapping(ROLE_B.getName()).backendRoles("mapsToRoleB") + ) .users(USER_A, USER_B) .build(); diff --git a/src/integrationTest/java/org/opensearch/test/framework/RolesMapping.java b/src/integrationTest/java/org/opensearch/test/framework/RolesMapping.java deleted file mode 100644 index 997e7e128b..0000000000 --- a/src/integrationTest/java/org/opensearch/test/framework/RolesMapping.java +++ /dev/null @@ -1,108 +0,0 @@ -/* -* Copyright OpenSearch Contributors -* SPDX-License-Identifier: Apache-2.0 -* -* The OpenSearch Contributors require contributions made to -* this file be licensed under the Apache-2.0 license or a -* compatible open source license. -* -*/ -package org.opensearch.test.framework; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import org.opensearch.core.xcontent.ToXContentObject; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.test.framework.TestSecurityConfig.Role; - -import static java.util.Objects.requireNonNull; - -/** -* The class represents mapping between backend roles {@link #backendRoles} to OpenSearch role defined by field {@link #roleName}. The -* class provides convenient builder-like methods and can be serialized to JSON. Serialization to JSON is required to store the class -* in an OpenSearch index which contains Security plugin configuration. -*/ -public class RolesMapping implements ToXContentObject { - - /** - * OpenSearch role name - */ - private String roleName; - - /** - * Backend role names - */ - private List backendRoles; - private List hostIPs; - - private boolean reserved = false; - - /** - * Creates roles mapping to OpenSearch role defined by parameter role - * @param role OpenSearch role, must not be null. - */ - public RolesMapping(Role role) { - requireNonNull(role); - this.roleName = requireNonNull(role.getName()); - this.backendRoles = new ArrayList<>(); - this.hostIPs = new ArrayList<>(); - } - - /** - * Defines backend role names - * @param backendRoles backend roles names - * @return current {@link RolesMapping} instance - */ - public RolesMapping backendRoles(String... backendRoles) { - this.backendRoles.addAll(Arrays.asList(backendRoles)); - return this; - } - - /** - * Defines host IP address - * @param hostIPs host IP address - * @return current {@link RolesMapping} instance - */ - public RolesMapping hostIPs(String... hostIPs) { - this.hostIPs.addAll(Arrays.asList(hostIPs)); - return this; - } - - /** - * Determines if role is reserved - * @param reserved true for reserved roles - * @return current {@link RolesMapping} instance - */ - public RolesMapping reserved(boolean reserved) { - this.reserved = reserved; - return this; - } - - /** - * Returns OpenSearch role name - * @return role name - */ - public String getRoleName() { - return roleName; - } - - /** - * Controls serialization to JSON - * @param xContentBuilder must not be null - * @param params not used parameter, but required by the interface {@link ToXContentObject} - * @return builder form parameter xContentBuilder - * @throws IOException denotes error during serialization to JSON - */ - @Override - public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { - xContentBuilder.startObject(); - xContentBuilder.field("reserved", reserved); - xContentBuilder.field("backend_roles", backendRoles); - xContentBuilder.field("hosts", hostIPs); - xContentBuilder.endObject(); - return xContentBuilder; - } -} diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java index 7957d1cfa4..22aab70521 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java @@ -53,6 +53,7 @@ import org.opensearch.action.update.UpdateRequest; import org.opensearch.client.Client; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; @@ -81,7 +82,9 @@ public class TestSecurityConfig { private Map internalUsers = new LinkedHashMap<>(); private Map roles = new LinkedHashMap<>(); private AuditConfiguration auditConfiguration; - private Map rolesMapping = new LinkedHashMap<>(); + private Map rolesMapping = new LinkedHashMap<>(); + + private Map actionGroups = new LinkedHashMap<>(); private String indexName = ".opendistro_security"; @@ -159,9 +162,9 @@ public TestSecurityConfig audit(AuditConfiguration auditConfiguration) { return this; } - public TestSecurityConfig rolesMapping(RolesMapping... mappings) { - for (RolesMapping mapping : mappings) { - String roleName = mapping.getRoleName(); + public TestSecurityConfig rolesMapping(RoleMapping... mappings) { + for (RoleMapping mapping : mappings) { + String roleName = mapping.name(); if (rolesMapping.containsKey(roleName)) { throw new IllegalArgumentException("Role mapping " + roleName + " already exists"); } @@ -170,6 +173,13 @@ public TestSecurityConfig rolesMapping(RolesMapping... mappings) { return this; } + public TestSecurityConfig actionGroups(ActionGroup... groups) { + for (final var group : groups) { + this.actionGroups.put(group.name, group); + } + return this; + } + public static class Config implements ToXContentObject { private boolean anonymousAuth; @@ -252,7 +262,88 @@ public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params } } - public static class User implements UserCredentialsHolder, ToXContentObject { + public static final class ActionGroup implements ToXContentObject { + + public enum Type { + + INDEX, + + CLUSTER; + + public String type() { + return name().toLowerCase(); + } + + } + + private final String name; + + private final String description; + + private final Type type; + + private final List allowedActions; + + private Boolean hidden = null; + + private Boolean reserved = null; + + public ActionGroup(String name, Type type, String... allowedActions) { + this(name, null, type, allowedActions); + } + + public ActionGroup(String name, String description, Type type, String... allowedActions) { + this.name = name; + this.description = description; + this.type = type; + this.allowedActions = Arrays.asList(allowedActions); + } + + public String name() { + return name; + } + + public ActionGroup hidden(boolean hidden) { + this.hidden = hidden; + return this; + } + + public ActionGroup reserved(boolean reserved) { + this.reserved = reserved; + return this; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (hidden != null) builder.field("hidden", hidden); + if (reserved != null) builder.field("reserved", reserved); + builder.field("type", type.type()); + builder.field("allowed_actions", allowedActions); + if (description != null) builder.field("description", description); + return builder.endObject(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ActionGroup that = (ActionGroup) o; + return Objects.equals(name, that.name) + && Objects.equals(description, that.description) + && type == that.type + && Objects.equals(allowedActions, that.allowedActions) + && Objects.equals(hidden, that.hidden) + && Objects.equals(reserved, that.reserved); + } + + @Override + public int hashCode() { + return Objects.hash(name, description, type, allowedActions, hidden, reserved); + } + } + + public static final class User implements UserCredentialsHolder, ToXContentObject { public final static TestSecurityConfig.User USER_ADMIN = new TestSecurityConfig.User("admin").roles( new Role("allaccess").indexPermissions("*").on("*").clusterPermissions("*") @@ -265,9 +356,20 @@ public static class User implements UserCredentialsHolder, ToXContentObject { String requestedTenant; private Map attributes = new HashMap<>(); + private Boolean hidden = null; + + private Boolean reserved = null; + + private String description; + public User(String name) { + this(name, null); + } + + public User(String name, String description) { this.name = name; this.password = "secret"; + this.description = description; } public User password(String password) { @@ -289,6 +391,16 @@ public User backendRoles(String... backendRoles) { return this; } + public User reserved(boolean reserved) { + this.reserved = reserved; + return this; + } + + public User hidden(boolean hidden) { + this.hidden = hidden; + return this; + } + public User attr(String key, String value) { this.attributes.put(key, value); return this; @@ -330,9 +442,33 @@ public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params xContentBuilder.field("attributes", attributes); } + if (hidden != null) xContentBuilder.field("hidden", hidden); + if (reserved != null) xContentBuilder.field("reserved", reserved); + if (!Strings.isNullOrEmpty(description)) xContentBuilder.field("description", description); xContentBuilder.endObject(); return xContentBuilder; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + User user = (User) o; + return Objects.equals(name, user.name) + && Objects.equals(password, user.password) + && Objects.equals(roles, user.roles) + && Objects.equals(backendRoles, user.backendRoles) + && Objects.equals(requestedTenant, user.requestedTenant) + && Objects.equals(attributes, user.attributes) + && Objects.equals(hidden, user.hidden) + && Objects.equals(reserved, user.reserved) + && Objects.equals(description, user.description); + } + + @Override + public int hashCode() { + return Objects.hash(name, password, roles, backendRoles, requestedTenant, attributes, hidden, reserved, description); + } } public static class Role implements ToXContentObject { @@ -343,8 +479,19 @@ public static class Role implements ToXContentObject { private List indexPermissions = new ArrayList<>(); + private Boolean hidden; + + private Boolean reserved; + + private String description; + public Role(String name) { + this(name, null); + } + + public Role(String name, String description) { this.name = name; + this.description = description; } public Role clusterPermissions(String... clusterPermissions) { @@ -365,6 +512,16 @@ public String getName() { return name; } + public Role hidden(boolean hidden) { + this.hidden = hidden; + return this; + } + + public Role reserved(boolean reserved) { + this.reserved = reserved; + return this; + } + public Role clone() { Role role = new Role(this.name); role.clusterPermissions.addAll(this.clusterPermissions); @@ -383,9 +540,117 @@ public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params if (!indexPermissions.isEmpty()) { xContentBuilder.field("index_permissions", indexPermissions); } + if (hidden != null) { + xContentBuilder.field("hidden", hidden); + } + if (reserved != null) { + xContentBuilder.field("reserved", reserved); + } + if (!Strings.isNullOrEmpty(description)) xContentBuilder.field("description", description); + return xContentBuilder.endObject(); + } - xContentBuilder.endObject(); - return xContentBuilder; + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Role role = (Role) o; + return Objects.equals(name, role.name) + && Objects.equals(clusterPermissions, role.clusterPermissions) + && Objects.equals(indexPermissions, role.indexPermissions) + && Objects.equals(hidden, role.hidden) + && Objects.equals(reserved, role.reserved) + && Objects.equals(description, role.description); + } + + @Override + public int hashCode() { + return Objects.hash(name, clusterPermissions, indexPermissions, hidden, reserved, description); + } + } + + public static class RoleMapping implements ToXContentObject { + + private List users = new ArrayList<>(); + private List hosts = new ArrayList<>(); + + private final String name; + + private Boolean hidden; + + private Boolean reserved; + + private final String description; + + private List backendRoles = new ArrayList<>(); + + public RoleMapping(final String name) { + this(name, null); + } + + public RoleMapping(final String name, final String description) { + this.name = name; + this.description = description; + } + + public String name() { + return name; + } + + public RoleMapping hidden(boolean hidden) { + this.hidden = hidden; + return this; + } + + public RoleMapping reserved(boolean reserved) { + this.reserved = reserved; + return this; + } + + public RoleMapping users(String... users) { + this.users.addAll(Arrays.asList(users)); + return this; + } + + public RoleMapping hosts(String... hosts) { + this.users.addAll(Arrays.asList(hosts)); + return this; + } + + public RoleMapping backendRoles(String... backendRoles) { + this.backendRoles.addAll(Arrays.asList(backendRoles)); + return this; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (hidden != null) builder.field("hidden", hidden); + if (reserved != null) builder.field("reserved", reserved); + if (users != null && !users.isEmpty()) builder.field("users", users); + if (hosts != null && !hosts.isEmpty()) builder.field("hosts", hosts); + if (description != null) builder.field("description", description); + builder.field("backend_roles", backendRoles); + return builder.endObject(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RoleMapping that = (RoleMapping) o; + return Objects.equals(users, that.users) + && Objects.equals(hosts, that.hosts) + && Objects.equals(name, that.name) + && Objects.equals(hidden, that.hidden) + && Objects.equals(reserved, that.reserved) + && Objects.equals(description, that.description) + && Objects.equals(backendRoles, that.backendRoles); + } + + @Override + public int hashCode() { + return Objects.hash(users, hosts, name, hidden, reserved, description, backendRoles); } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index 217ce99a81..135f1fb481 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -56,7 +56,6 @@ import org.opensearch.test.framework.AuthFailureListeners; import org.opensearch.test.framework.AuthzDomain; import org.opensearch.test.framework.OnBehalfOfConfig; -import org.opensearch.test.framework.RolesMapping; import org.opensearch.test.framework.TestIndex; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; @@ -442,7 +441,7 @@ public Builder roles(Role... roles) { return this; } - public Builder rolesMapping(RolesMapping... mappings) { + public Builder rolesMapping(TestSecurityConfig.RoleMapping... mappings) { testSecurityConfig.rolesMapping(mappings); return this; } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java index b228fed388..5e9fd75326 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -213,17 +213,19 @@ public void stop() { for (Node node : nodes) { stopFutures.add(node.stop(2, TimeUnit.SECONDS)); } - CompletableFuture.allOf(stopFutures.toArray(size -> new CompletableFuture[size])).join(); + CompletableFuture.allOf(stopFutures.toArray(CompletableFuture[]::new)).join(); } public void destroy() { - stop(); - nodes.clear(); - try { - FileUtils.deleteDirectory(clusterHomeDir); - } catch (IOException e) { - log.warn("Error while deleting " + clusterHomeDir, e); + stop(); + nodes.clear(); + } finally { + try { + FileUtils.deleteDirectory(clusterHomeDir); + } catch (IOException e) { + log.warn("Error while deleting " + clusterHomeDir, e); + } } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java index b7da92b270..bd625a5c31 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -417,6 +417,14 @@ public T getBodyAs(Class authInfoClass) { } } + public JsonNode bodyAsJsonNode() { + try { + return DefaultObjectMapper.readTree(getBody()); + } catch (IOException e) { + throw new RuntimeException("Cannot parse response body", e); + } + } + public void assertStatusCode(int expectedHttpStatus) { String reason = format("Expected status code is '%d', but was '%d'. Response body '%s'.", expectedHttpStatus, statusCode, body); assertThat(reason, statusCode, equalTo(expectedHttpStatus)); From f375765d7cf27faf7a2d9944f3ff18430a339595 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 28 Mar 2024 16:48:53 -0400 Subject: [PATCH 164/204] [FEATURE] Improve built-in secure transports support (#4179) Signed-off-by: Andriy Redko --- .../security/OpenSearchSecurityPlugin.java | 22 ++-- .../security/filter/SecurityRestFilter.java | 20 ++-- .../http/NonSslHttpServerTransport.java | 27 +---- .../http/SecureHttpServerTransport.java | 101 ------------------ .../ssl/OpenSearchSecureSettingsFactory.java | 101 ++++++++++++++---- .../ssl/OpenSearchSecuritySSLPlugin.java | 16 +-- .../netty/Netty4ConditionalDecompressor.java | 7 +- .../Netty4HttpRequestHeaderVerifier.java | 34 +++--- .../ssl/OpenSearchSecuritySSLPluginTest.java | 94 ++++++++++++++-- 9 files changed, 223 insertions(+), 199 deletions(-) delete mode 100644 src/main/java/org/opensearch/security/http/SecureHttpServerTransport.java diff --git a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java index a59d1f531d..6e3c22e695 100644 --- a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java +++ b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java @@ -107,6 +107,7 @@ import org.opensearch.extensions.ExtensionsManager; import org.opensearch.http.HttpServerTransport; import org.opensearch.http.HttpServerTransport.Dispatcher; +import org.opensearch.http.netty4.ssl.SecureNetty4HttpServerTransport; import org.opensearch.identity.Subject; import org.opensearch.identity.noop.NoopSubject; import org.opensearch.index.IndexModule; @@ -117,6 +118,7 @@ import org.opensearch.plugins.ExtensionAwarePlugin; import org.opensearch.plugins.IdentityPlugin; import org.opensearch.plugins.MapperPlugin; +import org.opensearch.plugins.SecureHttpTransportSettingsProvider; import org.opensearch.plugins.SecureSettingsFactory; import org.opensearch.plugins.SecureTransportSettingsProvider; import org.opensearch.repositories.RepositoriesService; @@ -156,7 +158,6 @@ import org.opensearch.security.filter.SecurityFilter; import org.opensearch.security.filter.SecurityRestFilter; import org.opensearch.security.http.NonSslHttpServerTransport; -import org.opensearch.security.http.SecureHttpServerTransport; import org.opensearch.security.http.XFFResolver; import org.opensearch.security.identity.SecurityTokenManager; import org.opensearch.security.privileges.PrivilegesEvaluator; @@ -239,7 +240,6 @@ public final class OpenSearchSecurityPlugin extends OpenSearchSecuritySSLPlugin private volatile PrivilegesEvaluator evaluator; private volatile UserService userService; private volatile RestLayerPrivilegesEvaluator restLayerEvaluator; - private volatile ThreadPool threadPool; private volatile ConfigurationRepository cr; private volatile AdminDNs adminDns; private volatile ClusterService cs; @@ -927,7 +927,7 @@ public Map> getSecureHttpTransports( NetworkService networkService, Dispatcher dispatcher, ClusterSettings clusterSettings, - SecureTransportSettingsProvider secureTransportSettingsProvider, + SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider, Tracer tracer ) { @@ -942,7 +942,7 @@ public Map> getSecureHttpTransports( networkService, dispatcher, clusterSettings, - secureTransportSettingsProvider, + secureHttpTransportSettingsProvider, tracer ); } @@ -958,7 +958,7 @@ public Map> getSecureHttpTransports( evaluateSslExceptionHandler() ); // TODO close odshst - final SecureHttpServerTransport odshst = new SecureHttpServerTransport( + final SecureNetty4HttpServerTransport odshst = new SecureNetty4HttpServerTransport( migrateSettings(settings), networkService, bigArrays, @@ -967,9 +967,8 @@ public Map> getSecureHttpTransports( validatingDispatcher, clusterSettings, sharedGroupFactory, - secureTransportSettingsProvider, - tracer, - securityRestHandler + secureHttpTransportSettingsProvider, + tracer ); return Collections.singletonMap("org.opensearch.security.http.SecurityHttpServerTransport", () -> odshst); @@ -985,9 +984,8 @@ public Map> getSecureHttpTransports( dispatcher, clusterSettings, sharedGroupFactory, - secureTransportSettingsProvider, - tracer, - securityRestHandler + secureHttpTransportSettingsProvider, + tracer ) ); } @@ -2032,7 +2030,7 @@ public SecurityTokenManager getTokenManager() { @Override public Optional getSecureSettingFactory(Settings settings) { - return Optional.of(new OpenSearchSecureSettingsFactory(settings, sks, sslExceptionHandler)); + return Optional.of(new OpenSearchSecureSettingsFactory(threadPool, sks, sslExceptionHandler, securityRestHandler)); } public static class GuiceHolder implements LifecycleComponent { diff --git a/src/main/java/org/opensearch/security/filter/SecurityRestFilter.java b/src/main/java/org/opensearch/security/filter/SecurityRestFilter.java index 8ccaa3041c..420211f29b 100644 --- a/src/main/java/org/opensearch/security/filter/SecurityRestFilter.java +++ b/src/main/java/org/opensearch/security/filter/SecurityRestFilter.java @@ -55,6 +55,7 @@ import org.opensearch.security.privileges.RestLayerPrivilegesEvaluator; import org.opensearch.security.securityconf.impl.AllowlistingSettings; import org.opensearch.security.securityconf.impl.WhitelistingSettings; +import org.opensearch.security.ssl.http.netty.Netty4HttpRequestHeaderVerifier; import org.opensearch.security.ssl.transport.PrincipalExtractor; import org.opensearch.security.ssl.util.ExceptionUtils; import org.opensearch.security.ssl.util.SSLRequestHelper; @@ -69,10 +70,6 @@ import static org.opensearch.security.OpenSearchSecurityPlugin.LEGACY_OPENDISTRO_PREFIX; import static org.opensearch.security.OpenSearchSecurityPlugin.PLUGINS_PREFIX; -import static org.opensearch.security.http.SecureHttpServerTransport.CONTEXT_TO_RESTORE; -import static org.opensearch.security.http.SecureHttpServerTransport.EARLY_RESPONSE; -import static org.opensearch.security.http.SecureHttpServerTransport.IS_AUTHENTICATED; -import static org.opensearch.security.http.SecureHttpServerTransport.UNCONSUMED_PARAMS; public class SecurityRestFilter { @@ -128,15 +125,18 @@ public AuthczRestHandler(RestHandler original, AdminDNs adminDNs) { @Override public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception { - final Optional maybeSavedResponse = NettyAttribute.popFrom(request, EARLY_RESPONSE); + final Optional maybeSavedResponse = NettyAttribute.popFrom( + request, + Netty4HttpRequestHeaderVerifier.EARLY_RESPONSE + ); if (maybeSavedResponse.isPresent()) { - NettyAttribute.clearAttribute(request, CONTEXT_TO_RESTORE); - NettyAttribute.clearAttribute(request, IS_AUTHENTICATED); + NettyAttribute.clearAttribute(request, Netty4HttpRequestHeaderVerifier.CONTEXT_TO_RESTORE); + NettyAttribute.clearAttribute(request, Netty4HttpRequestHeaderVerifier.IS_AUTHENTICATED); channel.sendResponse(maybeSavedResponse.get().asRestResponse()); return; } - NettyAttribute.popFrom(request, CONTEXT_TO_RESTORE).ifPresent(storedContext -> { + NettyAttribute.popFrom(request, Netty4HttpRequestHeaderVerifier.CONTEXT_TO_RESTORE).ifPresent(storedContext -> { // X_OPAQUE_ID will be overritten on restore - save to apply after restoring the saved context final String xOpaqueId = threadContext.getHeader(Task.X_OPAQUE_ID); storedContext.restore(); @@ -145,7 +145,7 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c } }); - NettyAttribute.popFrom(request, UNCONSUMED_PARAMS).ifPresent(unconsumedParams -> { + NettyAttribute.popFrom(request, Netty4HttpRequestHeaderVerifier.UNCONSUMED_PARAMS).ifPresent(unconsumedParams -> { for (String unconsumedParam : unconsumedParams) { // Consume the parameter on the RestRequest request.param(unconsumedParam); @@ -155,7 +155,7 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c final SecurityRequestChannel requestChannel = SecurityRequestFactory.from(request, channel); // Authenticate request - if (!NettyAttribute.popFrom(request, IS_AUTHENTICATED).orElse(false)) { + if (!NettyAttribute.popFrom(request, Netty4HttpRequestHeaderVerifier.IS_AUTHENTICATED).orElse(false)) { // we aren't authenticated so we should skip this step checkAndAuthenticateRequest(requestChannel); } diff --git a/src/main/java/org/opensearch/security/http/NonSslHttpServerTransport.java b/src/main/java/org/opensearch/security/http/NonSslHttpServerTransport.java index c97d872aca..d0d1d9f09f 100644 --- a/src/main/java/org/opensearch/security/http/NonSslHttpServerTransport.java +++ b/src/main/java/org/opensearch/security/http/NonSslHttpServerTransport.java @@ -34,22 +34,15 @@ import org.opensearch.http.HttpHandlingSettings; import org.opensearch.http.netty4.Netty4HttpServerTransport; import org.opensearch.http.netty4.ssl.SecureNetty4HttpServerTransport; -import org.opensearch.plugins.SecureTransportSettingsProvider; -import org.opensearch.security.filter.SecurityRestFilter; -import org.opensearch.security.ssl.http.netty.Netty4ConditionalDecompressor; -import org.opensearch.security.ssl.http.netty.Netty4HttpRequestHeaderVerifier; +import org.opensearch.plugins.SecureHttpTransportSettingsProvider; import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.SharedGroupFactory; import io.netty.channel.Channel; import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelInboundHandlerAdapter; public class NonSslHttpServerTransport extends SecureNetty4HttpServerTransport { - - private final ChannelInboundHandlerAdapter headerVerifier; - public NonSslHttpServerTransport( final Settings settings, final NetworkService networkService, @@ -59,9 +52,8 @@ public NonSslHttpServerTransport( final Dispatcher dispatcher, final ClusterSettings clusterSettings, final SharedGroupFactory sharedGroupFactory, - final SecureTransportSettingsProvider secureTransportSettingsProvider, - final Tracer tracer, - final SecurityRestFilter restFilter + final SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider, + final Tracer tracer ) { super( settings, @@ -72,10 +64,9 @@ public NonSslHttpServerTransport( dispatcher, clusterSettings, sharedGroupFactory, - secureTransportSettingsProvider, + secureHttpTransportSettingsProvider, tracer ); - headerVerifier = new Netty4HttpRequestHeaderVerifier(restFilter, threadPool, settings); } @Override @@ -94,14 +85,4 @@ protected void initChannel(Channel ch) throws Exception { super.initChannel(ch); } } - - @Override - protected ChannelInboundHandlerAdapter createHeaderVerifier() { - return headerVerifier; - } - - @Override - protected ChannelInboundHandlerAdapter createDecompressor() { - return new Netty4ConditionalDecompressor(); - } } diff --git a/src/main/java/org/opensearch/security/http/SecureHttpServerTransport.java b/src/main/java/org/opensearch/security/http/SecureHttpServerTransport.java deleted file mode 100644 index 170f39ffd6..0000000000 --- a/src/main/java/org/opensearch/security/http/SecureHttpServerTransport.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright 2015-2018 _floragunn_ GmbH - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.security.http; - -import java.util.Set; - -import org.opensearch.common.network.NetworkService; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.BigArrays; -import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.http.netty4.ssl.SecureNetty4HttpServerTransport; -import org.opensearch.plugins.SecureTransportSettingsProvider; -import org.opensearch.security.filter.SecurityResponse; -import org.opensearch.security.filter.SecurityRestFilter; -import org.opensearch.security.ssl.http.netty.Netty4ConditionalDecompressor; -import org.opensearch.security.ssl.http.netty.Netty4HttpRequestHeaderVerifier; -import org.opensearch.security.ssl.http.netty.ValidatingDispatcher; -import org.opensearch.telemetry.tracing.Tracer; -import org.opensearch.threadpool.ThreadPool; -import org.opensearch.transport.SharedGroupFactory; - -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.util.AttributeKey; - -public class SecureHttpServerTransport extends SecureNetty4HttpServerTransport { - - public static final AttributeKey EARLY_RESPONSE = AttributeKey.newInstance("opensearch-http-early-response"); - public static final AttributeKey> UNCONSUMED_PARAMS = AttributeKey.newInstance("opensearch-http-request-consumed-params"); - public static final AttributeKey CONTEXT_TO_RESTORE = AttributeKey.newInstance( - "opensearch-http-request-thread-context" - ); - public static final AttributeKey SHOULD_DECOMPRESS = AttributeKey.newInstance("opensearch-http-should-decompress"); - public static final AttributeKey IS_AUTHENTICATED = AttributeKey.newInstance("opensearch-http-is-authenticated"); - - private final ChannelInboundHandlerAdapter headerVerifier; - - public SecureHttpServerTransport( - final Settings settings, - final NetworkService networkService, - final BigArrays bigArrays, - final ThreadPool threadPool, - final NamedXContentRegistry namedXContentRegistry, - final ValidatingDispatcher dispatcher, - final ClusterSettings clusterSettings, - SharedGroupFactory sharedGroupFactory, - final SecureTransportSettingsProvider secureTransportSettingsProvider, - Tracer tracer, - SecurityRestFilter restFilter - ) { - super( - settings, - networkService, - bigArrays, - threadPool, - namedXContentRegistry, - dispatcher, - clusterSettings, - sharedGroupFactory, - secureTransportSettingsProvider, - tracer - ); - - headerVerifier = new Netty4HttpRequestHeaderVerifier(restFilter, threadPool, settings); - } - - @Override - protected ChannelInboundHandlerAdapter createHeaderVerifier() { - return headerVerifier; - } - - @Override - protected ChannelInboundHandlerAdapter createDecompressor() { - return new Netty4ConditionalDecompressor(); - } -} diff --git a/src/main/java/org/opensearch/security/ssl/OpenSearchSecureSettingsFactory.java b/src/main/java/org/opensearch/security/ssl/OpenSearchSecureSettingsFactory.java index d85f490d0c..5351eea57e 100644 --- a/src/main/java/org/opensearch/security/ssl/OpenSearchSecureSettingsFactory.java +++ b/src/main/java/org/opensearch/security/ssl/OpenSearchSecureSettingsFactory.java @@ -11,63 +11,124 @@ package org.opensearch.security.ssl; +import java.util.Collection; +import java.util.List; import java.util.Optional; import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLException; import org.opensearch.common.settings.Settings; import org.opensearch.http.HttpServerTransport; +import org.opensearch.http.netty4.ssl.SecureNetty4HttpServerTransport; +import org.opensearch.plugins.SecureHttpTransportSettingsProvider; import org.opensearch.plugins.SecureSettingsFactory; import org.opensearch.plugins.SecureTransportSettingsProvider; -import org.opensearch.transport.TcpTransport; +import org.opensearch.plugins.TransportExceptionHandler; +import org.opensearch.security.filter.SecurityRestFilter; +import org.opensearch.security.ssl.http.netty.Netty4ConditionalDecompressor; +import org.opensearch.security.ssl.http.netty.Netty4HttpRequestHeaderVerifier; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.Transport; +import org.opensearch.transport.TransportAdapterProvider; + +import io.netty.channel.ChannelInboundHandlerAdapter; public class OpenSearchSecureSettingsFactory implements SecureSettingsFactory { - private final Settings settings; + private final ThreadPool threadPool; private final SecurityKeyStore sks; private final SslExceptionHandler sslExceptionHandler; + private final SecurityRestFilter restFilter; - public OpenSearchSecureSettingsFactory(Settings settings, SecurityKeyStore sks, SslExceptionHandler sslExceptionHandler) { - this.settings = settings; + public OpenSearchSecureSettingsFactory( + ThreadPool threadPool, + SecurityKeyStore sks, + SslExceptionHandler sslExceptionHandler, + SecurityRestFilter restFilter + ) { + this.threadPool = threadPool; this.sks = sks; this.sslExceptionHandler = sslExceptionHandler; + this.restFilter = restFilter; } @Override public Optional getSecureTransportSettingsProvider(Settings settings) { return Optional.of(new SecureTransportSettingsProvider() { @Override - public Optional buildHttpServerExceptionHandler(Settings settings, HttpServerTransport transport) { - return Optional.of(new ServerExceptionHandler() { + public Optional buildServerTransportExceptionHandler(Settings settings, Transport transport) { + return Optional.of(new TransportExceptionHandler() { @Override public void onError(Throwable t) { - sslExceptionHandler.logError(t, true); + sslExceptionHandler.logError(t, false); } }); } @Override - public Optional buildServerTransportExceptionHandler(Settings settings, TcpTransport transport) { - return Optional.of(new ServerExceptionHandler() { - @Override - public void onError(Throwable t) { - sslExceptionHandler.logError(t, false); - } - }); + public Optional buildSecureServerTransportEngine(Settings settings, Transport transport) throws SSLException { + return Optional.of(sks.createServerTransportSSLEngine()); } @Override - public Optional buildSecureHttpServerEngine(Settings settings, HttpServerTransport transport) throws SSLException { - return Optional.of(sks.createHTTPSSLEngine()); + public Optional buildSecureClientTransportEngine(Settings settings, String hostname, int port) throws SSLException { + return Optional.of(sks.createClientTransportSSLEngine(hostname, port)); } + }); + } + @Override + public Optional getSecureHttpTransportSettingsProvider(Settings settings) { + return Optional.of(new SecureHttpTransportSettingsProvider() { @Override - public Optional buildSecureServerTransportEngine(Settings settings, TcpTransport transport) throws SSLException { - return Optional.of(sks.createServerTransportSSLEngine()); + public Collection> getHttpTransportAdapterProviders(Settings settings) { + return List.of(new TransportAdapterProvider() { + @Override + public String name() { + return SecureNetty4HttpServerTransport.REQUEST_DECOMPRESSOR; + } + + @SuppressWarnings("unchecked") + @Override + public Optional create(Settings settings, HttpServerTransport transport, Class adapterClass) { + if (transport instanceof SecureNetty4HttpServerTransport + && ChannelInboundHandlerAdapter.class.isAssignableFrom(adapterClass)) { + return Optional.of((C) new Netty4ConditionalDecompressor()); + } else { + return Optional.empty(); + } + } + }, new TransportAdapterProvider() { + @Override + public String name() { + return SecureNetty4HttpServerTransport.REQUEST_HEADER_VERIFIER; + } + + @SuppressWarnings("unchecked") + @Override + public Optional create(Settings settings, HttpServerTransport transport, Class adapterClass) { + if (transport instanceof SecureNetty4HttpServerTransport + && ChannelInboundHandlerAdapter.class.isAssignableFrom(adapterClass)) { + return Optional.of((C) new Netty4HttpRequestHeaderVerifier(restFilter, threadPool, settings)); + } else { + return Optional.empty(); + } + } + }); } @Override - public Optional buildSecureClientTransportEngine(Settings settings, String hostname, int port) throws SSLException { - return Optional.of(sks.createClientTransportSSLEngine(hostname, port)); + public Optional buildHttpServerExceptionHandler(Settings settings, HttpServerTransport transport) { + return Optional.of(new TransportExceptionHandler() { + @Override + public void onError(Throwable t) { + sslExceptionHandler.logError(t, true); + } + }); + } + + @Override + public Optional buildSecureHttpServerEngine(Settings settings, HttpServerTransport transport) throws SSLException { + return Optional.of(sks.createHTTPSSLEngine()); } }); } diff --git a/src/main/java/org/opensearch/security/ssl/OpenSearchSecuritySSLPlugin.java b/src/main/java/org/opensearch/security/ssl/OpenSearchSecuritySSLPlugin.java index 3acbce21cf..073193e9d4 100644 --- a/src/main/java/org/opensearch/security/ssl/OpenSearchSecuritySSLPlugin.java +++ b/src/main/java/org/opensearch/security/ssl/OpenSearchSecuritySSLPlugin.java @@ -61,8 +61,10 @@ import org.opensearch.env.NodeEnvironment; import org.opensearch.http.HttpServerTransport; import org.opensearch.http.HttpServerTransport.Dispatcher; +import org.opensearch.http.netty4.ssl.SecureNetty4HttpServerTransport; import org.opensearch.plugins.NetworkPlugin; import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.SecureHttpTransportSettingsProvider; import org.opensearch.plugins.SecureSettingsFactory; import org.opensearch.plugins.SecureTransportSettingsProvider; import org.opensearch.plugins.SystemIndexPlugin; @@ -73,7 +75,6 @@ import org.opensearch.security.DefaultObjectMapper; import org.opensearch.security.NonValidatingObjectMapper; import org.opensearch.security.filter.SecurityRestFilter; -import org.opensearch.security.http.SecureHttpServerTransport; import org.opensearch.security.ssl.http.netty.ValidatingDispatcher; import org.opensearch.security.ssl.rest.SecuritySSLInfoAction; import org.opensearch.security.ssl.transport.DefaultPrincipalExtractor; @@ -131,6 +132,7 @@ public class OpenSearchSecuritySSLPlugin extends Plugin implements SystemIndexPl private final static SslExceptionHandler NOOP_SSL_EXCEPTION_HANDLER = new SslExceptionHandler() { }; protected final SSLConfig SSLConfig; + protected volatile ThreadPool threadPool; // public OpenSearchSecuritySSLPlugin(final Settings settings, final Path configPath) { // this(settings, configPath, false); @@ -266,7 +268,7 @@ public Map> getSecureHttpTransports( NetworkService networkService, Dispatcher dispatcher, ClusterSettings clusterSettings, - SecureTransportSettingsProvider secureTransportSettingsProvider, + SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider, Tracer tracer ) { @@ -279,7 +281,7 @@ public Map> getSecureHttpTransports( configPath, NOOP_SSL_EXCEPTION_HANDLER ); - final SecureHttpServerTransport sgsnht = new SecureHttpServerTransport( + final SecureNetty4HttpServerTransport sgsnht = new SecureNetty4HttpServerTransport( migrateSettings(settings), networkService, bigArrays, @@ -288,9 +290,8 @@ public Map> getSecureHttpTransports( validatingDispatcher, clusterSettings, sharedGroupFactory, - secureTransportSettingsProvider, - tracer, - securityRestHandler + secureHttpTransportSettingsProvider, + tracer ); return Collections.singletonMap("org.opensearch.security.ssl.http.netty.SecuritySSLNettyHttpServerTransport", () -> sgsnht); @@ -381,6 +382,7 @@ public Collection createComponents( Supplier repositoriesServiceSupplier ) { + this.threadPool = threadPool; final List components = new ArrayList<>(1); if (client) { @@ -676,7 +678,7 @@ public List getSettingsFilter() { @Override public Optional getSecureSettingFactory(Settings settings) { - return Optional.of(new OpenSearchSecureSettingsFactory(settings, sks, NOOP_SSL_EXCEPTION_HANDLER)); + return Optional.of(new OpenSearchSecureSettingsFactory(threadPool, sks, NOOP_SSL_EXCEPTION_HANDLER, securityRestHandler)); } protected Settings migrateSettings(Settings settings) { diff --git a/src/main/java/org/opensearch/security/ssl/http/netty/Netty4ConditionalDecompressor.java b/src/main/java/org/opensearch/security/ssl/http/netty/Netty4ConditionalDecompressor.java index 8b2d4bb1d2..b36a12da48 100644 --- a/src/main/java/org/opensearch/security/ssl/http/netty/Netty4ConditionalDecompressor.java +++ b/src/main/java/org/opensearch/security/ssl/http/netty/Netty4ConditionalDecompressor.java @@ -13,15 +13,12 @@ import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.http.HttpContentDecompressor; -import static org.opensearch.security.http.SecureHttpServerTransport.EARLY_RESPONSE; -import static org.opensearch.security.http.SecureHttpServerTransport.SHOULD_DECOMPRESS; - public class Netty4ConditionalDecompressor extends HttpContentDecompressor { @Override protected EmbeddedChannel newContentDecoder(String contentEncoding) throws Exception { - final boolean hasAnEarlyReponse = NettyAttribute.peekFrom(ctx, EARLY_RESPONSE).isPresent(); - final boolean shouldDecompress = NettyAttribute.popFrom(ctx, SHOULD_DECOMPRESS).orElse(false); + final boolean hasAnEarlyReponse = NettyAttribute.peekFrom(ctx, Netty4HttpRequestHeaderVerifier.EARLY_RESPONSE).isPresent(); + final boolean shouldDecompress = NettyAttribute.popFrom(ctx, Netty4HttpRequestHeaderVerifier.SHOULD_DECOMPRESS).orElse(false); if (hasAnEarlyReponse || !shouldDecompress) { // If there was an error prompting an early response,... don't decompress // If there is no explicit decompress flag,... don't decompress diff --git a/src/main/java/org/opensearch/security/ssl/http/netty/Netty4HttpRequestHeaderVerifier.java b/src/main/java/org/opensearch/security/ssl/http/netty/Netty4HttpRequestHeaderVerifier.java index 052f1961e2..9afd6b0e22 100644 --- a/src/main/java/org/opensearch/security/ssl/http/netty/Netty4HttpRequestHeaderVerifier.java +++ b/src/main/java/org/opensearch/security/ssl/http/netty/Netty4HttpRequestHeaderVerifier.java @@ -8,6 +8,8 @@ package org.opensearch.security.ssl.http.netty; +import java.util.Set; + import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchSecurityException; import org.opensearch.common.settings.Settings; @@ -30,16 +32,19 @@ import io.netty.channel.SimpleChannelInboundHandler; import io.netty.handler.codec.http.DefaultHttpRequest; import io.netty.handler.codec.http.HttpRequest; +import io.netty.util.AttributeKey; import io.netty.util.ReferenceCountUtil; -import static org.opensearch.security.http.SecureHttpServerTransport.CONTEXT_TO_RESTORE; -import static org.opensearch.security.http.SecureHttpServerTransport.EARLY_RESPONSE; -import static org.opensearch.security.http.SecureHttpServerTransport.IS_AUTHENTICATED; -import static org.opensearch.security.http.SecureHttpServerTransport.SHOULD_DECOMPRESS; -import static org.opensearch.security.http.SecureHttpServerTransport.UNCONSUMED_PARAMS; - @Sharable public class Netty4HttpRequestHeaderVerifier extends SimpleChannelInboundHandler { + public static final AttributeKey IS_AUTHENTICATED = AttributeKey.newInstance("opensearch-http-is-authenticated"); + public static final AttributeKey SHOULD_DECOMPRESS = AttributeKey.newInstance("opensearch-http-should-decompress"); + public static final AttributeKey CONTEXT_TO_RESTORE = AttributeKey.newInstance( + "opensearch-http-request-thread-context" + ); + public static final AttributeKey> UNCONSUMED_PARAMS = AttributeKey.newInstance("opensearch-http-request-consumed-params"); + public static final AttributeKey EARLY_RESPONSE = AttributeKey.newInstance("opensearch-http-early-response"); + private final SecurityRestFilter restFilter; private final ThreadPool threadPool; private final SSLConfig sslConfig; @@ -72,8 +77,8 @@ public void channelRead0(ChannelHandlerContext ctx, DefaultHttpRequest msg) thro } // Start by setting this value to false, only requests that meet all the criteria will be decompressed - ctx.channel().attr(SHOULD_DECOMPRESS).set(Boolean.FALSE); - ctx.channel().attr(IS_AUTHENTICATED).set(Boolean.FALSE); + ctx.channel().attr(Netty4HttpRequestHeaderVerifier.SHOULD_DECOMPRESS).set(Boolean.FALSE); + ctx.channel().attr(Netty4HttpRequestHeaderVerifier.IS_AUTHENTICATED).set(Boolean.FALSE); final Netty4HttpChannel httpChannel = ctx.channel().attr(Netty4HttpServerTransport.HTTP_CHANNEL_KEY).get(); @@ -85,24 +90,25 @@ public void channelRead0(ChannelHandlerContext ctx, DefaultHttpRequest msg) thro // If request channel is completed and a response is sent, then there was a failure during authentication restFilter.checkAndAuthenticateRequest(requestChannel); - ctx.channel().attr(UNCONSUMED_PARAMS).set(requestChannel.getUnconsumedParams()); + ctx.channel().attr(Netty4HttpRequestHeaderVerifier.UNCONSUMED_PARAMS).set(requestChannel.getUnconsumedParams()); ThreadContext.StoredContext contextToRestore = threadPool.getThreadContext().newStoredContext(false); - ctx.channel().attr(CONTEXT_TO_RESTORE).set(contextToRestore); + ctx.channel().attr(Netty4HttpRequestHeaderVerifier.CONTEXT_TO_RESTORE).set(contextToRestore); - requestChannel.getQueuedResponse().ifPresent(response -> ctx.channel().attr(EARLY_RESPONSE).set(response)); + requestChannel.getQueuedResponse() + .ifPresent(response -> ctx.channel().attr(Netty4HttpRequestHeaderVerifier.EARLY_RESPONSE).set(response)); boolean shouldSkipAuthentication = SecurityRestUtils.shouldSkipAuthentication(requestChannel); boolean shouldDecompress = !shouldSkipAuthentication && requestChannel.getQueuedResponse().isEmpty(); if (requestChannel.getQueuedResponse().isEmpty() || shouldSkipAuthentication) { // Only allow decompression on authenticated requests that also aren't one of those ^ - ctx.channel().attr(SHOULD_DECOMPRESS).set(Boolean.valueOf(shouldDecompress)); - ctx.channel().attr(IS_AUTHENTICATED).set(Boolean.TRUE); + ctx.channel().attr(Netty4HttpRequestHeaderVerifier.SHOULD_DECOMPRESS).set(Boolean.valueOf(shouldDecompress)); + ctx.channel().attr(Netty4HttpRequestHeaderVerifier.IS_AUTHENTICATED).set(Boolean.TRUE); } } catch (final OpenSearchSecurityException e) { final SecurityResponse earlyResponse = new SecurityResponse(ExceptionsHelper.status(e).getStatus(), e); - ctx.channel().attr(EARLY_RESPONSE).set(earlyResponse); + ctx.channel().attr(Netty4HttpRequestHeaderVerifier.EARLY_RESPONSE).set(earlyResponse); } catch (final SecurityRequestChannelUnsupported srcu) { // Use defaults for unsupported channels } finally { diff --git a/src/test/java/org/opensearch/security/ssl/OpenSearchSecuritySSLPluginTest.java b/src/test/java/org/opensearch/security/ssl/OpenSearchSecuritySSLPluginTest.java index 03488fe17c..aefb12c0db 100644 --- a/src/test/java/org/opensearch/security/ssl/OpenSearchSecuritySSLPluginTest.java +++ b/src/test/java/org/opensearch/security/ssl/OpenSearchSecuritySSLPluginTest.java @@ -13,6 +13,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLException; @@ -26,16 +27,22 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.http.HttpServerTransport; +import org.opensearch.http.netty4.ssl.SecureNetty4HttpServerTransport; +import org.opensearch.plugins.SecureHttpTransportSettingsProvider; import org.opensearch.plugins.SecureTransportSettingsProvider; +import org.opensearch.plugins.TransportExceptionHandler; import org.opensearch.security.ssl.util.SSLConfigConstants; import org.opensearch.security.support.SecuritySettings; import org.opensearch.security.test.AbstractSecurityUnitTest; import org.opensearch.security.test.helper.file.FileHelper; import org.opensearch.telemetry.tracing.noop.NoopTracer; -import org.opensearch.transport.TcpTransport; import org.opensearch.transport.Transport; +import org.opensearch.transport.TransportAdapterProvider; + +import io.netty.channel.ChannelInboundHandlerAdapter; import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; @@ -44,6 +51,7 @@ public class OpenSearchSecuritySSLPluginTest extends AbstractSecurityUnitTest { private Settings settings; + private SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider; private SecureTransportSettingsProvider secureTransportSettingsProvider; private ClusterSettings clusterSettings; @@ -76,27 +84,29 @@ public void setUp() { secureTransportSettingsProvider = new SecureTransportSettingsProvider() { @Override - public Optional buildHttpServerExceptionHandler(Settings settings, HttpServerTransport transport) { + public Optional buildServerTransportExceptionHandler(Settings settings, Transport transport) { return Optional.empty(); } @Override - public Optional buildServerTransportExceptionHandler(Settings settings, TcpTransport transport) { + public Optional buildSecureServerTransportEngine(Settings settings, Transport transport) throws SSLException { return Optional.empty(); } @Override - public Optional buildSecureHttpServerEngine(Settings settings, HttpServerTransport transport) throws SSLException { + public Optional buildSecureClientTransportEngine(Settings settings, String hostname, int port) throws SSLException { return Optional.empty(); } + }; + secureHttpTransportSettingsProvider = new SecureHttpTransportSettingsProvider() { @Override - public Optional buildSecureServerTransportEngine(Settings settings, TcpTransport transport) throws SSLException { + public Optional buildHttpServerExceptionHandler(Settings settings, HttpServerTransport transport) { return Optional.empty(); } @Override - public Optional buildSecureClientTransportEngine(Settings settings, String hostname, int port) throws SSLException { + public Optional buildSecureHttpServerEngine(Settings settings, HttpServerTransport transport) throws SSLException { return Optional.empty(); } }; @@ -117,10 +127,14 @@ public void testRegisterSecureHttpTransport() throws IOException { null, null, clusterSettings, - secureTransportSettingsProvider, + secureHttpTransportSettingsProvider, NoopTracer.INSTANCE ); assertThat(transports, hasKey("org.opensearch.security.ssl.http.netty.SecuritySSLNettyHttpServerTransport")); + assertThat( + transports.get("org.opensearch.security.ssl.http.netty.SecuritySSLNettyHttpServerTransport").get(), + not(nullValue()) + ); } } @@ -138,6 +152,7 @@ public void testRegisterSecureTransport() throws IOException { NoopTracer.INSTANCE ); assertThat(transports, hasKey("org.opensearch.security.ssl.http.netty.SecuritySSLNettyTransport")); + assertThat(transports.get("org.opensearch.security.ssl.http.netty.SecuritySSLNettyTransport").get(), not(nullValue())); } } @@ -243,4 +258,69 @@ public void testRegisterSecureTransportWithDuplicateSettings() throws IOExceptio } } } + + @Test + public void testRegisterSecureHttpTransportWithRequestHeaderVerifier() throws IOException { + final AtomicBoolean created = new AtomicBoolean(false); + + class LocalHeaderVerifier extends ChannelInboundHandlerAdapter { + public LocalHeaderVerifier() { + created.set(true); + } + } + + final SecureHttpTransportSettingsProvider provider = new SecureHttpTransportSettingsProvider() { + @Override + public Collection> getHttpTransportAdapterProviders(Settings settings) { + return List.of(new TransportAdapterProvider() { + + @Override + public String name() { + return SecureNetty4HttpServerTransport.REQUEST_HEADER_VERIFIER; + } + + @SuppressWarnings("unchecked") + @Override + public Optional create(Settings settings, HttpServerTransport transport, Class adapterClass) { + return Optional.of((C) new LocalHeaderVerifier()); + } + + }); + } + + @Override + public Optional buildHttpServerExceptionHandler(Settings settings, HttpServerTransport transport) { + return Optional.empty(); + } + + @Override + public Optional buildSecureHttpServerEngine(Settings settings, HttpServerTransport transport) throws SSLException { + return Optional.empty(); + } + }; + + try (OpenSearchSecuritySSLPlugin plugin = new OpenSearchSecuritySSLPlugin(settings, null, false)) { + final Map> transports = plugin.getSecureHttpTransports( + settings, + MOCK_POOL, + null, + null, + null, + null, + null, + null, + clusterSettings, + provider, + NoopTracer.INSTANCE + ); + assertThat(transports, hasKey("org.opensearch.security.ssl.http.netty.SecuritySSLNettyHttpServerTransport")); + + assertThat( + transports.get("org.opensearch.security.ssl.http.netty.SecuritySSLNettyHttpServerTransport").get(), + not(nullValue()) + ); + + assertThat(created.get(), is(true)); + } + } } From ef72e4c1e539a98a83dd9882e5f3d10d33ad3a2d Mon Sep 17 00:00:00 2001 From: Sicheng Song Date: Fri, 29 Mar 2024 13:09:26 -0700 Subject: [PATCH 165/204] Refactor and update existing ml roles (#4151) Signed-off-by: Sicheng Song --- config/roles.yml | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/config/roles.yml b/config/roles.yml index e90a4e62a6..1b4c13745f 100644 --- a/config/roles.yml +++ b/config/roles.yml @@ -272,25 +272,39 @@ cross_cluster_search_remote_full_access: - 'indices:data/read/search' # Allow users to operate query assistant -ml_query_assistant_access: +query_assistant_access: reserved: true cluster_permissions: + - 'cluster:admin/opensearch/ml/config/get' - 'cluster:admin/opensearch/ml/execute' - - 'cluster:admin/opensearch/ml/memory/conversation/create' - - 'cluster:admin/opensearch/ml/memory/interaction/create' - 'cluster:admin/opensearch/ml/predict' + - 'cluster:admin/opensearch/ppl' # Allow users to read ML stats/models/tasks ml_read_access: reserved: true cluster_permissions: + - 'cluster:admin/opensearch/ml/config/get' - 'cluster:admin/opensearch/ml/connectors/get' - 'cluster:admin/opensearch/ml/connectors/search' + - 'cluster:admin/opensearch/ml/controllers/get' + - 'cluster:admin/opensearch/ml/memory/conversation/get' + - 'cluster:admin/opensearch/ml/memory/conversation/interaction/search' + - 'cluster:admin/opensearch/ml/memory/conversation/list' + - 'cluster:admin/opensearch/ml/memory/conversation/search' + - 'cluster:admin/opensearch/ml/memory/interaction/get' + - 'cluster:admin/opensearch/ml/memory/interaction/list' + - 'cluster:admin/opensearch/ml/memory/trace/get' + - 'cluster:admin/opensearch/ml/model_groups/get' - 'cluster:admin/opensearch/ml/model_groups/search' - 'cluster:admin/opensearch/ml/models/get' - 'cluster:admin/opensearch/ml/models/search' + - 'cluster:admin/opensearch/ml/profile/nodes' + - 'cluster:admin/opensearch/ml/stats/nodes' - 'cluster:admin/opensearch/ml/tasks/get' - 'cluster:admin/opensearch/ml/tasks/search' + - 'cluster:admin/opensearch/ml/tools/get' + - 'cluster:admin/opensearch/ml/tools/list' # Allows users to use all ML functionality ml_full_access: From 49bb15fb45854b61b3d34af1e4a05eda02deb077 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 07:50:11 -0400 Subject: [PATCH 166/204] Bump com.fasterxml.woodstox:woodstox-core from 6.6.1 to 6.6.2 (#4191) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 607649d082..a1c901fbb4 100644 --- a/build.gradle +++ b/build.gradle @@ -644,7 +644,7 @@ dependencies { runtimeOnly 'org.xerial.snappy:snappy-java:1.1.10.5' runtimeOnly 'org.codehaus.woodstox:stax2-api:4.2.2' runtimeOnly "org.glassfish.jaxb:txw2:${jaxb_version}" - runtimeOnly 'com.fasterxml.woodstox:woodstox-core:6.6.1' + runtimeOnly 'com.fasterxml.woodstox:woodstox-core:6.6.2' runtimeOnly 'org.apache.ws.xmlschema:xmlschema-core:2.3.1' runtimeOnly 'org.apache.santuario:xmlsec:2.3.4' runtimeOnly "com.github.luben:zstd-jni:${versions.zstd}" From 065eaffe97f7ca51b8fd66aeefa3531b23fb4db3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 07:51:30 -0400 Subject: [PATCH 167/204] Bump commons-io:commons-io from 2.15.1 to 2.16.0 (#4192) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a1c901fbb4..fe4785d8dd 100644 --- a/build.gradle +++ b/build.gradle @@ -720,7 +720,7 @@ dependencies { integrationTestImplementation 'junit:junit:4.13.2' integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}" integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}" - integrationTestImplementation 'commons-io:commons-io:2.15.1' + integrationTestImplementation 'commons-io:commons-io:2.16.0' integrationTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}" integrationTestImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}" integrationTestImplementation 'org.hamcrest:hamcrest:2.2' From 98080f836a51bdbc0aa381c399730b2c90f94032 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 07:52:01 -0400 Subject: [PATCH 168/204] Bump io.dropwizard.metrics:metrics-core from 4.2.15 to 4.2.25 (#4193) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index fe4785d8dd..17a801d82b 100644 --- a/build.gradle +++ b/build.gradle @@ -613,7 +613,7 @@ dependencies { //OpenSAML implementation 'net.shibboleth.utilities:java-support:8.4.1' - runtimeOnly "io.dropwizard.metrics:metrics-core:4.2.15" + runtimeOnly "io.dropwizard.metrics:metrics-core:4.2.25" implementation "com.onelogin:java-saml:${one_login_java_saml}" implementation "com.onelogin:java-saml-core:${one_login_java_saml}" implementation "org.opensaml:opensaml-core:${open_saml_version}" From 759ef7e991a277c43d98fe6326b4f88f42853849 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 12:12:53 +0000 Subject: [PATCH 169/204] Bump Wandalen/wretry.action from 2.1.0 to 3.0.0 (#4194) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1873fba2a..90ad1614da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,7 +81,7 @@ jobs: working-directory: downloaded-artifacts - name: Upload Coverage with retry - uses: Wandalen/wretry.action@v2.1.0 + uses: Wandalen/wretry.action@v3.0.0 with: attempt_limit: 5 attempt_delay: 2000 From 4e8297aa4253c675526a803651a9214fe4e02962 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 2 Apr 2024 14:08:54 -0400 Subject: [PATCH 170/204] Remove date from release notes (#4203) Signed-off-by: Craig Perkins --- release-notes/opensearch-security.release-notes-2.13.0.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-notes/opensearch-security.release-notes-2.13.0.0.md b/release-notes/opensearch-security.release-notes-2.13.0.0.md index 42ccd8fff0..48ecfa9b87 100644 --- a/release-notes/opensearch-security.release-notes-2.13.0.0.md +++ b/release-notes/opensearch-security.release-notes-2.13.0.0.md @@ -1,4 +1,4 @@ -## 2024-03-19 Version 2.13.0.0 +## Version 2.13.0.0 Compatible with OpenSearch 2.13.0 From 90fe3bb65dda815bbcf9b9ce87c1044f631d8a8b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 08:21:08 -0400 Subject: [PATCH 171/204] Bump com.nulab-inc:zxcvbn from 1.8.2 to 1.9.0 (#4217) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 17a801d82b..6b3f2ebfed 100644 --- a/build.gradle +++ b/build.gradle @@ -631,7 +631,7 @@ dependencies { runtimeOnly "org.opensaml:opensaml-soap-impl:${open_saml_version}" implementation "org.opensaml:opensaml-storage-api:${open_saml_version}" - implementation "com.nulab-inc:zxcvbn:1.8.2" + implementation "com.nulab-inc:zxcvbn:1.9.0" runtimeOnly 'com.google.guava:failureaccess:1.0.2' runtimeOnly 'org.apache.commons:commons-text:1.11.0' From b9f864672e52847ea3d1922215302bd3bd85bc50 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 08:21:16 -0400 Subject: [PATCH 172/204] Bump com.google.googlejavaformat:google-java-format from 1.21.0 to 1.22.0 (#4216) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 6b3f2ebfed..51d9fedb7a 100644 --- a/build.gradle +++ b/build.gradle @@ -737,7 +737,7 @@ dependencies { integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5" //spotless - implementation('com.google.googlejavaformat:google-java-format:1.21.0') { + implementation('com.google.googlejavaformat:google-java-format:1.22.0') { exclude group: 'com.google.guava' } } From 02947dd009c052cea6ccca1926d701019b192edb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 08:21:59 -0400 Subject: [PATCH 173/204] Bump Wandalen/wretry.action from 3.0.0 to 3.1.0 (#4218) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90ad1614da..45f306318e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,7 +81,7 @@ jobs: working-directory: downloaded-artifacts - name: Upload Coverage with retry - uses: Wandalen/wretry.action@v3.0.0 + uses: Wandalen/wretry.action@v3.1.0 with: attempt_limit: 5 attempt_delay: 2000 From ba74d1493edba694441786519fdbec5161849a16 Mon Sep 17 00:00:00 2001 From: Andrey Pleskach Date: Mon, 8 Apr 2024 14:46:23 +0200 Subject: [PATCH 174/204] Fix bug for PATCH in EndpointValidator (#4190) Signed-off-by: Andrey Pleskach --- .../dlic/rest/api/AbstractApiAction.java | 2 +- .../dlic/rest/api/RolesApiAction.java | 9 +- .../dlic/rest/api/RolesMappingApiAction.java | 11 +- .../rest/validation/EndpointValidator.java | 12 +- .../dlic/rest/api/ActionGroupsApiTest.java | 72 +++++++--- .../api/RolesApiActionValidationTest.java | 14 +- .../security/dlic/rest/api/RolesApiTest.java | 113 ++++++---------- .../RolesMappingApiActionValidationTest.java | 5 +- .../dlic/rest/api/RolesMappingApiTest.java | 125 ++++++++---------- src/test/resources/restapi/action_groups.yml | 13 ++ src/test/resources/restapi/roles.yml | 26 ++++ src/test/resources/restapi/roles_mapping.yml | 3 + 12 files changed, 214 insertions(+), 191 deletions(-) diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/AbstractApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/AbstractApiAction.java index a38d618da0..157d44da73 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/AbstractApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/AbstractApiAction.java @@ -241,7 +241,7 @@ protected ValidationResult patchEntities( for (final var entityName : patchEntityNames(patchContent)) { final var beforePatchEntity = configurationAsJson.get(entityName); final var patchedEntity = patchedConfigurationAsJson.get(entityName); - // verify we can process exising or updated entities + // verify we can process existing or updated entities if (beforePatchEntity != null && !Objects.equals(beforePatchEntity, patchedEntity)) { final var checkEntityCanBeProcess = endpointValidator.isAllowedToChangeImmutableEntity( SecurityConfiguration.of(entityName, configuration) diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/RolesApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/RolesApiAction.java index 50fac9b80c..8b25fd5702 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/RolesApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/RolesApiAction.java @@ -141,12 +141,9 @@ public RestApiAdminPrivilegesEvaluator restApiAdminPrivilegesEvaluator() { @Override public ValidationResult isAllowedToChangeImmutableEntity(SecurityConfiguration securityConfiguration) throws IOException { - return EndpointValidator.super.isAllowedToChangeImmutableEntity(securityConfiguration).map(ignore -> { - if (isCurrentUserAdmin()) { - return ValidationResult.success(securityConfiguration); - } - return isAllowedToChangeEntityWithRestAdminPermissions(securityConfiguration); - }); + return EndpointValidator.super.isAllowedToChangeImmutableEntity(securityConfiguration).map( + ignore -> isAllowedToChangeEntityWithRestAdminPermissions(securityConfiguration) + ); } @Override diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/RolesMappingApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/RolesMappingApiAction.java index b980a1e4ba..bc2e515e09 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/RolesMappingApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/RolesMappingApiAction.java @@ -106,14 +106,11 @@ public ValidationResult isAllowedToChangeImmutableEntity( public ValidationResult isAllowedToChangeRoleMappingWithRestAdminPermissions( SecurityConfiguration securityConfiguration ) throws IOException { - return loadConfiguration(CType.ROLES, false, false).map(rolesConfiguration -> { - if (isCurrentUserAdmin()) { - return ValidationResult.success(securityConfiguration); - } - return isAllowedToChangeEntityWithRestAdminPermissions( + return loadConfiguration(CType.ROLES, false, false).map( + rolesConfiguration -> isAllowedToChangeEntityWithRestAdminPermissions( SecurityConfiguration.of(securityConfiguration.entityName(), rolesConfiguration) - ); - }).map(ignore -> ValidationResult.success(securityConfiguration)); + ) + ).map(ignore -> ValidationResult.success(securityConfiguration)); } @Override diff --git a/src/main/java/org/opensearch/security/dlic/rest/validation/EndpointValidator.java b/src/main/java/org/opensearch/security/dlic/rest/validation/EndpointValidator.java index 5879272b30..17c8b1f2ba 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/validation/EndpointValidator.java +++ b/src/main/java/org/opensearch/security/dlic/rest/validation/EndpointValidator.java @@ -156,21 +156,19 @@ default ValidationResult> validateRoles( default ValidationResult isAllowedToChangeEntityWithRestAdminPermissions( final SecurityConfiguration securityConfiguration ) throws IOException { + final var configuration = securityConfiguration.configuration(); if (securityConfiguration.entityExists()) { - final var configuration = securityConfiguration.configuration(); final var existingEntity = configuration.getCEntry(securityConfiguration.entityName()); if (restApiAdminPrivilegesEvaluator().containsRestApiAdminPermissions(existingEntity)) { - return ValidationResult.error(RestStatus.FORBIDDEN, forbiddenMessage("Access denied")); } - } else { - final var configuration = securityConfiguration.configuration(); - final var configEntityContent = Utils.toConfigObject( + } + if (securityConfiguration.requestContent() != null) { + final var newConfigEntityContent = Utils.toConfigObject( securityConfiguration.requestContent(), configuration.getImplementingClass() ); - if (restApiAdminPrivilegesEvaluator().containsRestApiAdminPermissions(configEntityContent)) { - + if (restApiAdminPrivilegesEvaluator().containsRestApiAdminPermissions(newConfigEntityContent)) { return ValidationResult.error(RestStatus.FORBIDDEN, forbiddenMessage("Access denied")); } } diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/ActionGroupsApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/ActionGroupsApiTest.java index fb166779ac..fbeb3473fc 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/ActionGroupsApiTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/ActionGroupsApiTest.java @@ -441,25 +441,46 @@ public void testActionGroupsApiForActionGroupsRestApiAdmin() throws Exception { } @Test - public void testCreateActionGroupWithRestAdminPermissionsForbidden() throws Exception { + public void testCreateOrUpdateRestApiAdminActionGroupForbidden() throws Exception { setupWithRestRoles(Settings.builder().put(SECURITY_RESTAPI_ADMIN_ENABLED, true).build()); rh.sendAdminCertificate = false; - final Header restApiAdminHeader = encodeBasicHeader("rest_api_admin_user", "rest_api_admin_user"); - final Header restApiAdminActionGroupsHeader = encodeBasicHeader("rest_api_admin_actiongroups", "rest_api_admin_actiongroups"); - final Header restApiHeader = encodeBasicHeader("test", "test"); + final var userHeaders = List.of( + encodeBasicHeader("admin", "admin"), + encodeBasicHeader("test", "test"), + encodeBasicHeader("rest_api_admin_user", "rest_api_admin_user"), + encodeBasicHeader("rest_api_admin_actiongroups", "rest_api_admin_actiongroups") + ); + for (final var userHeader : userHeaders) { + // attempt to create new action group with REST admin permissions + verifyPutForbidden("new_rest_api_admin_group", restAdminAllowedActions(), userHeader); + verifyPatchForbidden(createPatchRestAdminPermissionsPayload("new_rest_api_admin_group", "add"), userHeader); + + // attempt to update existing action group which has REST admin permissions + verifyPutForbidden("rest_admin_action_group", restAdminAllowedActions(), userHeader); + verifyPatchForbidden(createPatchRestAdminPermissionsPayload("rest_admin_action_group", "replace"), userHeader); + + // attempt to update existing action group with REST admin permissions + verifyPutForbidden("OPENDISTRO_SECURITY_CLUSTER_ALL", restAdminAllowedActions(), userHeader); + verifyPatchForbidden(createPatchRestAdminPermissionsPayload("OPENDISTRO_SECURITY_CLUSTER_ALL", "replace"), userHeader); + + // attempt to delete + verifyDeleteForbidden("rest_admin_action_group", userHeader); + verifyPatchForbidden(createPatchRestAdminPermissionsPayload("rest_admin_action_group", "remove"), userHeader); + } + } - HttpResponse response = rh.executePutRequest(ENDPOINT + "/rest_api_admin_group", restAdminAllowedActions(), restApiAdminHeader); - Assert.assertEquals(response.getBody(), HttpStatus.SC_FORBIDDEN, response.getStatusCode()); - response = rh.executePutRequest(ENDPOINT + "/rest_api_admin_group", restAdminAllowedActions(), restApiAdminActionGroupsHeader); - Assert.assertEquals(response.getBody(), HttpStatus.SC_FORBIDDEN, response.getStatusCode()); - response = rh.executePutRequest(ENDPOINT + "/rest_api_admin_group", restAdminAllowedActions(), restApiHeader); + void verifyPutForbidden(final String actionGroupName, final String payload, final Header... header) { + HttpResponse response = rh.executePutRequest(ENDPOINT + "/" + actionGroupName, payload, header); Assert.assertEquals(response.getBody(), HttpStatus.SC_FORBIDDEN, response.getStatusCode()); + } - response = rh.executePatchRequest(ENDPOINT, restAdminPatchBody(), restApiAdminHeader); - Assert.assertEquals(response.getBody(), HttpStatus.SC_FORBIDDEN, response.getStatusCode()); - response = rh.executePatchRequest(ENDPOINT, restAdminPatchBody(), restApiAdminActionGroupsHeader); + void verifyPatchForbidden(final String payload, final Header... header) { + HttpResponse response = rh.executePatchRequest(ENDPOINT, payload, header); Assert.assertEquals(response.getBody(), HttpStatus.SC_FORBIDDEN, response.getStatusCode()); - response = rh.executePatchRequest(ENDPOINT, restAdminPatchBody(), restApiHeader); + } + + void verifyDeleteForbidden(final String actionGroupName, final Header... header) { + HttpResponse response = rh.executeDeleteRequest(ENDPOINT + "/" + actionGroupName, header); Assert.assertEquals(response.getBody(), HttpStatus.SC_FORBIDDEN, response.getStatusCode()); } @@ -469,13 +490,30 @@ String restAdminAllowedActions() throws JsonProcessingException { return DefaultObjectMapper.objectMapper.writeValueAsString(rootNode); } - String restAdminPatchBody() throws JsonProcessingException { + private String createPatchRestAdminPermissionsPayload(final String actionGroup, final String op) throws JsonProcessingException { final ArrayNode rootNode = DefaultObjectMapper.objectMapper.createArrayNode(); - final ObjectNode opAddRootNode = DefaultObjectMapper.objectMapper.createObjectNode(); + final ObjectNode opAddObjectNode = DefaultObjectMapper.objectMapper.createObjectNode(); final ObjectNode allowedActionsNode = DefaultObjectMapper.objectMapper.createObjectNode(); allowedActionsNode.set("allowed_actions", clusterPermissionsForRestAdmin("cluster/*")); - opAddRootNode.put("op", "add").put("path", "/rest_api_admin_group").set("value", allowedActionsNode); - rootNode.add(opAddRootNode); + if ("add".equals(op)) { + opAddObjectNode.put("op", "add").put("path", "/" + actionGroup).set("value", allowedActionsNode); + rootNode.add(opAddObjectNode); + } + + if ("remove".equals(op)) { + final ObjectNode opRemoveObjectNode = DefaultObjectMapper.objectMapper.createObjectNode(); + opRemoveObjectNode.put("op", "remove").put("path", "/" + actionGroup); + rootNode.add(opRemoveObjectNode); + } + + if ("replace".equals(op)) { + final ObjectNode replaceRemoveObjectNode = DefaultObjectMapper.objectMapper.createObjectNode(); + replaceRemoveObjectNode.put("op", "replace") + .put("path", "/" + actionGroup + "/allowed_actions") + .set("value", clusterPermissionsForRestAdmin("*")); + + rootNode.add(replaceRemoveObjectNode); + } return DefaultObjectMapper.objectMapper.writeValueAsString(rootNode); } diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/RolesApiActionValidationTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/RolesApiActionValidationTest.java index 88a358dcb2..7fe089c0ba 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/RolesApiActionValidationTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/RolesApiActionValidationTest.java @@ -28,21 +28,18 @@ public class RolesApiActionValidationTest extends AbstractApiActionValidationTes @Test public void isAllowedToChangeImmutableEntity() throws Exception { - when(restApiAdminPrivilegesEvaluator.isCurrentUserAdminFor(Endpoint.ROLES)).thenReturn(true); - final var role = new RoleV7(); role.setCluster_permissions(restApiAdminPermissions()); - final var rolesApiActionEndpointValidator = new RolesApiAction(clusterService, threadPool, securityApiDependencies).createEndpointValidator(); - final var result = rolesApiActionEndpointValidator.isAllowedToChangeImmutableEntity(SecurityConfiguration.of( "sss", configuration)); + final var rolesApiActionEndpointValidator = new RolesApiAction(clusterService, threadPool, securityApiDependencies) + .createEndpointValidator(); + final var result = rolesApiActionEndpointValidator.isAllowedToChangeImmutableEntity(SecurityConfiguration.of("sss", configuration)); assertTrue(result.isValid()); } @Test public void isNotAllowedRightsToChangeImmutableEntity() throws Exception { - when(restApiAdminPrivilegesEvaluator.isCurrentUserAdminFor(Endpoint.ROLES)).thenReturn(false); - final var role = new RoleV7(); role.setCluster_permissions(restApiAdminPermissions()); @@ -50,8 +47,9 @@ public void isNotAllowedRightsToChangeImmutableEntity() throws Exception { Mockito.when(configuration.getCEntry("sss")).thenReturn(role); when(restApiAdminPrivilegesEvaluator.containsRestApiAdminPermissions(any(Object.class))).thenCallRealMethod(); - final var rolesApiActionEndpointValidator = new RolesApiAction(clusterService, threadPool, securityApiDependencies).createEndpointValidator(); - final var result = rolesApiActionEndpointValidator.isAllowedToChangeImmutableEntity(SecurityConfiguration.of( "sss", configuration)); + final var rolesApiActionEndpointValidator = new RolesApiAction(clusterService, threadPool, securityApiDependencies) + .createEndpointValidator(); + final var result = rolesApiActionEndpointValidator.isAllowedToChangeImmutableEntity(SecurityConfiguration.of("sss", configuration)); assertFalse(result.isValid()); assertEquals(RestStatus.FORBIDDEN, result.status()); diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/RolesApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/RolesApiTest.java index bb11bb2226..eb3ad7d4e5 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/RolesApiTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/RolesApiTest.java @@ -691,105 +691,74 @@ public void testRolesApiWithRestApiRolePermission() throws Exception { } @Test - public void testCreateOrUpdateRestApiAdminRoleForbiddenForNonSuperAdmin() throws Exception { + public void testCrudRestApiAdminRoleForbidden() throws Exception { setupWithRestRoles(Settings.builder().put(SECURITY_RESTAPI_ADMIN_ENABLED, true).build()); rh.sendAdminCertificate = false; - final Header restApiAdminHeader = encodeBasicHeader("rest_api_admin_user", "rest_api_admin_user"); - final Header adminHeader = encodeBasicHeader("admin", "admin"); - final Header restApiHeader = encodeBasicHeader("test", "test"); - - final String restAdminPermissionsPayload = createRestAdminPermissionsPayload("cluster/*"); - HttpResponse response = rh.executePutRequest( - ENDPOINT + "/roles/new_rest_admin_role", - restAdminPermissionsPayload, - restApiAdminHeader - ); - Assert.assertEquals(HttpStatus.SC_CREATED, response.getStatusCode()); - response = rh.executePutRequest(ENDPOINT + "/roles/rest_admin_role_to_delete", restAdminPermissionsPayload, restApiAdminHeader); - Assert.assertEquals(HttpStatus.SC_CREATED, response.getStatusCode()); - - // attempt to create a new rest admin role by admin - response = rh.executePutRequest(ENDPOINT + "/roles/some_rest_admin_role", restAdminPermissionsPayload, adminHeader); - Assert.assertEquals(HttpStatus.SC_FORBIDDEN, response.getStatusCode()); - - // attempt to update exiting admin role - response = rh.executePutRequest(ENDPOINT + "/roles/new_rest_admin_role", restAdminPermissionsPayload, adminHeader); - Assert.assertEquals(HttpStatus.SC_FORBIDDEN, response.getStatusCode()); - - // attempt to patch exiting admin role - response = rh.executePatchRequest( - ENDPOINT + "/roles/new_rest_admin_role", - createPatchRestAdminPermissionsPayload("replace"), - adminHeader - ); - Assert.assertEquals(HttpStatus.SC_FORBIDDEN, response.getStatusCode()); - - // attempt to update exiting admin role - response = rh.executePutRequest(ENDPOINT + "/roles/new_rest_admin_role", restAdminPermissionsPayload, restApiHeader); - Assert.assertEquals(HttpStatus.SC_FORBIDDEN, response.getStatusCode()); - - // attempt to create a new rest admin role by admin - response = rh.executePutRequest(ENDPOINT + "/roles/some_rest_admin_role", restAdminPermissionsPayload, restApiHeader); - Assert.assertEquals(HttpStatus.SC_FORBIDDEN, response.getStatusCode()); - - // attempt to patch exiting admin role and crate a new one - response = rh.executePatchRequest(ENDPOINT + "/roles", createPatchRestAdminPermissionsPayload("replace"), restApiHeader); - Assert.assertEquals(HttpStatus.SC_FORBIDDEN, response.getStatusCode()); + final var userHeaders = List.of( + encodeBasicHeader("admin", "admin"), + encodeBasicHeader("test", "test"), + encodeBasicHeader("rest_api_admin_user", "rest_api_admin_user"), + encodeBasicHeader("rest_api_admin_roles", "rest_api_admin_roles") + ); + for (final var userHeader : userHeaders) { + final String restAdminPermissionsPayload = createRestAdminPermissionsPayload("cluster/*"); + // attempt to create a new role + verifyPutForbidden("new_rest_admin_role", restAdminPermissionsPayload, userHeader); + verifyPatchForbidden(createPatchRestAdminPermissionsPayload("new_rest_admin_role", "add"), userHeader); + + // attempt to update existing rest admin role + verifyPutForbidden("rest_api_admin_full_access", restAdminPermissionsPayload, userHeader); + verifyPatchForbidden(createPatchRestAdminPermissionsPayload("rest_api_admin_full_access", "replace"), userHeader); + + // attempt to update non rest admin role with REST admin permissions + verifyPutForbidden("opendistro_security_role_starfleet_captains", restAdminPermissionsPayload, userHeader); + verifyPatchForbidden( + createPatchRestAdminPermissionsPayload("opendistro_security_role_starfleet_captains", "replace"), + userHeader + ); + + // attempt to remove REST admin role + verifyDeleteForbidden("rest_api_admin_full_access", userHeader); + verifyPatchForbidden(createPatchRestAdminPermissionsPayload("rest_api_admin_full_access", "remove"), userHeader); + } + } - response = rh.executePatchRequest(ENDPOINT + "/roles", createPatchRestAdminPermissionsPayload("add"), restApiHeader); - Assert.assertEquals(HttpStatus.SC_FORBIDDEN, response.getStatusCode()); - response = rh.executePatchRequest(ENDPOINT + "/roles", createPatchRestAdminPermissionsPayload("remove"), restApiHeader); + void verifyPutForbidden(final String roleName, final String restAdminPermissionsPayload, final Header... header) { + HttpResponse response = rh.executePutRequest(ENDPOINT + "/roles/" + roleName, restAdminPermissionsPayload, header); Assert.assertEquals(HttpStatus.SC_FORBIDDEN, response.getStatusCode()); } - @Test - public void testDeleteRestApiAdminRoleForbiddenForNonSuperAdmin() throws Exception { - setupWithRestRoles(Settings.builder().put(SECURITY_RESTAPI_ADMIN_ENABLED, true).build()); - rh.sendAdminCertificate = false; - - final Header restApiAdminHeader = encodeBasicHeader("rest_api_admin_user", "rest_api_admin_user"); - final Header adminHeader = encodeBasicHeader("admin", "admin"); - final Header restApiHeader = encodeBasicHeader("test", "test"); - - final String allRestAdminPermissionsPayload = createRestAdminPermissionsPayload("cluster/*"); - - HttpResponse response = rh.executePutRequest( - ENDPOINT + "/roles/new_rest_admin_role", - allRestAdminPermissionsPayload, - restApiAdminHeader - ); - Assert.assertEquals(HttpStatus.SC_CREATED, response.getStatusCode()); - - // attempt to update exiting admin role - response = rh.executeDeleteRequest(ENDPOINT + "/roles/new_rest_admin_role", adminHeader); - Assert.assertEquals(HttpStatus.SC_FORBIDDEN, response.getStatusCode()); + void verifyPatchForbidden(final String restAdminPermissionsPayload, final Header... header) { + HttpResponse response = rh.executePatchRequest(ENDPOINT + "/roles", restAdminPermissionsPayload, header); + Assert.assertEquals(response.getBody(), HttpStatus.SC_FORBIDDEN, response.getStatusCode()); + } - // true to change - response = rh.executeDeleteRequest(ENDPOINT + "/roles/new_rest_admin_role", allRestAdminPermissionsPayload, restApiHeader); + void verifyDeleteForbidden(final String roleName, final Header... header) { + HttpResponse response = rh.executeDeleteRequest(ENDPOINT + "/roles/" + roleName, header); Assert.assertEquals(HttpStatus.SC_FORBIDDEN, response.getStatusCode()); } - private String createPatchRestAdminPermissionsPayload(final String op) throws JsonProcessingException { + private String createPatchRestAdminPermissionsPayload(final String roleName, final String op) throws JsonProcessingException { final ArrayNode rootNode = DefaultObjectMapper.objectMapper.createArrayNode(); final ObjectNode opAddObjectNode = DefaultObjectMapper.objectMapper.createObjectNode(); final ObjectNode clusterPermissionsNode = DefaultObjectMapper.objectMapper.createObjectNode(); clusterPermissionsNode.set("cluster_permissions", clusterPermissionsForRestAdmin("cluster/*")); if ("add".equals(op)) { - opAddObjectNode.put("op", "add").put("path", "/some_rest_admin_role").set("value", clusterPermissionsNode); + opAddObjectNode.put("op", "add").put("path", "/" + roleName).set("value", clusterPermissionsNode); rootNode.add(opAddObjectNode); } if ("remove".equals(op)) { final ObjectNode opRemoveObjectNode = DefaultObjectMapper.objectMapper.createObjectNode(); - opRemoveObjectNode.put("op", "remove").put("path", "/rest_admin_role_to_delete"); + opRemoveObjectNode.put("op", "remove").put("path", "/" + roleName); rootNode.add(opRemoveObjectNode); } if ("replace".equals(op)) { final ObjectNode replaceRemoveObjectNode = DefaultObjectMapper.objectMapper.createObjectNode(); replaceRemoveObjectNode.put("op", "replace") - .put("path", "/new_rest_admin_role/cluster_permissions") + .put("path", "/" + roleName + "/cluster_permissions") .set("value", clusterPermissionsForRestAdmin("*")); rootNode.add(replaceRemoveObjectNode); diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/RolesMappingApiActionValidationTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/RolesMappingApiActionValidationTest.java index 5c041989a6..f7d1d4da0b 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/RolesMappingApiActionValidationTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/RolesMappingApiActionValidationTest.java @@ -35,18 +35,16 @@ public void setupRoles() throws Exception { @Test public void isAllowedRightsToChangeRoleEntity() throws Exception { - when(restApiAdminPrivilegesEvaluator.isCurrentUserAdminFor(Endpoint.ROLESMAPPING)).thenReturn(true); final var rolesMappingApiActionEndpointValidator = new RolesMappingApiAction(clusterService, threadPool, securityApiDependencies) .createEndpointValidator(); final var result = rolesMappingApiActionEndpointValidator.isAllowedToChangeImmutableEntity( - SecurityConfiguration.of("rest_api_admin_role", configuration) + SecurityConfiguration.of("rest_api_admin_role", configuration) ); assertTrue(result.isValid()); } @Test public void isNotAllowedNoRightsToChangeRoleEntity() throws Exception { - when(restApiAdminPrivilegesEvaluator.isCurrentUserAdminFor(Endpoint.ROLESMAPPING)).thenReturn(false); when(restApiAdminPrivilegesEvaluator.containsRestApiAdminPermissions(any(Object.class))).thenCallRealMethod(); final var rolesApiActionEndpointValidator = @@ -61,7 +59,6 @@ public void isNotAllowedNoRightsToChangeRoleEntity() throws Exception { @Test public void onConfigChangeShouldCheckRoles() throws Exception { - when(restApiAdminPrivilegesEvaluator.isCurrentUserAdminFor(Endpoint.ROLESMAPPING)).thenReturn(false); when(restApiAdminPrivilegesEvaluator.containsRestApiAdminPermissions(any(Object.class))).thenCallRealMethod(); when(configurationRepository.getConfigurationsFromIndex(List.of(CType.ROLES), false)) .thenReturn(Map.of(CType.ROLES, rolesConfiguration)); diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/RolesMappingApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/RolesMappingApiTest.java index 077c852466..dc2ba33e6e 100644 --- a/src/test/java/org/opensearch/security/dlic/rest/api/RolesMappingApiTest.java +++ b/src/test/java/org/opensearch/security/dlic/rest/api/RolesMappingApiTest.java @@ -14,6 +14,7 @@ import java.util.List; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.hc.core5.http.Header; @@ -558,97 +559,83 @@ void verifyNonSuperAdminUser(final Header[] header) throws Exception { } @Test - public void testChangeRestApiAdminRoleMappingForbiddenForNonSuperAdmin() throws Exception { + public void testChangeRestApiAdminRoleMappingForbidden() throws Exception { setupWithRestRoles(Settings.builder().put(SECURITY_RESTAPI_ADMIN_ENABLED, true).build()); rh.sendAdminCertificate = false; - final Header restApiAdminHeader = encodeBasicHeader("rest_api_admin_user", "rest_api_admin_user"); - final Header adminHeader = encodeBasicHeader("admin", "admin"); - final Header restApiHeader = encodeBasicHeader("test", "test"); - - HttpResponse response = rh.executePutRequest( - ENDPOINT + "/roles/new_rest_api_role", - createRestAdminPermissionsPayload(), - restApiAdminHeader - ); - Assert.assertEquals(response.getBody(), HttpStatus.SC_CREATED, response.getStatusCode()); - response = rh.executePutRequest( - ENDPOINT + "/roles/new_rest_api_role_without_mapping", - createRestAdminPermissionsPayload(), - restApiAdminHeader + final var userHeaders = List.of( + encodeBasicHeader("admin", "admin"), + encodeBasicHeader("test", "test"), + encodeBasicHeader("rest_api_admin_user", "rest_api_admin_user"), + encodeBasicHeader("rest_api_admin_rolesmapping", "rest_api_admin_rolesmapping") ); - Assert.assertEquals(HttpStatus.SC_CREATED, response.getStatusCode()); - response = rh.executePutRequest( - ENDPOINT + "/rolesmapping/new_rest_api_role", - createUsersPayload("a", "b", "c"), - restApiAdminHeader - ); - Assert.assertEquals(HttpStatus.SC_CREATED, response.getStatusCode()); - verifyRestApiPutAndDeleteForNonRestApiAdmin(adminHeader); - verifyRestApiPutAndDeleteForNonRestApiAdmin(restApiHeader); - verifyRestApiPatchForNonRestApiAdmin(adminHeader, false); - verifyRestApiPatchForNonRestApiAdmin(restApiHeader, false); - verifyRestApiPatchForNonRestApiAdmin(adminHeader, true); - verifyRestApiPatchForNonRestApiAdmin(restApiHeader, true); - } + for (final var userHeader : userHeaders) { + // create new mapping for existing group + verifyPutForbidden("rest_api_admin_roles_mapping_test_without_mapping", createUsers("c", "d"), userHeader); + verifyPatchForbidden(createPatchPayload("rest_api_admin_roles_mapping_test_without_mapping", "add"), userHeader); - private void verifyRestApiPutAndDeleteForNonRestApiAdmin(final Header header) throws Exception { - HttpResponse response = rh.executePutRequest( - ENDPOINT + "/rolesmapping/new_rest_api_role", - createUsersPayload("a", "b", "c"), - header - ); - Assert.assertEquals(HttpStatus.SC_FORBIDDEN, response.getStatusCode()); + // update existing mapping with additional users + verifyPutForbidden("rest_api_admin_roles_mapping_test_with_mapping", createUsers("c", "d"), userHeader); + verifyPatchForbidden(createPatchPayload("rest_api_admin_roles_mapping_test_with_mapping", "replace"), userHeader); - response = rh.executeDeleteRequest(ENDPOINT + "/rolesmapping/new_rest_api_role", "", header); - Assert.assertEquals(HttpStatus.SC_FORBIDDEN, response.getStatusCode()); + // delete existing role mapping forbidden + verifyDeleteForbidden("rest_api_admin_roles_mapping_test_with_mapping", userHeader); + verifyPatchForbidden(createPatchPayload("rest_api_admin_roles_mapping_test_with_mapping", "remove"), userHeader); + } } - private void verifyRestApiPatchForNonRestApiAdmin(final Header header, boolean bulk) throws Exception { - String path = ENDPOINT + "/rolesmapping"; - if (!bulk) { - path += "/new_rest_api_role"; - } - HttpResponse response = rh.executePatchRequest(path, createPathPayload("add"), header); - System.err.println(response.getBody()); + void verifyPutForbidden(final String roleMappingName, final String payload, final Header... header) { + HttpResponse response = rh.executePutRequest(ENDPOINT + "/rolesmapping/" + roleMappingName, payload, header); Assert.assertEquals(HttpStatus.SC_FORBIDDEN, response.getStatusCode()); + } - response = rh.executePatchRequest(path, createPathPayload("replace"), header); + void verifyPatchForbidden(final String payload, final Header... header) { + HttpResponse response = rh.executePatchRequest(ENDPOINT + "/rolesmapping", payload, header); Assert.assertEquals(HttpStatus.SC_FORBIDDEN, response.getStatusCode()); + } - response = rh.executePatchRequest(path, createPathPayload("remove"), header); + void verifyDeleteForbidden(final String roleMappingName, final Header... header) { + HttpResponse response = rh.executeDeleteRequest(ENDPOINT + "/rolesmapping/" + roleMappingName, header); Assert.assertEquals(HttpStatus.SC_FORBIDDEN, response.getStatusCode()); } - private ObjectNode createUsersObjectNode(final String... users) { - final ArrayNode usersArray = DefaultObjectMapper.objectMapper.createArrayNode(); - for (final String user : users) { - usersArray.add(user); + private String createPatchPayload(final String roleName, final String op) throws JsonProcessingException { + final ArrayNode rootNode = DefaultObjectMapper.objectMapper.createArrayNode(); + final ObjectNode opAddObjectNode = DefaultObjectMapper.objectMapper.createObjectNode(); + final ObjectNode clusterPermissionsNode = DefaultObjectMapper.objectMapper.createObjectNode(); + clusterPermissionsNode.set("users", createUsersArray("c", "d")); + if ("add".equals(op)) { + opAddObjectNode.put("op", "add").put("path", "/" + roleName).set("value", clusterPermissionsNode); + rootNode.add(opAddObjectNode); } - return DefaultObjectMapper.objectMapper.createObjectNode().set("users", usersArray); - } - private String createUsersPayload(final String... users) throws JsonProcessingException { - return DefaultObjectMapper.objectMapper.writeValueAsString(createUsersObjectNode(users)); - } - - private String createPathPayload(final String op) throws JsonProcessingException { - final ArrayNode arrayNode = DefaultObjectMapper.objectMapper.createArrayNode(); - final ObjectNode opNode = DefaultObjectMapper.objectMapper.createObjectNode(); - opNode.put("op", op); - if ("add".equals(op)) { - opNode.put("path", "/new_rest_api_role_without_mapping"); - opNode.set("value", createUsersObjectNode("d", "e", "f")); + if ("remove".equals(op)) { + final ObjectNode opRemoveObjectNode = DefaultObjectMapper.objectMapper.createObjectNode(); + opRemoveObjectNode.put("op", "remove").put("path", "/" + roleName); + rootNode.add(opRemoveObjectNode); } + if ("replace".equals(op)) { - opNode.put("path", "/new_rest_api_role"); - opNode.set("value", createUsersObjectNode("g", "h", "i")); + final ObjectNode replaceRemoveObjectNode = DefaultObjectMapper.objectMapper.createObjectNode(); + replaceRemoveObjectNode.put("op", "replace").put("path", "/" + roleName + "/users").set("value", createUsersArray("c", "d")); + + rootNode.add(replaceRemoveObjectNode); } - if ("remove".equals(op)) { - opNode.put("path", "/new_rest_api_role"); + return DefaultObjectMapper.objectMapper.writeValueAsString(rootNode); + } + + private String createUsers(final String... users) throws JsonProcessingException { + final var o = DefaultObjectMapper.objectMapper.createObjectNode().set("users", createUsersArray("c", "d")); + return DefaultObjectMapper.writeValueAsString(o, false); + } + + private JsonNode createUsersArray(final String... users) { + final ArrayNode usersArray = DefaultObjectMapper.objectMapper.createArrayNode(); + for (final String user : users) { + usersArray.add(user); } - return DefaultObjectMapper.objectMapper.writeValueAsString(arrayNode.add(opNode)); + return usersArray; } @Test diff --git a/src/test/resources/restapi/action_groups.yml b/src/test/resources/restapi/action_groups.yml index 638f65f72f..4ec858a69e 100644 --- a/src/test/resources/restapi/action_groups.yml +++ b/src/test/resources/restapi/action_groups.yml @@ -132,3 +132,16 @@ OPENDISTRO_SECURITY_SUGGEST: - "indices:data/read/suggest*" type: "index" description: "Migrated from v6" +rest_admin_action_group: + allowed_actions: + - 'restapi:admin/actiongroups' + - 'restapi:admin/allowlist' + - 'restapi:admin/config/update' + - 'restapi:admin/internalusers' + - 'restapi:admin/nodesdn' + - 'restapi:admin/roles' + - 'restapi:admin/rolesmapping' + - 'restapi:admin/ssl/certs/info' + - 'restapi:admin/ssl/certs/reload' + - 'restapi:admin/tenants' + type: "cluster" diff --git a/src/test/resources/restapi/roles.yml b/src/test/resources/restapi/roles.yml index 1c3756cb4d..925b051bb3 100644 --- a/src/test/resources/restapi/roles.yml +++ b/src/test/resources/restapi/roles.yml @@ -393,6 +393,32 @@ opendistro_security_role_starfleet_captains: allowed_actions: - "CRUD_UT" tenant_permissions: [] +rest_api_admin_roles_mapping_test_without_mapping: + reserved: true + cluster_permissions: + - 'restapi:admin/actiongroups' + - 'restapi:admin/allowlist' + - 'restapi:admin/config/update' + - 'restapi:admin/internalusers' + - 'restapi:admin/nodesdn' + - 'restapi:admin/roles' + - 'restapi:admin/rolesmapping' + - 'restapi:admin/ssl/certs/info' + - 'restapi:admin/ssl/certs/reload' + - 'restapi:admin/tenants' +rest_api_admin_roles_mapping_test_with_mapping: + reserved: true + cluster_permissions: + - 'restapi:admin/actiongroups' + - 'restapi:admin/allowlist' + - 'restapi:admin/config/update' + - 'restapi:admin/internalusers' + - 'restapi:admin/nodesdn' + - 'restapi:admin/roles' + - 'restapi:admin/rolesmapping' + - 'restapi:admin/ssl/certs/info' + - 'restapi:admin/ssl/certs/reload' + - 'restapi:admin/tenants' rest_api_admin_full_access: reserved: true cluster_permissions: diff --git a/src/test/resources/restapi/roles_mapping.yml b/src/test/resources/restapi/roles_mapping.yml index 8bfe826247..73673ae3ff 100644 --- a/src/test/resources/restapi/roles_mapping.yml +++ b/src/test/resources/restapi/roles_mapping.yml @@ -217,6 +217,9 @@ opendistro_security_role_host2: - "opendistro_security_host_localhost" and_backend_roles: [] description: "Migrated from v6" +rest_api_admin_roles_mapping_test_with_mapping: + reserved: true + users: [a, b] rest_api_admin_full_access: reserved: false hidden: true From d87ab3f7372ff799ca75fbf9741f5111e12d84dd Mon Sep 17 00:00:00 2001 From: Darshit Chanpura <35282393+DarshitChanpura@users.noreply.github.com> Date: Tue, 9 Apr 2024 11:10:53 -0400 Subject: [PATCH 175/204] Adds saml auth header to differentiate saml requests and prevents auto login as anonymous user when basic authentication fails (#4152) Signed-off-by: Darshit Chanpura --- .../http/BasicWithAnonymousAuthTests.java | 112 ++++++++++++++++++ .../security/auth/BackendRegistry.java | 47 +++++--- .../security/rest/SecurityInfoAction.java | 3 + 3 files changed, 147 insertions(+), 15 deletions(-) create mode 100644 src/integrationTest/java/org/opensearch/security/http/BasicWithAnonymousAuthTests.java diff --git a/src/integrationTest/java/org/opensearch/security/http/BasicWithAnonymousAuthTests.java b/src/integrationTest/java/org/opensearch/security/http/BasicWithAnonymousAuthTests.java new file mode 100644 index 0000000000..842d5c4dd5 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/http/BasicWithAnonymousAuthTests.java @@ -0,0 +1,112 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security.http; + +import java.util.Map; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain; +import org.opensearch.test.framework.TestSecurityConfig.User; +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; + +import static org.apache.http.HttpStatus.SC_OK; +import static org.apache.http.HttpStatus.SC_UNAUTHORIZED; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class BasicWithAnonymousAuthTests { + static final User TEST_USER = new User("test_user").password("s3cret"); + + public static final String CUSTOM_ATTRIBUTE_NAME = "superhero"; + static final User SUPER_USER = new User("super-user").password("super-password").attr(CUSTOM_ATTRIBUTE_NAME, "true"); + public static final String NOT_EXISTING_USER = "not-existing-user"; + public static final String INVALID_PASSWORD = "secret-password"; + + public static final AuthcDomain AUTHC_DOMAIN = new AuthcDomain("basic", 0).httpAuthenticatorWithChallenge("basic").backend("internal"); + + @ClassRule + public static final LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.SINGLENODE) + .anonymousAuth(true) + .authc(AUTHC_DOMAIN) + .users(TEST_USER, SUPER_USER) + .build(); + + /** No automatic login post anonymous auth request **/ + @Test + public void testShouldRespondWith401WhenUserDoesNotExist() { + try (TestRestClient client = cluster.getRestClient(NOT_EXISTING_USER, INVALID_PASSWORD)) { + HttpResponse response = client.getAuthInfo(); + + assertThat(response, is(notNullValue())); + response.assertStatusCode(SC_UNAUTHORIZED); + } + } + + @Test + public void testShouldRespondWith401WhenUserNameIsIncorrect() { + try (TestRestClient client = cluster.getRestClient(NOT_EXISTING_USER, TEST_USER.getPassword())) { + HttpResponse response = client.getAuthInfo(); + + assertThat(response, is(notNullValue())); + response.assertStatusCode(SC_UNAUTHORIZED); + } + } + + @Test + public void testShouldRespondWith401WhenPasswordIsIncorrect() { + try (TestRestClient client = cluster.getRestClient(TEST_USER.getName(), INVALID_PASSWORD)) { + HttpResponse response = client.getAuthInfo(); + + assertThat(response, is(notNullValue())); + response.assertStatusCode(SC_UNAUTHORIZED); + } + } + + /** Test `?auth_type=""` param to authinfo request **/ + @Test + public void testShouldAutomaticallyLoginAsAnonymousIfNoCredentialsArePassed() { + try (TestRestClient client = cluster.getRestClient()) { + + HttpResponse response = client.getAuthInfo(); + + assertThat(response, is(notNullValue())); + response.assertStatusCode(SC_OK); + + HttpResponse response2 = client.getAuthInfo(Map.of("auth_type", "anonymous")); + + assertThat(response2, is(notNullValue())); + response2.assertStatusCode(SC_OK); + } + } + + @Test + public void testShouldNotAutomaticallyLoginAsAnonymousIfRequestIsNonAnonymousLogin() { + try (TestRestClient client = cluster.getRestClient()) { + + HttpResponse response = client.getAuthInfo(Map.of("auth_type", "saml")); + + assertThat(response, is(notNullValue())); + response.assertStatusCode(SC_UNAUTHORIZED); + + // should contain a redirect link + assertThat(response.containHeader("WWW-Authenticate"), is(true)); + } + } +} diff --git a/src/main/java/org/opensearch/security/auth/BackendRegistry.java b/src/main/java/org/opensearch/security/auth/BackendRegistry.java index 3ab9a2afc9..97c060be35 100644 --- a/src/main/java/org/opensearch/security/auth/BackendRegistry.java +++ b/src/main/java/org/opensearch/security/auth/BackendRegistry.java @@ -32,6 +32,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.SortedSet; @@ -44,6 +45,7 @@ import com.google.common.cache.RemovalListener; import com.google.common.cache.RemovalNotification; import com.google.common.collect.Multimap; +import org.apache.http.HttpHeaders; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -190,7 +192,7 @@ public void onDynamicConfigModelChanged(DynamicConfigModel dcm) { * @param request * @return The authenticated user, null means another roundtrip * @throws OpenSearchSecurityException - */ + */ public boolean authenticate(final SecurityRequestChannel request) { final boolean isDebugEnabled = log.isDebugEnabled(); final boolean isBlockedBasedOnAddress = request.getRemoteAddress() @@ -286,7 +288,7 @@ public boolean authenticate(final SecurityRequestChannel request) { if (ac == null) { // no credentials found in request - if (anonymousAuthEnabled) { + if (anonymousAuthEnabled && isRequestForAnonymousLogin(request.params(), request.getHeaders())) { continue; } @@ -386,19 +388,6 @@ public boolean authenticate(final SecurityRequestChannel request) { log.debug("User still not authenticated after checking {} auth domains", restAuthDomains.size()); } - if (authCredentials == null && anonymousAuthEnabled) { - final String tenant = resolveTenantFrom(request); - User anonymousUser = new User(User.ANONYMOUS.getName(), new HashSet(User.ANONYMOUS.getRoles()), null); - anonymousUser.setRequestedTenant(tenant); - - threadPool.getThreadContext().putTransient(ConfigConstants.OPENDISTRO_SECURITY_USER, anonymousUser); - auditLog.logSucceededLogin(anonymousUser.getName(), false, null, request); - if (isDebugEnabled) { - log.debug("Anonymous User is authenticated"); - } - return true; - } - Optional challengeResponse = Optional.empty(); if (firstChallengingHttpAuthenticator != null) { @@ -415,6 +404,19 @@ public boolean authenticate(final SecurityRequestChannel request) { } } + if (authCredentials == null && anonymousAuthEnabled && isRequestForAnonymousLogin(request.params(), request.getHeaders())) { + final String tenant = resolveTenantFrom(request); + User anonymousUser = new User(User.ANONYMOUS.getName(), new HashSet(User.ANONYMOUS.getRoles()), null); + anonymousUser.setRequestedTenant(tenant); + + threadPool.getThreadContext().putTransient(ConfigConstants.OPENDISTRO_SECURITY_USER, anonymousUser); + auditLog.logSucceededLogin(anonymousUser.getName(), false, null, request); + if (isDebugEnabled) { + log.debug("Anonymous User is authenticated"); + } + return true; + } + log.warn( "Authentication finally failed for {} from {}", authCredentials == null ? null : authCredentials.getUsername(), @@ -432,6 +434,21 @@ public boolean authenticate(final SecurityRequestChannel request) { return authenticated; } + /** + * Checks if incoming auth request is from an anonymous user + * Defaults all requests to yes, to allow anonymous authentication to succeed + * @param params the query parameters passed in this request + * @return false only if an explicit `auth_type` param is supplied, and its value is not anonymous, OR + * if request contains no authorization headers + * otherwise returns true + */ + private boolean isRequestForAnonymousLogin(Map params, Map> headers) { + if (params.containsKey("auth_type")) { + return params.get("auth_type").equals("anonymous"); + } + return !headers.containsKey(HttpHeaders.AUTHORIZATION); + } + private String resolveTenantFrom(final SecurityRequest request) { return Optional.ofNullable(request.header("securitytenant")).orElse(request.header("security_tenant")); } diff --git a/src/main/java/org/opensearch/security/rest/SecurityInfoAction.java b/src/main/java/org/opensearch/security/rest/SecurityInfoAction.java index f6cf7f82ee..64075d5d0e 100644 --- a/src/main/java/org/opensearch/security/rest/SecurityInfoAction.java +++ b/src/main/java/org/opensearch/security/rest/SecurityInfoAction.java @@ -91,6 +91,9 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { final boolean verbose = request.paramAsBoolean("verbose", false); + // need to consume `auth_type` param, without which a 500 is thrown on front-end + final String authType = request.param("auth_type", ""); + return new RestChannelConsumer() { @Override From a5a5e032bd491c42c54d8d313d7359c366093cd1 Mon Sep 17 00:00:00 2001 From: Chenyang Ji Date: Thu, 11 Apr 2024 05:52:20 -0700 Subject: [PATCH 176/204] Add index permissions for query insights exporters (#4229) Signed-off-by: Chenyang Ji --- config/roles.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/roles.yml b/config/roles.yml index 1b4c13745f..7b45deb813 100644 --- a/config/roles.yml +++ b/config/roles.yml @@ -429,3 +429,8 @@ query_insights_full_access: reserved: true cluster_permissions: - 'cluster:admin/opensearch/insights/top_queries/*' + index_permissions: + - index_patterns: + - 'top_queries_by_*' + allowed_actions: + - "indices_all" From a5b891463699b1cded2acdf55fd44fd4b5bcac63 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 11 Apr 2024 14:25:02 -0400 Subject: [PATCH 177/204] Upgrade codecov/codecov-action to v4 (#4236) Signed-off-by: Craig Perkins --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45f306318e..9a656e02b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,7 +85,7 @@ jobs: with: attempt_limit: 5 attempt_delay: 2000 - action: codecov/codecov-action@v3 + action: codecov/codecov-action@v4 with: | token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true From e4786673e041a1db83abf839e89d5a3b980c510d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 07:41:06 -0400 Subject: [PATCH 178/204] Bump open_saml_version from 4.3.1 to 4.3.2 (#4239) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 51d9fedb7a..8c4b17bcc6 100644 --- a/build.gradle +++ b/build.gradle @@ -27,7 +27,7 @@ buildscript { common_utils_version = System.getProperty("common_utils.version", '3.0.0.0-SNAPSHOT') kafka_version = '3.7.0' apache_cxf_version = '4.0.4' - open_saml_version = '4.3.1' + open_saml_version = '4.3.2' one_login_java_saml = '2.9.0' jjwt_version = '0.12.5' guava_version = '32.1.3-jre' From 276fc016ea95fcd2d43a6327769331cfa5311377 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 07:42:13 -0400 Subject: [PATCH 179/204] Bump net.shibboleth.utilities:java-support from 8.4.1 to 8.4.2 (#4240) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8c4b17bcc6..c38da1dbb0 100644 --- a/build.gradle +++ b/build.gradle @@ -612,7 +612,7 @@ dependencies { testImplementation 'org.apache.camel:camel-xmlsecurity:3.22.1' //OpenSAML - implementation 'net.shibboleth.utilities:java-support:8.4.1' + implementation 'net.shibboleth.utilities:java-support:8.4.2' runtimeOnly "io.dropwizard.metrics:metrics-core:4.2.25" implementation "com.onelogin:java-saml:${one_login_java_saml}" implementation "com.onelogin:java-saml-core:${one_login_java_saml}" From f539650eb2acf423450ccf612e96d6641939793d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 07:43:31 -0400 Subject: [PATCH 180/204] Bump commons-io:commons-io from 2.16.0 to 2.16.1 (#4242) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index c38da1dbb0..7cf0c9c361 100644 --- a/build.gradle +++ b/build.gradle @@ -720,7 +720,7 @@ dependencies { integrationTestImplementation 'junit:junit:4.13.2' integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}" integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}" - integrationTestImplementation 'commons-io:commons-io:2.16.0' + integrationTestImplementation 'commons-io:commons-io:2.16.1' integrationTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}" integrationTestImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}" integrationTestImplementation 'org.hamcrest:hamcrest:2.2' From c33be3fb99f708f17762415e44b18aeb8b582298 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 07:44:00 -0400 Subject: [PATCH 181/204] Bump Wandalen/wretry.action from 3.1.0 to 3.3.0 (#4243) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a656e02b6..94fbf11765 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,7 +81,7 @@ jobs: working-directory: downloaded-artifacts - name: Upload Coverage with retry - uses: Wandalen/wretry.action@v3.1.0 + uses: Wandalen/wretry.action@v3.3.0 with: attempt_limit: 5 attempt_delay: 2000 From 1862787d5ba4e8c1d7dea6ca4c865c4f9606d5fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 07:44:37 -0400 Subject: [PATCH 182/204] Bump ch.qos.logback:logback-classic from 1.5.3 to 1.5.5 (#4244) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 7cf0c9c361..a668c38adc 100644 --- a/build.gradle +++ b/build.gradle @@ -496,7 +496,7 @@ configurations { force "org.apache.httpcomponents:httpcore:4.4.16" force "com.google.errorprone:error_prone_annotations:2.26.1" force "org.checkerframework:checker-qual:3.42.0" - force "ch.qos.logback:logback-classic:1.5.3" + force "ch.qos.logback:logback-classic:1.5.5" } } From dc855137125ce2dc39236eb8276fdb6d71ab5f89 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 08:40:21 -0400 Subject: [PATCH 183/204] Bump spring_version from 5.3.33 to 5.3.34 (#4241) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a668c38adc..a4e4a7d037 100644 --- a/build.gradle +++ b/build.gradle @@ -32,7 +32,7 @@ buildscript { jjwt_version = '0.12.5' guava_version = '32.1.3-jre' jaxb_version = '2.3.9' - spring_version = '5.3.33' + spring_version = '5.3.34' if (buildVersionQualifier) { opensearch_build += "-${buildVersionQualifier}" From a6b04ae751c5233aeaf50ff63a918fccb58caa31 Mon Sep 17 00:00:00 2001 From: Andrey Pleskach Date: Mon, 15 Apr 2024 18:02:12 +0200 Subject: [PATCH 184/204] Move REST API tests into integration tests (Part 1) (#4153) Signed-off-by: Andrey Pleskach --- .../security/ConfigurationFiles.java | 44 ++- .../security/SecurityConfigurationTests.java | 5 +- .../api/AbstractApiIntegrationTest.java | 353 ++++++++++++++++++ .../api/AccountRestApiIntegrationTest.java | 174 +++++++++ .../security/api/DashboardsInfoTest.java | 54 +-- .../api/DashboardsInfoWithSettingsTest.java | 53 +-- ...DefaultApiAvailabilityIntegrationTest.java | 134 +++++++ .../test/framework/TestSecurityConfig.java | 25 ++ .../framework/cluster/TestRestClient.java | 8 + .../security/securityconf/impl/CType.java | 4 + .../dlic/rest/api/AccountApiTest.java | 227 ----------- .../rest/api/DashboardsInfoActionTest.java | 49 --- .../dlic/rest/api/FlushCacheApiTest.java | 70 ---- .../api/legacy/LegacyAccountApiTests.java | 23 -- .../LegacyDashboardsInfoActionTests.java | 23 -- .../api/legacy/LegacyFlushCacheApiTests.java | 23 -- 16 files changed, 763 insertions(+), 506 deletions(-) create mode 100644 src/integrationTest/java/org/opensearch/security/api/AbstractApiIntegrationTest.java create mode 100644 src/integrationTest/java/org/opensearch/security/api/AccountRestApiIntegrationTest.java create mode 100644 src/integrationTest/java/org/opensearch/security/api/DefaultApiAvailabilityIntegrationTest.java delete mode 100644 src/test/java/org/opensearch/security/dlic/rest/api/AccountApiTest.java delete mode 100644 src/test/java/org/opensearch/security/dlic/rest/api/DashboardsInfoActionTest.java delete mode 100644 src/test/java/org/opensearch/security/dlic/rest/api/FlushCacheApiTest.java delete mode 100644 src/test/java/org/opensearch/security/dlic/rest/api/legacy/LegacyAccountApiTests.java delete mode 100644 src/test/java/org/opensearch/security/dlic/rest/api/legacy/LegacyDashboardsInfoActionTests.java delete mode 100644 src/test/java/org/opensearch/security/dlic/rest/api/legacy/LegacyFlushCacheApiTests.java diff --git a/src/integrationTest/java/org/opensearch/security/ConfigurationFiles.java b/src/integrationTest/java/org/opensearch/security/ConfigurationFiles.java index f3b7613aa1..04f2892eeb 100644 --- a/src/integrationTest/java/org/opensearch/security/ConfigurationFiles.java +++ b/src/integrationTest/java/org/opensearch/security/ConfigurationFiles.java @@ -9,38 +9,34 @@ */ package org.opensearch.security; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.Objects; -class ConfigurationFiles { +import org.opensearch.core.common.Strings; +import org.opensearch.security.securityconf.impl.CType; - public static void createRoleMappingFile(File destination) { - String resource = "roles_mapping.yml"; - copyResourceToFile(resource, destination); - } +public class ConfigurationFiles { public static Path createConfigurationDirectory() { try { Path tempDirectory = Files.createTempDirectory("test-security-config"); String[] configurationFiles = { - "config.yml", - "action_groups.yml", - "internal_users.yml", - "nodes_dn.yml", - "roles.yml", - "roles_mapping.yml", + CType.ACTIONGROUPS.configFileName(), + CType.CONFIG.configFileName(), + CType.INTERNALUSERS.configFileName(), + CType.NODESDN.configFileName(), + CType.ROLES.configFileName(), + CType.ROLESMAPPING.configFileName(), "security_tenants.yml", - "tenants.yml", - "whitelist.yml" }; + CType.TENANTS.configFileName(), + CType.WHITELIST.configFileName() }; for (String fileName : configurationFiles) { - Path configFileDestination = tempDirectory.resolve(fileName); - copyResourceToFile(fileName, configFileDestination.toFile()); + copyResourceToFile(fileName, tempDirectory.resolve(fileName)); } return tempDirectory.toAbsolutePath(); } catch (IOException ex) { @@ -48,10 +44,18 @@ public static Path createConfigurationDirectory() { } } - private static void copyResourceToFile(String resource, File destination) { + public static void writeToConfig(final CType cType, final Path configFolder, final String content) throws IOException { + if (Strings.isNullOrEmpty(content)) return; + try (final var out = Files.newOutputStream(cType.configFile(configFolder), StandardOpenOption.APPEND)) { + out.write(content.getBytes(StandardCharsets.UTF_8)); + out.flush(); + } + } + + public static void copyResourceToFile(String resource, Path destination) { try (InputStream input = ConfigurationFiles.class.getClassLoader().getResourceAsStream(resource)) { Objects.requireNonNull(input, "Cannot find source resource " + resource); - try (OutputStream output = new FileOutputStream(destination)) { + try (final var output = Files.newOutputStream(destination)) { input.transferTo(output); } } catch (IOException e) { diff --git a/src/integrationTest/java/org/opensearch/security/SecurityConfigurationTests.java b/src/integrationTest/java/org/opensearch/security/SecurityConfigurationTests.java index 3889fa2a3c..73c55bc667 100644 --- a/src/integrationTest/java/org/opensearch/security/SecurityConfigurationTests.java +++ b/src/integrationTest/java/org/opensearch/security/SecurityConfigurationTests.java @@ -29,6 +29,7 @@ import org.junit.runner.RunWith; import org.opensearch.client.Client; +import org.opensearch.security.securityconf.impl.CType; import org.opensearch.test.framework.AsyncActions; import org.opensearch.test.framework.TestSecurityConfig.Role; import org.opensearch.test.framework.TestSecurityConfig.User; @@ -225,8 +226,8 @@ public void shouldAccessIndexWithPlaceholder_negative() { @Test public void shouldUseSecurityAdminTool() throws Exception { SecurityAdminLauncher securityAdminLauncher = new SecurityAdminLauncher(cluster.getHttpPort(), cluster.getTestCertificates()); - File rolesMapping = configurationDirectory.newFile("roles_mapping.yml"); - ConfigurationFiles.createRoleMappingFile(rolesMapping); + File rolesMapping = configurationDirectory.newFile(CType.ROLESMAPPING.configFileName()); + ConfigurationFiles.copyResourceToFile(CType.ROLESMAPPING.configFileName(), rolesMapping.toPath()); int exitCode = securityAdminLauncher.updateRoleMappings(rolesMapping); diff --git a/src/integrationTest/java/org/opensearch/security/api/AbstractApiIntegrationTest.java b/src/integrationTest/java/org/opensearch/security/api/AbstractApiIntegrationTest.java new file mode 100644 index 0000000000..678b1df161 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/api/AbstractApiIntegrationTest.java @@ -0,0 +1,353 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.api; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.StringJoiner; + +import com.carrotsearch.randomizedtesting.RandomizedTest; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import com.google.common.collect.ImmutableMap; +import org.apache.commons.io.FileUtils; +import org.apache.http.HttpStatus; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.awaitility.Awaitility; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.runner.RunWith; + +import org.opensearch.common.CheckedConsumer; +import org.opensearch.common.CheckedSupplier; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.security.ConfigurationFiles; +import org.opensearch.security.dlic.rest.api.Endpoint; +import org.opensearch.security.securityconf.impl.CType; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.certificate.CertificateData; +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.opensearch.security.CrossClusterSearchTests.PLUGINS_SECURITY_RESTAPI_ROLES_ENABLED; +import static org.opensearch.security.OpenSearchSecurityPlugin.LEGACY_OPENDISTRO_PREFIX; +import static org.opensearch.security.OpenSearchSecurityPlugin.PLUGINS_PREFIX; +import static org.opensearch.security.dlic.rest.api.RestApiAdminPrivilegesEvaluator.CERTS_INFO_ACTION; +import static org.opensearch.security.dlic.rest.api.RestApiAdminPrivilegesEvaluator.ENDPOINTS_WITH_PERMISSIONS; +import static org.opensearch.security.dlic.rest.api.RestApiAdminPrivilegesEvaluator.RELOAD_CERTS_ACTION; +import static org.opensearch.security.dlic.rest.api.RestApiAdminPrivilegesEvaluator.SECURITY_CONFIG_UPDATE; +import static org.opensearch.security.support.ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX; +import static org.opensearch.security.support.ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_USE_CLUSTER_STATE; +import static org.opensearch.test.framework.TestSecurityConfig.REST_ADMIN_REST_API_ACCESS; + +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +public abstract class AbstractApiIntegrationTest extends RandomizedTest { + + private static final Logger LOGGER = LogManager.getLogger(TestSecurityConfig.class); + + public static final String NEW_USER = "new-user"; + + public static final String REST_ADMIN_USER = "rest-api-admin"; + + public static final String ADMIN_USER_NAME = "admin"; + + public static final String DEFAULT_PASSWORD = "secret"; + + public static final ToXContentObject EMPTY_BODY = (builder, params) -> builder.startObject().endObject(); + + public static Path configurationFolder; + + public static ImmutableMap.Builder clusterSettings = ImmutableMap.builder(); + + protected static TestSecurityConfig testSecurityConfig = new TestSecurityConfig(); + + public static LocalCluster localCluster; + + @BeforeClass + public static void startCluster() throws IOException { + configurationFolder = ConfigurationFiles.createConfigurationDirectory(); + extendConfiguration(); + clusterSettings.put(SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, true) + .put(PLUGINS_SECURITY_RESTAPI_ROLES_ENABLED, List.of("user_admin__all_access", REST_ADMIN_REST_API_ACCESS)) + .put(SECURITY_ALLOW_DEFAULT_INIT_USE_CLUSTER_STATE, randomBoolean()); + final var clusterManager = randomFrom(List.of(ClusterManager.THREE_CLUSTER_MANAGERS, ClusterManager.SINGLENODE)); + final var localClusterBuilder = new LocalCluster.Builder().clusterManager(clusterManager) + .nodeSettings(clusterSettings.buildKeepingLast()) + .defaultConfigurationInitDirectory(configurationFolder.toString()) + .loadConfigurationIntoIndex(false); + localCluster = localClusterBuilder.build(); + localCluster.before(); + try (TestRestClient client = localCluster.getRestClient(ADMIN_USER_NAME, DEFAULT_PASSWORD)) { + Awaitility.await() + .alias("Load default configuration") + .until(() -> client.securityHealth().getTextFromJsonBody("/status"), equalTo("UP")); + } + } + + private static void extendConfiguration() throws IOException { + extendActionGroups(configurationFolder, testSecurityConfig.actionGroups()); + extendRoles(configurationFolder, testSecurityConfig.roles()); + extendRolesMapping(configurationFolder, testSecurityConfig.rolesMapping()); + extendUsers(configurationFolder, testSecurityConfig.getUsers()); + } + + private static void extendUsers(final Path configFolder, final List users) throws IOException { + if (users == null) return; + if (users.isEmpty()) return; + LOGGER.info("Adding users to the default configuration: "); + try (final var contentBuilder = XContentFactory.yamlBuilder()) { + contentBuilder.startObject(); + for (final var u : users) { + LOGGER.info("\t\t - {}", u.getName()); + contentBuilder.field(u.getName()); + u.toXContent(contentBuilder, ToXContent.EMPTY_PARAMS); + } + contentBuilder.endObject(); + ConfigurationFiles.writeToConfig(CType.INTERNALUSERS, configFolder, removeDashes(contentBuilder.toString())); + } + } + + private static void extendActionGroups(final Path configFolder, final List actionGroups) + throws IOException { + if (actionGroups == null) return; + if (actionGroups.isEmpty()) return; + LOGGER.info("Adding action groups to the default configuration: "); + try (final var contentBuilder = XContentFactory.yamlBuilder()) { + contentBuilder.startObject(); + for (final var ag : actionGroups) { + LOGGER.info("\t\t - {}", ag.name()); + contentBuilder.field(ag.name()); + ag.toXContent(contentBuilder, ToXContent.EMPTY_PARAMS); + } + contentBuilder.endObject(); + ConfigurationFiles.writeToConfig(CType.ACTIONGROUPS, configFolder, removeDashes(contentBuilder.toString())); + } + } + + private static void extendRoles(final Path configFolder, final List roles) throws IOException { + if (roles == null) return; + if (roles.isEmpty()) return; + LOGGER.info("Adding roles to the default configuration: "); + try (final var contentBuilder = XContentFactory.yamlBuilder()) { + contentBuilder.startObject(); + for (final var r : roles) { + LOGGER.info("\t\t - {}", r.getName()); + contentBuilder.field(r.getName()); + r.toXContent(contentBuilder, ToXContent.EMPTY_PARAMS); + } + contentBuilder.endObject(); + ConfigurationFiles.writeToConfig(CType.ROLES, configFolder, removeDashes(contentBuilder.toString())); + } + } + + private static void extendRolesMapping(final Path configFolder, final List rolesMapping) + throws IOException { + if (rolesMapping == null) return; + if (rolesMapping.isEmpty()) return; + LOGGER.info("Adding roles mapping to the default configuration: "); + try (final var contentBuilder = XContentFactory.yamlBuilder()) { + contentBuilder.startObject(); + for (final var rm : rolesMapping) { + LOGGER.info("\t\t - {}", rm.name()); + contentBuilder.field(rm.name()); + rm.toXContent(contentBuilder, ToXContent.EMPTY_PARAMS); + } + contentBuilder.endObject(); + ConfigurationFiles.writeToConfig(CType.ROLESMAPPING, configFolder, removeDashes(contentBuilder.toString())); + } + } + + private static String removeDashes(final String content) { + return content.replace("---", ""); + } + + protected static String[] allRestAdminPermissions() { + final var permissions = new String[ENDPOINTS_WITH_PERMISSIONS.size() + 3]; // 2 actions for SSL + update config action + var counter = 0; + for (final var e : ENDPOINTS_WITH_PERMISSIONS.entrySet()) { + if (e.getKey() == Endpoint.SSL) { + permissions[counter] = e.getValue().build(CERTS_INFO_ACTION); + permissions[++counter] = e.getValue().build(RELOAD_CERTS_ACTION); + } else if (e.getKey() == Endpoint.CONFIG) { + permissions[counter++] = e.getValue().build(SECURITY_CONFIG_UPDATE); + } else { + permissions[counter++] = e.getValue().build(); + } + } + return permissions; + } + + protected static String restAdminPermission(Endpoint endpoint) { + return restAdminPermission(endpoint, null); + } + + protected static String restAdminPermission(Endpoint endpoint, String action) { + if (action != null) { + return ENDPOINTS_WITH_PERMISSIONS.get(endpoint).build(action); + } else { + return ENDPOINTS_WITH_PERMISSIONS.get(endpoint).build(); + } + } + + @AfterClass + public static void stopCluster() throws IOException { + if (localCluster != null) localCluster.close(); + FileUtils.deleteDirectory(configurationFolder.toFile()); + } + + protected void withUser(final String user, final CheckedConsumer restClientHandler) throws Exception { + withUser(user, DEFAULT_PASSWORD, restClientHandler); + } + + protected void withUser(final String user, final String password, final CheckedConsumer restClientHandler) + throws Exception { + try (TestRestClient client = localCluster.getRestClient(user, password)) { + restClientHandler.accept(client); + } + } + + protected void withUser( + final String user, + final CertificateData certificateData, + final CheckedConsumer restClientHandler + ) throws Exception { + withUser(user, DEFAULT_PASSWORD, certificateData, restClientHandler); + } + + protected void withUser( + final String user, + final String password, + final CertificateData certificateData, + final CheckedConsumer restClientHandler + ) throws Exception { + try (TestRestClient client = localCluster.getRestClient(user, password, certificateData)) { + restClientHandler.accept(client); + } + } + + protected String securityPath(String... path) { + final var fullPath = new StringJoiner("/"); + fullPath.add(randomFrom(List.of(LEGACY_OPENDISTRO_PREFIX, PLUGINS_PREFIX))); + if (path != null) { + for (final var p : path) + fullPath.add(p); + } + return fullPath.toString(); + } + + protected String api() { + return String.format("%s/api", securityPath()); + } + + protected String apiPath(final String... path) { + + final var fullPath = new StringJoiner("/"); + fullPath.add(api()); + + for (final var p : path) { + fullPath.add(p); + } + return fullPath.toString(); + } + + TestRestClient.HttpResponse badRequest(final CheckedSupplier endpointCallback) + throws Exception { + final var response = endpointCallback.get(); + assertThat(response.getBody(), response.getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST)); + assertResponseBody(response.getBody()); + return response; + } + + TestRestClient.HttpResponse created(final CheckedSupplier endpointCallback) throws Exception { + final var response = endpointCallback.get(); + assertThat(response.getBody(), response.getStatusCode(), equalTo(HttpStatus.SC_CREATED)); + assertResponseBody(response.getBody()); + return response; + } + + TestRestClient.HttpResponse forbidden(final CheckedSupplier endpointCallback) throws Exception { + final var response = endpointCallback.get(); + assertThat(response.getBody(), response.getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN)); + assertResponseBody(response.getBody()); + return response; + } + + TestRestClient.HttpResponse methodNotAllowed(final CheckedSupplier endpointCallback) + throws Exception { + final var response = endpointCallback.get(); + assertThat(response.getBody(), response.getStatusCode(), equalTo(HttpStatus.SC_METHOD_NOT_ALLOWED)); + assertResponseBody(response.getBody()); + return response; + } + + TestRestClient.HttpResponse notImplemented(final CheckedSupplier endpointCallback) + throws Exception { + final var response = endpointCallback.get(); + assertThat(response.getBody(), response.getStatusCode(), is(HttpStatus.SC_NOT_IMPLEMENTED)); + assertResponseBody(response.getBody()); + return response; + } + + TestRestClient.HttpResponse notFound(final CheckedSupplier endpointCallback) throws Exception { + final var response = endpointCallback.get(); + assertThat(response.getBody(), response.getStatusCode(), equalTo(HttpStatus.SC_NOT_FOUND)); + assertResponseBody(response.getBody()); + return response; + } + + TestRestClient.HttpResponse ok(final CheckedSupplier endpointCallback) throws Exception { + final var response = endpointCallback.get(); + assertThat(response.getBody(), response.getStatusCode(), equalTo(HttpStatus.SC_OK)); + assertResponseBody(response.getBody()); + return response; + } + + TestRestClient.HttpResponse unauthorized(final CheckedSupplier endpointCallback) + throws Exception { + final var response = endpointCallback.get(); + assertThat(response.getBody(), response.getStatusCode(), equalTo(HttpStatus.SC_UNAUTHORIZED)); + // TODO assert response body here + return response; + } + + void assertResponseBody(final String responseBody) { + assertThat(responseBody, notNullValue()); + assertThat(responseBody, not(equalTo(""))); + } + + void assertInvalidKeys(final TestRestClient.HttpResponse response, final String expectedInvalidKeys) { + assertThat(response.getBody(), response.getTextFromJsonBody("/reason"), equalTo("Invalid configuration")); + assertThat(response.getBody(), response.getTextFromJsonBody("/invalid_keys/keys"), equalTo(expectedInvalidKeys)); + } + + void assertSpecifyOneOf(final TestRestClient.HttpResponse response, final String expectedSpecifyOneOfKeys) { + assertThat(response.getBody(), response.getTextFromJsonBody("/reason"), equalTo("Invalid configuration")); + assertThat(response.getBody(), response.getTextFromJsonBody("/specify_one_of/keys"), containsString(expectedSpecifyOneOfKeys)); + } + + void assertNullValuesInArray(CheckedSupplier endpointCallback) throws Exception { + final var response = endpointCallback.get(); + assertThat(response.getBody(), response.getTextFromJsonBody("/reason"), equalTo("`null` is not allowed as json array element")); + } + +} diff --git a/src/integrationTest/java/org/opensearch/security/api/AccountRestApiIntegrationTest.java b/src/integrationTest/java/org/opensearch/security/api/AccountRestApiIntegrationTest.java new file mode 100644 index 0000000000..7fa298c1e4 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/api/AccountRestApiIntegrationTest.java @@ -0,0 +1,174 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.security.api; + +import org.junit.Test; + +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.cluster.TestRestClient; + +import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.not; +import static org.opensearch.security.dlic.rest.support.Utils.hash; + +public class AccountRestApiIntegrationTest extends AbstractApiIntegrationTest { + + private final static String TEST_USER = "test-user"; + + private final static String RESERVED_USER = "reserved-user"; + + private final static String HIDDEN_USERS = "hidden-user"; + + public final static String TEST_USER_PASSWORD = randomAlphabetic(10); + + public final static String TEST_USER_NEW_PASSWORD = randomAlphabetic(10); + + static { + testSecurityConfig.user(new TestSecurityConfig.User(TEST_USER).password(TEST_USER_PASSWORD)) + .user(new TestSecurityConfig.User(RESERVED_USER).reserved(true)) + .user(new TestSecurityConfig.User(HIDDEN_USERS).hidden(true)); + } + + private String accountPath() { + return super.apiPath("account"); + } + + @Test + public void accountInfo() throws Exception { + withUser(NEW_USER, client -> { + var response = ok(() -> client.get(accountPath())); + final var account = response.bodyAsJsonNode(); + assertThat(response.getBody(), account.get("user_name").asText(), is(NEW_USER)); + assertThat(response.getBody(), not(account.get("is_reserved").asBoolean())); + assertThat(response.getBody(), not(account.get("is_hidden").asBoolean())); + assertThat(response.getBody(), account.get("is_internal_user").asBoolean()); + assertThat(response.getBody(), account.get("user_requested_tenant").isNull()); + assertThat(response.getBody(), account.get("backend_roles").isArray()); + assertThat(response.getBody(), account.get("custom_attribute_names").isArray()); + assertThat(response.getBody(), account.get("tenants").isObject()); + assertThat(response.getBody(), account.get("roles").isArray()); + }); + withUser(NEW_USER, "a", client -> unauthorized(() -> client.get(accountPath()))); + withUser("a", "b", client -> unauthorized(() -> client.get(accountPath()))); + } + + @Test + public void changeAccountPassword() throws Exception { + withUser(TEST_USER, TEST_USER_PASSWORD, this::verifyWrongPayload); + verifyPasswordCanBeChanged(); + + withUser(RESERVED_USER, client -> { + var response = ok(() -> client.get(accountPath())); + assertThat(response.getBody(), response.getBooleanFromJsonBody("/is_reserved")); + forbidden(() -> client.putJson(accountPath(), changePasswordPayload(DEFAULT_PASSWORD, randomAlphabetic(10)))); + }); + withUser(HIDDEN_USERS, client -> { + var response = ok(() -> client.get(accountPath())); + assertThat(response.getBody(), response.getBooleanFromJsonBody("/is_hidden")); + notFound(() -> client.putJson(accountPath(), changePasswordPayload(DEFAULT_PASSWORD, randomAlphabetic(10)))); + }); + withUser(ADMIN_USER_NAME, localCluster.getAdminCertificate(), client -> { + ok(() -> client.get(accountPath())); + notFound(() -> client.putJson(accountPath(), changePasswordPayload(DEFAULT_PASSWORD, randomAlphabetic(10)))); + }); + } + + private void verifyWrongPayload(final TestRestClient client) throws Exception { + badRequest(() -> client.putJson(accountPath(), EMPTY_BODY)); + badRequest(() -> client.putJson(accountPath(), changePasswordPayload(null, "new_password"))); + badRequest(() -> client.putJson(accountPath(), changePasswordPayload("wrong-password", "some_new_pwd"))); + badRequest(() -> client.putJson(accountPath(), changePasswordPayload(TEST_USER_PASSWORD, null))); + badRequest(() -> client.putJson(accountPath(), changePasswordPayload(TEST_USER_PASSWORD, ""))); + badRequest(() -> client.putJson(accountPath(), changePasswordWithHashPayload(TEST_USER_PASSWORD, null))); + badRequest(() -> client.putJson(accountPath(), changePasswordWithHashPayload(TEST_USER_PASSWORD, ""))); + badRequest( + () -> client.putJson( + accountPath(), + (builder, params) -> builder.startObject() + .field("current_password", TEST_USER_PASSWORD) + .startArray("backend_roles") + .endArray() + .endObject() + ) + ); + } + + private void verifyPasswordCanBeChanged() throws Exception { + final var newPassword = randomAlphabetic(10); + withUser( + TEST_USER, + TEST_USER_PASSWORD, + client -> ok( + () -> client.putJson(accountPath(), changePasswordWithHashPayload(TEST_USER_PASSWORD, hash(newPassword.toCharArray()))) + ) + ); + withUser( + TEST_USER, + newPassword, + client -> ok(() -> client.putJson(accountPath(), changePasswordPayload(newPassword, TEST_USER_NEW_PASSWORD))) + ); + } + + @Test + public void testPutAccountRetainsAccountInformation() throws Exception { + final var username = "test"; + final String password = randomAlphabetic(10); + final String newPassword = randomAlphabetic(10); + withUser( + ADMIN_USER_NAME, + client -> created( + () -> client.putJson( + apiPath("internalusers", username), + (builder, params) -> builder.startObject() + .field("password", password) + .field("backend_roles") + .startArray() + .value("test-backend-role") + .endArray() + .field("opendistro_security_roles") + .startArray() + .value("user_limited-user__limited-role") + .endArray() + .field("attributes") + .startObject() + .field("foo", "bar") + .endObject() + .endObject() + ) + ) + ); + withUser(username, password, client -> ok(() -> client.putJson(accountPath(), changePasswordPayload(password, newPassword)))); + withUser(ADMIN_USER_NAME, client -> { + final var response = ok(() -> client.get(apiPath("internalusers", username))); + final var user = response.bodyAsJsonNode().get(username); + assertThat(user.toPrettyString(), user.get("backend_roles").get(0).asText(), is("test-backend-role")); + assertThat(user.toPrettyString(), user.get("opendistro_security_roles").get(0).asText(), is("user_limited-user__limited-role")); + assertThat(user.toPrettyString(), user.get("attributes").get("foo").asText(), is("bar")); + }); + } + + private ToXContentObject changePasswordPayload(final String currentPassword, final String newPassword) { + return (builder, params) -> { + builder.startObject(); + if (currentPassword != null) builder.field("current_password", currentPassword); + if (newPassword != null) builder.field("password", newPassword); + return builder.endObject(); + }; + } + + private ToXContentObject changePasswordWithHashPayload(final String currentPassword, final String hash) { + return (builder, params) -> builder.startObject().field("current_password", currentPassword).field("hash", hash).endObject(); + } + +} diff --git a/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoTest.java b/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoTest.java index 15634e8890..635d9ecff4 100644 --- a/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoTest.java +++ b/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoTest.java @@ -11,58 +11,40 @@ package org.opensearch.security.api; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import org.apache.http.HttpStatus; -import org.junit.ClassRule; +import java.util.List; + import org.junit.Test; -import org.junit.runner.RunWith; -import org.opensearch.security.securityconf.impl.DashboardSignInOption; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; -import org.opensearch.test.framework.cluster.ClusterManager; -import org.opensearch.test.framework.cluster.LocalCluster; -import org.opensearch.test.framework.cluster.TestRestClient; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; +import static org.opensearch.security.OpenSearchSecurityPlugin.LEGACY_OPENDISTRO_PREFIX; +import static org.opensearch.security.OpenSearchSecurityPlugin.PLUGINS_PREFIX; import static org.opensearch.security.rest.DashboardsInfoAction.DEFAULT_PASSWORD_MESSAGE; import static org.opensearch.security.rest.DashboardsInfoAction.DEFAULT_PASSWORD_REGEX; -import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; -@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) -@ThreadLeakScope(ThreadLeakScope.Scope.NONE) -public class DashboardsInfoTest { +public class DashboardsInfoTest extends AbstractApiIntegrationTest { - protected final static TestSecurityConfig.User DASHBOARDS_USER = new TestSecurityConfig.User("dashboards_user").roles( - new Role("dashboards_role").indexPermissions("read").on("*").clusterPermissions("cluster_composite_ops") - ); + static { + testSecurityConfig.user( + new TestSecurityConfig.User("dashboards_user").roles( + new Role("dashboards_role").indexPermissions("read").on("*").clusterPermissions("cluster_composite_ops") + ) + ); + } - @ClassRule - public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS) - .authc(AUTHC_HTTPBASIC_INTERNAL) - .users(DASHBOARDS_USER) - .build(); + private String apiPath() { + return randomFrom(List.of(PLUGINS_PREFIX + "/dashboardsinfo", LEGACY_OPENDISTRO_PREFIX + "/kibanainfo")); + } @Test public void testDashboardsInfoValidationMessage() throws Exception { - - try (TestRestClient client = cluster.getRestClient(DASHBOARDS_USER)) { - TestRestClient.HttpResponse response = client.get("_plugins/_security/dashboardsinfo"); - assertThat(response.getStatusCode(), equalTo(HttpStatus.SC_OK)); + withUser("dashboards_user", client -> { + final var response = ok(() -> client.get(apiPath())); assertThat(response.getTextFromJsonBody("/password_validation_error_message"), equalTo(DEFAULT_PASSWORD_MESSAGE)); assertThat(response.getTextFromJsonBody("/password_validation_regex"), equalTo(DEFAULT_PASSWORD_REGEX)); - } - } - - @Test - public void testDashboardsInfoContainsSignInOptions() throws Exception { - - try (TestRestClient client = cluster.getRestClient(DASHBOARDS_USER)) { - TestRestClient.HttpResponse response = client.get("_plugins/_security/dashboardsinfo"); - assertThat(response.getStatusCode(), equalTo(HttpStatus.SC_OK)); - assertThat(response.getTextArrayFromJsonBody("/sign_in_options").contains(DashboardSignInOption.BASIC.toString()), is(true)); - } + }); } } diff --git a/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoWithSettingsTest.java b/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoWithSettingsTest.java index 6e4444d049..96aed9ddd8 100644 --- a/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoWithSettingsTest.java +++ b/src/integrationTest/java/org/opensearch/security/api/DashboardsInfoWithSettingsTest.java @@ -11,60 +11,47 @@ package org.opensearch.security.api; -import java.util.Map; +import java.util.List; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import org.apache.http.HttpStatus; -import org.junit.ClassRule; import org.junit.Test; -import org.junit.runner.RunWith; import org.opensearch.security.support.ConfigConstants; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; -import org.opensearch.test.framework.cluster.ClusterManager; -import org.opensearch.test.framework.cluster.LocalCluster; -import org.opensearch.test.framework.cluster.TestRestClient; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.security.OpenSearchSecurityPlugin.LEGACY_OPENDISTRO_PREFIX; +import static org.opensearch.security.OpenSearchSecurityPlugin.PLUGINS_PREFIX; -@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) -@ThreadLeakScope(ThreadLeakScope.Scope.NONE) -public class DashboardsInfoWithSettingsTest { +public class DashboardsInfoWithSettingsTest extends AbstractApiIntegrationTest { - protected final static TestSecurityConfig.User DASHBOARDS_USER = new TestSecurityConfig.User("dashboards_user").roles( - new Role("dashboards_role").indexPermissions("read").on("*").clusterPermissions("cluster_composite_ops") - ); + private static final String CUSTOM_PASSWORD_REGEX = "(?=.*[A-Z])(?=.*[^a-zA-Z\\d])(?=.*[0-9])(?=.*[a-z]).{5,}"; private static final String CUSTOM_PASSWORD_MESSAGE = "Password must be minimum 5 characters long and must contain at least one uppercase letter, one lowercase letter, one digit, and one special character."; - private static final String CUSTOM_PASSWORD_REGEX = "(?=.*[A-Z])(?=.*[^a-zA-Z\\d])(?=.*[0-9])(?=.*[a-z]).{5,}"; - - @ClassRule - public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS) - .authc(AUTHC_HTTPBASIC_INTERNAL) - .users(DASHBOARDS_USER) - .nodeSettings( - Map.of( - ConfigConstants.SECURITY_RESTAPI_PASSWORD_VALIDATION_REGEX, - CUSTOM_PASSWORD_REGEX, - ConfigConstants.SECURITY_RESTAPI_PASSWORD_VALIDATION_ERROR_MESSAGE, - CUSTOM_PASSWORD_MESSAGE + static { + clusterSettings.put(ConfigConstants.SECURITY_RESTAPI_PASSWORD_VALIDATION_REGEX, CUSTOM_PASSWORD_REGEX) + .put(ConfigConstants.SECURITY_RESTAPI_PASSWORD_VALIDATION_ERROR_MESSAGE, CUSTOM_PASSWORD_MESSAGE); + testSecurityConfig.user( + new TestSecurityConfig.User("dashboards_user").roles( + new Role("dashboards_role").indexPermissions("read").on("*").clusterPermissions("cluster_composite_ops") ) - ) - .build(); + ); + } + + private String apiPath() { + return randomFrom(List.of(PLUGINS_PREFIX + "/dashboardsinfo", LEGACY_OPENDISTRO_PREFIX + "/kibanainfo")); + } @Test public void testDashboardsInfoValidationMessageWithCustomMessage() throws Exception { - try (TestRestClient client = cluster.getRestClient(DASHBOARDS_USER)) { - TestRestClient.HttpResponse response = client.get("_plugins/_security/dashboardsinfo"); - assertThat(response.getStatusCode(), equalTo(HttpStatus.SC_OK)); + withUser("dashboards_user", client -> { + final var response = ok(() -> client.get(apiPath())); assertThat(response.getTextFromJsonBody("/password_validation_error_message"), equalTo(CUSTOM_PASSWORD_MESSAGE)); assertThat(response.getTextFromJsonBody("/password_validation_regex"), equalTo(CUSTOM_PASSWORD_REGEX)); - } + }); } } diff --git a/src/integrationTest/java/org/opensearch/security/api/DefaultApiAvailabilityIntegrationTest.java b/src/integrationTest/java/org/opensearch/security/api/DefaultApiAvailabilityIntegrationTest.java new file mode 100644 index 0000000000..eaecb275db --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/api/DefaultApiAvailabilityIntegrationTest.java @@ -0,0 +1,134 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.api; + +import org.junit.Test; + +import org.opensearch.test.framework.cluster.TestRestClient; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; + +public class DefaultApiAvailabilityIntegrationTest extends AbstractApiIntegrationTest { + + @Test + public void nodesDnApiIsNotAvailableByDefault() throws Exception { + withUser(NEW_USER, this::verifyNodesDnApi); + withUser(ADMIN_USER_NAME, localCluster.getAdminCertificate(), this::verifyNodesDnApi); + } + + private void verifyNodesDnApi(final TestRestClient client) throws Exception { + badRequest(() -> client.get(apiPath("nodesdn"))); + badRequest(() -> client.putJson(apiPath("nodesdn", "cluster_1"), EMPTY_BODY)); + badRequest(() -> client.delete(apiPath("nodesdn", "cluster_1"))); + badRequest(() -> client.patch(apiPath("nodesdn", "cluster_1"), EMPTY_BODY)); + } + + @Test + public void securityConfigIsNotAvailableByDefault() throws Exception { + withUser(NEW_USER, client -> { + forbidden(() -> client.get(apiPath("securityconfig"))); + verifySecurityConfigApi(client); + }); + withUser(ADMIN_USER_NAME, localCluster.getAdminCertificate(), client -> { + ok(() -> client.get(apiPath("securityconfig"))); + verifySecurityConfigApi(client); + }); + } + + private void verifySecurityConfigApi(final TestRestClient client) throws Exception { + methodNotAllowed(() -> client.putJson(apiPath("securityconfig"), EMPTY_BODY)); + methodNotAllowed(() -> client.postJson(apiPath("securityconfig"), EMPTY_BODY)); + methodNotAllowed(() -> client.delete(apiPath("securityconfig"))); + forbidden( + () -> client.patch( + apiPath("securityconfig"), + (builder, params) -> builder.startArray() + .startObject() + .field("op", "replace") + .field("path", "/a/b/c") + .field("value", "other") + .endObject() + .endArray() + ) + ); + } + + @Test + public void securityHealth() throws Exception { + withUser(NEW_USER, client -> ok(() -> client.get(securityPath("health")))); + withUser(ADMIN_USER_NAME, localCluster.getAdminCertificate(), client -> ok(() -> client.get(securityPath("health")))); + } + + @Test + public void securityAuthInfo() throws Exception { + withUser(NEW_USER, this::verifyAuthInfoApi); + withUser(ADMIN_USER_NAME, localCluster.getAdminCertificate(), this::verifyAuthInfoApi); + } + + private void verifyAuthInfoApi(final TestRestClient client) throws Exception { + final var verbose = randomBoolean(); + + final TestRestClient.HttpResponse response; + if (verbose) response = ok(() -> client.get(securityPath("authinfo?verbose=" + verbose))); + else response = ok(() -> client.get(securityPath("authinfo"))); + final var body = response.bodyAsJsonNode(); + assertThat(response.getBody(), body.has("user")); + assertThat(response.getBody(), body.has("user_name")); + assertThat(response.getBody(), body.has("user_requested_tenant")); + assertThat(response.getBody(), body.has("remote_address")); + assertThat(response.getBody(), body.has("backend_roles")); + assertThat(response.getBody(), body.has("custom_attribute_names")); + assertThat(response.getBody(), body.has("roles")); + assertThat(response.getBody(), body.has("tenants")); + assertThat(response.getBody(), body.has("principal")); + assertThat(response.getBody(), body.has("peer_certificates")); + assertThat(response.getBody(), body.has("sso_logout_url")); + + if (verbose) { + assertThat(response.getBody(), body.has("size_of_user")); + assertThat(response.getBody(), body.has("size_of_custom_attributes")); + assertThat(response.getBody(), body.has("size_of_backendroles")); + } + + } + + @Test + public void flushCache() throws Exception { + withUser(NEW_USER, client -> { + forbidden(() -> client.get(apiPath("cache"))); + forbidden(() -> client.postJson(apiPath("cache"), EMPTY_BODY)); + forbidden(() -> client.putJson(apiPath("cache"), EMPTY_BODY)); + forbidden(() -> client.delete(apiPath("cache"))); + }); + withUser(ADMIN_USER_NAME, localCluster.getAdminCertificate(), client -> { + notImplemented(() -> client.get(apiPath("cache"))); + notImplemented(() -> client.postJson(apiPath("cache"), EMPTY_BODY)); + notImplemented(() -> client.putJson(apiPath("cache"), EMPTY_BODY)); + final var response = ok(() -> client.delete(apiPath("cache"))); + assertThat(response.getBody(), response.getTextFromJsonBody("/message"), is("Cache flushed successfully.")); + }); + } + + @Test + public void reloadSSLCertsNotAvailable() throws Exception { + withUser(NEW_USER, client -> { + forbidden(() -> client.putJson(apiPath("ssl", "http", "reloadcerts"), EMPTY_BODY)); + forbidden(() -> client.putJson(apiPath("ssl", "transport", "reloadcerts"), EMPTY_BODY)); + }); + withUser(ADMIN_USER_NAME, localCluster.getAdminCertificate(), client -> { + badRequest(() -> client.putJson(apiPath("ssl", "http", "reloadcerts"), EMPTY_BODY)); + badRequest(() -> client.putJson(apiPath("ssl", "transport", "reloadcerts"), EMPTY_BODY)); + }); + } + +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java index 22aab70521..79f10a76cf 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java @@ -78,6 +78,8 @@ public class TestSecurityConfig { private static final Logger log = LogManager.getLogger(TestSecurityConfig.class); + public final static String REST_ADMIN_REST_API_ACCESS = "rest_admin__rest_api_access"; + private Config config = new Config(); private Map internalUsers = new LinkedHashMap<>(); private Map roles = new LinkedHashMap<>(); @@ -142,6 +144,17 @@ public TestSecurityConfig user(User user) { return this; } + public TestSecurityConfig withRestAdminUser(final String name, final String... permissions) { + if (internalUsers.containsKey(name)) throw new RuntimeException("REST Admin " + name + " already exists"); + user(new User(name, "REST Admin with permissions: " + Arrays.toString(permissions)).reserved(true)); + final var roleName = name + "__rest_admin_role"; + roles(new Role(roleName).clusterPermissions(permissions)); + + rolesMapping.computeIfAbsent(roleName, RoleMapping::new).users(name); + rolesMapping.computeIfAbsent(REST_ADMIN_REST_API_ACCESS, RoleMapping::new).users(name); + return this; + } + public List getUsers() { return new ArrayList<>(internalUsers.values()); } @@ -157,6 +170,10 @@ public TestSecurityConfig roles(Role... roles) { return this; } + public List roles() { + return List.copyOf(roles.values()); + } + public TestSecurityConfig audit(AuditConfiguration auditConfiguration) { this.auditConfiguration = auditConfiguration; return this; @@ -173,6 +190,10 @@ public TestSecurityConfig rolesMapping(RoleMapping... mappings) { return this; } + public List rolesMapping() { + return List.copyOf(rolesMapping.values()); + } + public TestSecurityConfig actionGroups(ActionGroup... groups) { for (final var group : groups) { this.actionGroups.put(group.name, group); @@ -180,6 +201,10 @@ public TestSecurityConfig actionGroups(ActionGroup... groups) { return this; } + public List actionGroups() { + return List.copyOf(actionGroups.values()); + } + public static class Config implements ToXContentObject { private boolean anonymousAuth; diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java index bd625a5c31..e7355711fc 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -114,6 +114,10 @@ public HttpResponse getAuthInfo(Header... headers) { return executeRequest(new HttpGet(getHttpServerUri() + "/_opendistro/_security/authinfo?pretty"), headers); } + public HttpResponse securityHealth(Header... headers) { + return executeRequest(new HttpGet(getHttpServerUri() + "/_plugins/_security/health"), headers); + } + public HttpResponse getAuthInfo(Map urlParams, Header... headers) { String urlParamsString = "?" + urlParams.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&")); @@ -183,6 +187,10 @@ public HttpResponse post(String path) { return executeRequest(uriRequest); } + public HttpResponse patch(String path, ToXContentObject body) { + return patch(path, Strings.toString(XContentType.JSON, body)); + } + public HttpResponse patch(String path, String body) { HttpPatch uriRequest = new HttpPatch(getHttpServerUri() + "/" + path); uriRequest.setEntity(new StringEntity(body)); diff --git a/src/main/java/org/opensearch/security/securityconf/impl/CType.java b/src/main/java/org/opensearch/security/securityconf/impl/CType.java index 23158e5850..8a51686225 100644 --- a/src/main/java/org/opensearch/security/securityconf/impl/CType.java +++ b/src/main/java/org/opensearch/security/securityconf/impl/CType.java @@ -112,6 +112,10 @@ public Path configFile(final Path configDir) { return configDir.resolve(this.configFileName); } + public String configFileName() { + return configFileName; + } + private static Map> toMap(Object... objects) { final ImmutableMap.Builder> map = ImmutableMap.builder(); for (int i = 0; i < objects.length; i = i + 2) { diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/AccountApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/AccountApiTest.java deleted file mode 100644 index 21f68d11df..0000000000 --- a/src/test/java/org/opensearch/security/dlic/rest/api/AccountApiTest.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.security.dlic.rest.api; - -import org.apache.hc.core5.http.Header; -import org.apache.http.HttpStatus; -import org.junit.Assert; -import org.junit.Test; - -import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.XContentType; -import org.opensearch.security.securityconf.impl.CType; -import org.opensearch.security.test.helper.rest.RestHelper.HttpResponse; - -import static org.opensearch.security.OpenSearchSecurityPlugin.PLUGINS_PREFIX; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -public class AccountApiTest extends AbstractRestApiUnitTest { - private final String BASE_ENDPOINT; - private final String ENDPOINT; - - protected String getEndpointPrefix() { - return PLUGINS_PREFIX; - } - - public AccountApiTest() { - BASE_ENDPOINT = getEndpointPrefix() + "/api/"; - ENDPOINT = getEndpointPrefix() + "/api/account"; - } - - @Test - public void testGetAccount() throws Exception { - // arrange - setup(); - final String testUser = "test-user"; - final String testPass = "some password for user"; - addUserWithPassword(testUser, testPass, HttpStatus.SC_CREATED); - - // test - unauthorized access as credentials are missing. - HttpResponse response = rh.executeGetRequest(ENDPOINT, new Header[0]); - assertEquals(HttpStatus.SC_UNAUTHORIZED, response.getStatusCode()); - - // test - incorrect password - response = rh.executeGetRequest(ENDPOINT, encodeBasicHeader(testUser, "wrong-pass")); - assertEquals(HttpStatus.SC_UNAUTHORIZED, response.getStatusCode()); - - // test - incorrect user - response = rh.executeGetRequest(ENDPOINT, encodeBasicHeader("wrong-user", testPass)); - assertEquals(HttpStatus.SC_UNAUTHORIZED, response.getStatusCode()); - - // test - valid request - response = rh.executeGetRequest(ENDPOINT, encodeBasicHeader(testUser, testPass)); - Settings body = Settings.builder().loadFromSource(response.getBody(), XContentType.JSON).build(); - assertEquals(HttpStatus.SC_OK, response.getStatusCode()); - assertEquals(testUser, body.get("user_name")); - assertFalse(body.getAsBoolean("is_reserved", true)); - assertFalse(body.getAsBoolean("is_hidden", true)); - assertTrue(body.getAsBoolean("is_internal_user", false)); - assertNull(body.get("user_requested_tenant")); - assertNotNull(body.getAsList("backend_roles").size()); - assertNotNull(body.getAsList("custom_attribute_names").size()); - assertNotNull(body.getAsSettings("tenants")); - assertNotNull(body.getAsList("roles")); - } - - @Test - public void testPutAccount() throws Exception { - // arrange - setup(); - final String testUser = "test-user"; - final String testPass = "test-old-pass"; - final String testPassHash = "$2y$12$b7TNPn2hgl0nS7gXJ.beuOd8JGl6Nz5NsTyxofglGCItGNyDdwivK"; // hash for test-old-pass - final String testNewPass = "test-new-pass"; - final String testNewPassHash = "$2y$12$cclJJdVdXMMVzkhqQhEoE.hoERKE8bDzctR0S3aYj2EPHq45Y.GXC"; // hash for test-old-pass - addUserWithPassword(testUser, testPass, HttpStatus.SC_CREATED); - - // test - unauthorized access as credentials are missing. - HttpResponse response = rh.executePutRequest(ENDPOINT, "", new Header[0]); - assertEquals(HttpStatus.SC_UNAUTHORIZED, response.getStatusCode()); - - // test - bad request as body is missing - response = rh.executePutRequest(ENDPOINT, "", encodeBasicHeader(testUser, testPass)); - assertEquals(HttpStatus.SC_BAD_REQUEST, response.getStatusCode()); - - // test - bad request as current password is missing - String payload = "{\"password\":\"new-pass\"}"; - response = rh.executePutRequest(ENDPOINT, payload, encodeBasicHeader(testUser, testPass)); - assertEquals(HttpStatus.SC_BAD_REQUEST, response.getStatusCode()); - - // test - bad request as current password is incorrect - payload = "{\"password\":\"" + testNewPass + "\", \"current_password\":\"" + "wrong-pass" + "\"}"; - response = rh.executePutRequest(ENDPOINT, payload, encodeBasicHeader(testUser, testPass)); - assertEquals(HttpStatus.SC_BAD_REQUEST, response.getStatusCode()); - - // test - bad request as hash/password is missing - payload = "{\"current_password\":\"" + testPass + "\"}"; - response = rh.executePutRequest(ENDPOINT, payload, encodeBasicHeader(testUser, testPass)); - assertEquals(HttpStatus.SC_BAD_REQUEST, response.getStatusCode()); - - // test - bad request as password is empty - payload = "{\"password\":\"" + "" + "\", \"current_password\":\"" + testPass + "\"}"; - response = rh.executePutRequest(ENDPOINT, payload, encodeBasicHeader(testUser, testPass)); - assertEquals(HttpStatus.SC_BAD_REQUEST, response.getStatusCode()); - - // test - bad request as hash is empty - payload = "{\"hash\":\"" + "" + "\", \"current_password\":\"" + testPass + "\"}"; - response = rh.executePutRequest(ENDPOINT, payload, encodeBasicHeader(testUser, testPass)); - assertEquals(HttpStatus.SC_BAD_REQUEST, response.getStatusCode()); - - // test - bad request as hash and password are empty - payload = "{\"hash\": \"\", \"password\": \"\", \"current_password\":\"" + testPass + "\"}"; - response = rh.executePutRequest(ENDPOINT, payload, encodeBasicHeader(testUser, testPass)); - assertEquals(HttpStatus.SC_BAD_REQUEST, response.getStatusCode()); - - // test - bad request as invalid parameters are present - payload = "{\"password\":\"new-pass\", \"current_password\":\"" + testPass + "\", \"backend_roles\": []}"; - response = rh.executePutRequest(ENDPOINT, payload, encodeBasicHeader(testUser, testPass)); - assertEquals(HttpStatus.SC_BAD_REQUEST, response.getStatusCode()); - - // test - invalid user - payload = "{\"password\":\"" + testNewPass + "\", \"current_password\":\"" + testPass + "\"}"; - response = rh.executePutRequest(ENDPOINT, payload, encodeBasicHeader("wrong-user", testPass)); - assertEquals(HttpStatus.SC_UNAUTHORIZED, response.getStatusCode()); - - // test - valid password change with hash - payload = "{\"hash\":\"" + testNewPassHash + "\", \"current_password\":\"" + testPass + "\"}"; - response = rh.executePutRequest(ENDPOINT, payload, encodeBasicHeader(testUser, testPass)); - assertEquals(HttpStatus.SC_OK, response.getStatusCode()); - - // test - valid password change - payload = "{\"password\":\"" + testPass + "\", \"current_password\":\"" + testNewPass + "\"}"; - response = rh.executePutRequest(ENDPOINT, payload, encodeBasicHeader(testUser, testNewPass)); - assertEquals(HttpStatus.SC_OK, response.getStatusCode()); - - // create users from - resources/restapi/internal_users.yml - rh.keystore = "restapi/kirk-keystore.jks"; - rh.sendAdminCertificate = true; - response = rh.executeGetRequest(BASE_ENDPOINT + CType.INTERNALUSERS.toLCString()); - rh.sendAdminCertificate = false; - Assert.assertEquals(response.getBody(), HttpStatus.SC_OK, response.getStatusCode()); - - // test - reserved user - sarek - response = rh.executeGetRequest(ENDPOINT, encodeBasicHeader("sarek", "sarek")); - Settings body = Settings.builder().loadFromSource(response.getBody(), XContentType.JSON).build(); - // check reserved user exists - assertTrue(body.getAsBoolean("is_reserved", false)); - payload = "{\"password\":\"" + testPass + "\", \"current_password\":\"" + "sarek" + "\"}"; - response = rh.executePutRequest(ENDPOINT, payload, encodeBasicHeader("sarek", "sarek")); - assertEquals(HttpStatus.SC_FORBIDDEN, response.getStatusCode()); - - // test - hidden user - hide - response = rh.executeGetRequest(ENDPOINT, encodeBasicHeader("hide", "hide")); - body = Settings.builder().loadFromSource(response.getBody(), XContentType.JSON).build(); - // check hidden user exists - assertTrue(body.getAsBoolean("is_hidden", false)); - payload = "{\"password\":\"" + testPass + "\", \"current_password\":\"" + "hide" + "\"}"; - response = rh.executePutRequest(ENDPOINT, payload, encodeBasicHeader("hide", "hide")); - assertEquals(HttpStatus.SC_NOT_FOUND, response.getStatusCode()); - - // test - admin with admin cert - internal user does not exist - rh.keystore = "restapi/kirk-keystore.jks"; - rh.sendAdminCertificate = true; - response = rh.executeGetRequest(ENDPOINT, encodeBasicHeader("admin", "admin")); - body = Settings.builder().loadFromSource(response.getBody(), XContentType.JSON).build(); - assertEquals("CN=kirk,OU=client,O=client,L=Test,C=DE", body.get("user_name")); - assertEquals(HttpStatus.SC_OK, response.getStatusCode()); // check admin user exists - payload = "{\"password\":\"" + testPass + "\", \"current_password\":\"" + "admin" + "\"}"; - response = rh.executePutRequest(ENDPOINT, payload, encodeBasicHeader("admin", "admin")); - assertEquals(HttpStatus.SC_NOT_FOUND, response.getStatusCode()); - } - - @Test - public void testPutAccountRetainsAccountInformation() throws Exception { - // arrange - setup(); - final String testUsername = "test"; - final String testPassword = "test-password"; - final String newPassword = "new-password"; - final String createInternalUserPayload = "{\n" - + " \"password\": \"" - + testPassword - + "\",\n" - + " \"backend_roles\": [\"test-backend-role-1\"],\n" - + " \"opendistro_security_roles\": [\"opendistro_security_all_access\"],\n" - + " \"attributes\": {\n" - + " \"attribute1\": \"value1\"\n" - + " }\n" - + "}"; - final String changePasswordPayload = "{\"password\":\"" + newPassword + "\", \"current_password\":\"" + testPassword + "\"}"; - final String internalUserEndpoint = BASE_ENDPOINT + "internalusers/" + testUsername; - - // create user - rh.sendAdminCertificate = true; - HttpResponse response = rh.executePutRequest(internalUserEndpoint, createInternalUserPayload); - assertEquals(HttpStatus.SC_OK, response.getStatusCode()); - rh.sendAdminCertificate = false; - - // change password to new-password - response = rh.executePutRequest(ENDPOINT, changePasswordPayload, encodeBasicHeader(testUsername, testPassword)); - assertEquals(HttpStatus.SC_OK, response.getStatusCode()); - - // assert account information has not changed - rh.sendAdminCertificate = true; - response = rh.executeGetRequest(internalUserEndpoint); - assertEquals(HttpStatus.SC_OK, response.getStatusCode()); - Settings responseBody = Settings.builder() - .loadFromSource(response.getBody(), XContentType.JSON) - .build() - .getAsSettings(testUsername); - assertTrue(responseBody.getAsList("backend_roles").contains("test-backend-role-1")); - assertTrue(responseBody.getAsList("opendistro_security_roles").contains("opendistro_security_all_access")); - assertEquals(responseBody.getAsSettings("attributes").get("attribute1"), "value1"); - } -} diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/DashboardsInfoActionTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/DashboardsInfoActionTest.java deleted file mode 100644 index 647e7a2a33..0000000000 --- a/src/test/java/org/opensearch/security/dlic/rest/api/DashboardsInfoActionTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.security.dlic.rest.api; - -import org.apache.http.HttpStatus; -import org.junit.Assert; -import org.junit.Test; - -import org.opensearch.common.settings.Settings; -import org.opensearch.security.support.ConfigConstants; -import org.opensearch.security.test.helper.rest.RestHelper; - -import static org.opensearch.security.OpenSearchSecurityPlugin.PLUGINS_PREFIX; - -public class DashboardsInfoActionTest extends AbstractRestApiUnitTest { - private final String ENDPOINT; - - protected String getEndpoint() { - return PLUGINS_PREFIX + "/dashboardsinfo"; - } - - public DashboardsInfoActionTest() { - ENDPOINT = getEndpoint(); - } - - @Test - public void testDashboardsInfo() throws Exception { - Settings settings = Settings.builder() - .put(ConfigConstants.SECURITY_UNSUPPORTED_RESTAPI_ALLOW_SECURITYCONFIG_MODIFICATION, true) - .build(); - setup(settings); - - rh.keystore = "restapi/kirk-keystore.jks"; - rh.sendAdminCertificate = true; - - RestHelper.HttpResponse response = rh.executeGetRequest(ENDPOINT); - Assert.assertEquals(HttpStatus.SC_OK, response.getStatusCode()); - } - -} diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/FlushCacheApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/FlushCacheApiTest.java deleted file mode 100644 index ee75ccc984..0000000000 --- a/src/test/java/org/opensearch/security/dlic/rest/api/FlushCacheApiTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.security.dlic.rest.api; - -import org.apache.hc.core5.http.Header; -import org.apache.http.HttpStatus; -import org.junit.Assert; -import org.junit.Test; - -import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.XContentType; -import org.opensearch.security.test.helper.rest.RestHelper.HttpResponse; - -import static org.opensearch.security.OpenSearchSecurityPlugin.PLUGINS_PREFIX; - -public class FlushCacheApiTest extends AbstractRestApiUnitTest { - private final String ENDPOINT; - - protected String getEndpointPrefix() { - return PLUGINS_PREFIX; - } - - public FlushCacheApiTest() { - ENDPOINT = getEndpointPrefix() + "/api/cache"; - } - - @Test - public void testFlushCache() throws Exception { - - setup(); - - // Only DELETE is allowed for flush cache - rh.keystore = "restapi/kirk-keystore.jks"; - rh.sendAdminCertificate = true; - - // GET - HttpResponse response = rh.executeGetRequest(ENDPOINT); - Assert.assertEquals(HttpStatus.SC_NOT_IMPLEMENTED, response.getStatusCode()); - Settings settings = Settings.builder().loadFromSource(response.getBody(), XContentType.JSON).build(); - Assert.assertEquals(settings.get("message"), "Method GET not supported for this action."); - - // PUT - response = rh.executePutRequest(ENDPOINT, "{}", new Header[0]); - Assert.assertEquals(HttpStatus.SC_NOT_IMPLEMENTED, response.getStatusCode()); - settings = Settings.builder().loadFromSource(response.getBody(), XContentType.JSON).build(); - Assert.assertEquals(settings.get("message"), "Method PUT not supported for this action."); - - // POST - response = rh.executePostRequest(ENDPOINT, "{}", new Header[0]); - Assert.assertEquals(HttpStatus.SC_NOT_IMPLEMENTED, response.getStatusCode()); - settings = Settings.builder().loadFromSource(response.getBody(), XContentType.JSON).build(); - Assert.assertEquals(settings.get("message"), "Method POST not supported for this action."); - - // DELETE - response = rh.executeDeleteRequest(ENDPOINT, new Header[0]); - Assert.assertEquals(HttpStatus.SC_OK, response.getStatusCode()); - settings = Settings.builder().loadFromSource(response.getBody(), XContentType.JSON).build(); - Assert.assertEquals(settings.get("message"), "Cache flushed successfully."); - - } -} diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/legacy/LegacyAccountApiTests.java b/src/test/java/org/opensearch/security/dlic/rest/api/legacy/LegacyAccountApiTests.java deleted file mode 100644 index 925d90ccba..0000000000 --- a/src/test/java/org/opensearch/security/dlic/rest/api/legacy/LegacyAccountApiTests.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.security.dlic.rest.api.legacy; - -import org.opensearch.security.dlic.rest.api.AccountApiTest; - -import static org.opensearch.security.OpenSearchSecurityPlugin.LEGACY_OPENDISTRO_PREFIX; - -public class LegacyAccountApiTests extends AccountApiTest { - @Override - protected String getEndpointPrefix() { - return LEGACY_OPENDISTRO_PREFIX; - } -} diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/legacy/LegacyDashboardsInfoActionTests.java b/src/test/java/org/opensearch/security/dlic/rest/api/legacy/LegacyDashboardsInfoActionTests.java deleted file mode 100644 index ee39f93ee0..0000000000 --- a/src/test/java/org/opensearch/security/dlic/rest/api/legacy/LegacyDashboardsInfoActionTests.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.security.dlic.rest.api.legacy; - -import org.opensearch.security.dlic.rest.api.DashboardsInfoActionTest; - -import static org.opensearch.security.OpenSearchSecurityPlugin.LEGACY_OPENDISTRO_PREFIX; - -public class LegacyDashboardsInfoActionTests extends DashboardsInfoActionTest { - @Override - protected String getEndpoint() { - return LEGACY_OPENDISTRO_PREFIX + "/kibanainfo"; - } -} diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/legacy/LegacyFlushCacheApiTests.java b/src/test/java/org/opensearch/security/dlic/rest/api/legacy/LegacyFlushCacheApiTests.java deleted file mode 100644 index df9cc3d59d..0000000000 --- a/src/test/java/org/opensearch/security/dlic/rest/api/legacy/LegacyFlushCacheApiTests.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.security.dlic.rest.api.legacy; - -import org.opensearch.security.dlic.rest.api.FlushCacheApiTest; - -import static org.opensearch.security.OpenSearchSecurityPlugin.LEGACY_OPENDISTRO_PREFIX; - -public class LegacyFlushCacheApiTests extends FlushCacheApiTest { - @Override - protected String getEndpointPrefix() { - return LEGACY_OPENDISTRO_PREFIX; - } -} From cb3496234268d900aa1fb2b1c33d47ff2bdf80a8 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Mon, 15 Apr 2024 14:08:04 -0400 Subject: [PATCH 185/204] Move Dave Lago to emeritus (#4235) Signed-off-by: Derek Ho --- .github/CODEOWNERS | 2 +- MAINTAINERS.md | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5041f4b625..3b3a442b83 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @cliu123 @cwperks @DarshitChanpura @davidlago @peternied @RyanL1997 @scrawfor99 @reta @willyborankin +* @cliu123 @cwperks @DarshitChanpura @peternied @RyanL1997 @scrawfor99 @reta @willyborankin diff --git a/MAINTAINERS.md b/MAINTAINERS.md index f27e192923..bafadc11bd 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -16,7 +16,6 @@ This document contains a list of maintainers in this repo. See [opensearch-proje | ---------------- | ----------------------------------------------------- | ----------- | | Chang Liu | [cliu123](https://github.com/cliu123) | Amazon | | Darshit Chanpura | [DarshitChanpura](https://github.com/DarshitChanpura) | Amazon | -| Dave Lago | [davidlago](https://github.com/davidlago) | Amazon | | Peter Nied | [peternied](https://github.com/peternied) | Amazon | | Craig Perkins | [cwperks](https://github.com/cwperks) | Amazon | | Ryan Liang | [RyanL1997](https://github.com/RyanL1997) | Amazon | @@ -24,6 +23,12 @@ This document contains a list of maintainers in this repo. See [opensearch-proje | Andriy Redko | [reta](https://github.com/reta) | Aiven | | Andrey Pleskach | [willyborankin](https://github.com/willyborankin) | Aiven | +## Emeritus + +| Maintainer | GitHub ID | Affiliation | +| ------------- | --------------------------------------------------- | ----------- | +| Dave Lago | [davidlago](https://github.com/davidlago) | Contributor | + ## Practices ### Updating Practices From 0bfe7653b0f50ff6b6757411af4e49d9784a063b Mon Sep 17 00:00:00 2001 From: Aayush Singhal <42822023+Aayush8394@users.noreply.github.com> Date: Tue, 16 Apr 2024 20:16:04 +0530 Subject: [PATCH 186/204] Handle exceptions for adding request body audit log if rest request is invalid (#4232) Signed-off-by: Aayush Singhal --- .../security/auditlog/impl/AuditMessage.java | 7 ++- .../auditlog/impl/AuditMessageTest.java | 44 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/security/auditlog/impl/AuditMessage.java b/src/main/java/org/opensearch/security/auditlog/impl/AuditMessage.java index 716e141ffd..bf3395f193 100644 --- a/src/main/java/org/opensearch/security/auditlog/impl/AuditMessage.java +++ b/src/main/java/org/opensearch/security/auditlog/impl/AuditMessage.java @@ -27,6 +27,8 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.commons.codec.digest.DigestUtils; import org.apache.hc.core5.net.URIBuilder; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.ExceptionsHelper; import org.opensearch.cluster.service.ClusterService; @@ -59,6 +61,8 @@ public final class AuditMessage { + private static final Logger log = LogManager.getLogger(AuditMessage.class); + // clustername and cluster uuid private static final WildcardMatcher AUTHORIZATION_HEADER = WildcardMatcher.from("Authorization", false); private static final String SENSITIVE_KEY = "password"; @@ -417,8 +421,9 @@ void addRestRequestInfo(final SecurityRequest request, final AuditConfig.Filter } else { auditInfo.put(REQUEST_BODY, requestBody); } - } catch (IOException e) { + } catch (Exception e) { auditInfo.put(REQUEST_BODY, "ERROR: Unable to generate request body"); + log.error("Error while generating request body for audit log", e); } } } diff --git a/src/test/java/org/opensearch/security/auditlog/impl/AuditMessageTest.java b/src/test/java/org/opensearch/security/auditlog/impl/AuditMessageTest.java index 3b7fc916ef..08e4c2ea61 100644 --- a/src/test/java/org/opensearch/security/auditlog/impl/AuditMessageTest.java +++ b/src/test/java/org/opensearch/security/auditlog/impl/AuditMessageTest.java @@ -26,9 +26,16 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.collect.Tuple; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.http.HttpChannel; +import org.opensearch.http.HttpRequest; +import org.opensearch.rest.RestRequest; import org.opensearch.security.auditlog.AuditLog; import org.opensearch.security.auditlog.config.AuditConfig; +import org.opensearch.security.filter.SecurityRequest; +import org.opensearch.security.filter.SecurityRequestFactory; import org.opensearch.security.securityconf.impl.CType; import static org.junit.Assert.assertEquals; @@ -155,4 +162,41 @@ public void testBCryptHashIsRedacted() { message.addSecurityConfigTupleToRequestBody(new Tuple<>(XContentType.JSON, ref), internalUsersDocId); assertEquals("Hash in tuple is __HASH__", message.getAsMap().get(AuditMessage.REQUEST_BODY)); } + + @Test + public void testRequestBodyLoggingWithInvalidSourceOrContentTypeParam() { + when(auditConfig.getFilter().shouldLogRequestBody()).thenReturn(true); + + HttpRequest httpRequest = mock(HttpRequest.class); + + // No content or Source paramater + when(httpRequest.uri()).thenReturn(""); + when(httpRequest.content()).thenReturn(new BytesArray(new byte[0])); + + RestRequest restRequest = RestRequest.request(mock(NamedXContentRegistry.class), httpRequest, mock(HttpChannel.class)); + SecurityRequest request = SecurityRequestFactory.from(restRequest); + + message.addRestRequestInfo(request, auditConfig.getFilter()); + assertNull(message.getAsMap().get(AuditMessage.REQUEST_BODY)); + + // No source parameter, content present but Invalid content-type header + when(httpRequest.uri()).thenReturn(""); + when(httpRequest.content()).thenReturn(new BytesArray(new byte[1])); + + restRequest = RestRequest.request(mock(NamedXContentRegistry.class), httpRequest, mock(HttpChannel.class)); + request = SecurityRequestFactory.from(restRequest); + + message.addRestRequestInfo(request, auditConfig.getFilter()); + assertEquals("ERROR: Unable to generate request body", message.getAsMap().get(AuditMessage.REQUEST_BODY)); + + // No content, source parameter present but Invalid source-content-type parameter + when(httpRequest.uri()).thenReturn("/aaaa?source=request_body"); + when(httpRequest.content()).thenReturn(new BytesArray(new byte[0])); + + restRequest = RestRequest.request(mock(NamedXContentRegistry.class), httpRequest, mock(HttpChannel.class)); + request = SecurityRequestFactory.from(restRequest); + + message.addRestRequestInfo(request, auditConfig.getFilter()); + assertEquals("ERROR: Unable to generate request body", message.getAsMap().get(AuditMessage.REQUEST_BODY)); + } } From 9a85f23e88725f7447a407e180f3536539fb5736 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 18 Apr 2024 03:09:11 -0400 Subject: [PATCH 187/204] Add getProperty.org.bouncycastle.pkcs12.default to plugin-security.policy (#4265) Signed-off-by: Craig Perkins --- plugin-security.policy | 1 + 1 file changed, 1 insertion(+) diff --git a/plugin-security.policy b/plugin-security.policy index 2969e47b04..d4e1fe6998 100644 --- a/plugin-security.policy +++ b/plugin-security.policy @@ -63,6 +63,7 @@ grant { permission java.security.SecurityPermission "removeProviderProperty.BC"; permission java.security.SecurityPermission "getProperty.org.bouncycastle.rsa.max_size"; permission java.security.SecurityPermission "getProperty.org.bouncycastle.rsa.max_mr_tests"; + permission java.security.SecurityPermission "getProperty.org.bouncycastle.pkcs12.default"; permission java.lang.RuntimePermission "accessUserInformation"; From a528c9135f48dfa07c058d7ff6dbc21d9fe1eb4d Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 18 Apr 2024 15:12:49 -0400 Subject: [PATCH 188/204] Ensure that challenge response contains body (#4233) Signed-off-by: Craig Perkins --- .../opensearch/security/ResourceFocusedTests.java | 10 +++++++--- .../opensearch/security/http/BasicAuthTests.java | 13 +++++++++++++ .../security/http/HTTPBasicAuthenticator.java | 6 +++++- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java b/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java index 61a1e32023..7264a33542 100644 --- a/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java +++ b/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java @@ -41,6 +41,8 @@ import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS; @@ -127,11 +129,13 @@ private void runResourceTest( final var requests = AsyncActions.generate(() -> { final HttpPost post = new HttpPost(client.getHttpServerUri() + requestPath); post.setEntity(new ByteArrayEntity(compressedRequestBody, ContentType.APPLICATION_JSON)); - return client.executeRequest(post); + TestRestClient.HttpResponse response = client.executeRequest(post); + return response.getStatusCode(); }, parrallelism, totalNumberOfRequests); - AsyncActions.getAll(requests, 2, TimeUnit.MINUTES) - .forEach((response) -> { response.assertStatusCode(HttpStatus.SC_UNAUTHORIZED); }); + AsyncActions.getAll(requests, 2, TimeUnit.MINUTES).forEach((responseCode) -> { + assertThat(responseCode, equalTo(HttpStatus.SC_UNAUTHORIZED)); + }); } } diff --git a/src/integrationTest/java/org/opensearch/security/http/BasicAuthTests.java b/src/integrationTest/java/org/opensearch/security/http/BasicAuthTests.java index 1e424ab115..a9888d281e 100644 --- a/src/integrationTest/java/org/opensearch/security/http/BasicAuthTests.java +++ b/src/integrationTest/java/org/opensearch/security/http/BasicAuthTests.java @@ -106,6 +106,19 @@ public void testBrowserShouldRequestForCredentials() { } } + @Test + public void shouldRespondWithChallengeWhenNoCredentialsArePresent() { + try (TestRestClient client = cluster.getRestClient()) { + HttpResponse response = client.getAuthInfo(); + + assertThat(response, is(notNullValue())); + response.assertStatusCode(SC_UNAUTHORIZED); + assertThat(response.getHeader("WWW-Authenticate"), is(notNullValue())); + assertThat(response.getHeader("WWW-Authenticate").getValue(), equalTo("Basic realm=\"OpenSearch Security\"")); + assertThat(response.getBody(), equalTo("Unauthorized")); + } + } + @Test public void testUserShouldNotHaveAssignedCustomAttributes() { try (TestRestClient client = cluster.getRestClient(TEST_USER)) { diff --git a/src/main/java/org/opensearch/security/http/HTTPBasicAuthenticator.java b/src/main/java/org/opensearch/security/http/HTTPBasicAuthenticator.java index ff07db147e..4aec26db3d 100644 --- a/src/main/java/org/opensearch/security/http/HTTPBasicAuthenticator.java +++ b/src/main/java/org/opensearch/security/http/HTTPBasicAuthenticator.java @@ -68,7 +68,11 @@ public AuthCredentials extractCredentials(final SecurityRequest request, final T @Override public Optional reRequestAuthentication(final SecurityRequest request, AuthCredentials creds) { return Optional.of( - new SecurityResponse(HttpStatus.SC_UNAUTHORIZED, Map.of("WWW-Authenticate", "Basic realm=\"OpenSearch Security\""), "") + new SecurityResponse( + HttpStatus.SC_UNAUTHORIZED, + Map.of("WWW-Authenticate", "Basic realm=\"OpenSearch Security\""), + "Unauthorized" + ) ); } From b2142552bf876f4f1a252967e06d5201b8493cee Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Fri, 19 Apr 2024 10:21:58 -0400 Subject: [PATCH 189/204] Add getProperty.org.bouncycastle.ec.max_f2m_field_size to plugin-security.policy (#4269) Signed-off-by: Craig Perkins --- plugin-security.policy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugin-security.policy b/plugin-security.policy index d4e1fe6998..6a78a5cc91 100644 --- a/plugin-security.policy +++ b/plugin-security.policy @@ -61,9 +61,10 @@ grant { permission java.security.SecurityPermission "putProviderProperty.BC"; permission java.security.SecurityPermission "insertProvider.BC"; permission java.security.SecurityPermission "removeProviderProperty.BC"; + permission java.security.SecurityPermission "getProperty.org.bouncycastle.ec.max_f2m_field_size"; + permission java.security.SecurityPermission "getProperty.org.bouncycastle.pkcs12.default"; permission java.security.SecurityPermission "getProperty.org.bouncycastle.rsa.max_size"; permission java.security.SecurityPermission "getProperty.org.bouncycastle.rsa.max_mr_tests"; - permission java.security.SecurityPermission "getProperty.org.bouncycastle.pkcs12.default"; permission java.lang.RuntimePermission "accessUserInformation"; From 9cf5cdfb38fce6b47c1710c17a58a9b521680d69 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Apr 2024 10:48:17 -0400 Subject: [PATCH 190/204] Bump ch.qos.logback:logback-classic from 1.5.5 to 1.5.6 (#4277) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a4e4a7d037..bbba0600da 100644 --- a/build.gradle +++ b/build.gradle @@ -496,7 +496,7 @@ configurations { force "org.apache.httpcomponents:httpcore:4.4.16" force "com.google.errorprone:error_prone_annotations:2.26.1" force "org.checkerframework:checker-qual:3.42.0" - force "ch.qos.logback:logback-classic:1.5.5" + force "ch.qos.logback:logback-classic:1.5.6" } } From 630f40b7bfeef392c1e4acbf80b1eb5d05acdc3e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Apr 2024 10:48:48 -0400 Subject: [PATCH 191/204] Bump Wandalen/wretry.action from 3.3.0 to 3.4.0 (#4276) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94fbf11765..75174cd260 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,7 +81,7 @@ jobs: working-directory: downloaded-artifacts - name: Upload Coverage with retry - uses: Wandalen/wretry.action@v3.3.0 + uses: Wandalen/wretry.action@v3.4.0 with: attempt_limit: 5 attempt_delay: 2000 From c09fad59f26b56e0e4f43086b0709571d2aedaf1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Apr 2024 10:49:07 -0400 Subject: [PATCH 192/204] Bump org.apache.commons:commons-text from 1.11.0 to 1.12.0 (#4278) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index bbba0600da..445a6457b2 100644 --- a/build.gradle +++ b/build.gradle @@ -634,7 +634,7 @@ dependencies { implementation "com.nulab-inc:zxcvbn:1.9.0" runtimeOnly 'com.google.guava:failureaccess:1.0.2' - runtimeOnly 'org.apache.commons:commons-text:1.11.0' + runtimeOnly 'org.apache.commons:commons-text:1.12.0' runtimeOnly "org.glassfish.jaxb:jaxb-runtime:${jaxb_version}" runtimeOnly 'com.google.j2objc:j2objc-annotations:2.8' compileOnly 'com.google.code.findbugs:jsr305:3.0.2' From 0d7af4d37e1f1b6dca25ed8edbf8c2cf3f131742 Mon Sep 17 00:00:00 2001 From: Terry Quigley <77437788+terryquigleysas@users.noreply.github.com> Date: Tue, 23 Apr 2024 15:50:08 +0100 Subject: [PATCH 193/204] Replace bouncy castle blake2b (#4275) Signed-off-by: Terry Quigley --- build.gradle | 1 + .../opensearch/security/configuration/MaskedField.java | 9 ++++++--- .../opensearch/security/test/helper/rest/RestHelper.java | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 445a6457b2..ad0a58313f 100644 --- a/build.gradle +++ b/build.gradle @@ -580,6 +580,7 @@ dependencies { implementation "org.bouncycastle:bcprov-jdk18on:${versions.bouncycastle}" implementation 'org.ldaptive:ldaptive:1.2.3' implementation 'com.nimbusds:nimbus-jose-jwt:9.37.3' + implementation 'com.rfksystems:blake2b:2.0.0' //JWT implementation "io.jsonwebtoken:jjwt-api:${jjwt_version}" diff --git a/src/main/java/org/opensearch/security/configuration/MaskedField.java b/src/main/java/org/opensearch/security/configuration/MaskedField.java index 8cb20ccdfe..2636047568 100644 --- a/src/main/java/org/opensearch/security/configuration/MaskedField.java +++ b/src/main/java/org/opensearch/security/configuration/MaskedField.java @@ -21,9 +21,10 @@ import com.google.common.base.Splitter; import org.apache.lucene.util.BytesRef; -import org.bouncycastle.crypto.digests.Blake2bDigest; import org.bouncycastle.util.encoders.Hex; +import com.rfksystems.blake2b.Blake2b; + public class MaskedField { private final String name; @@ -164,10 +165,12 @@ private String customHash(String in) { } private byte[] blake2bHash(byte[] in) { - final Blake2bDigest hash = new Blake2bDigest(null, 32, null, defaultSalt); + // Salt is passed incorrectly but order of parameters is retained at present to ensure full backwards compatibility + // Tracking with https://github.com/opensearch-project/security/issues/4274 + final Blake2b hash = new Blake2b(null, 32, null, defaultSalt); hash.update(in, 0, in.length); final byte[] out = new byte[hash.getDigestSize()]; - hash.doFinal(out, 0); + hash.digest(out, 0); return Hex.encode(out); } diff --git a/src/test/java/org/opensearch/security/test/helper/rest/RestHelper.java b/src/test/java/org/opensearch/security/test/helper/rest/RestHelper.java index c137591825..1710a93875 100644 --- a/src/test/java/org/opensearch/security/test/helper/rest/RestHelper.java +++ b/src/test/java/org/opensearch/security/test/helper/rest/RestHelper.java @@ -397,7 +397,7 @@ public static class HttpResponse { public HttpResponse(SimpleHttpResponse inner) throws IllegalStateException, IOException { super(); this.inner = inner; - if (inner.getBody() == null) { // head request does not have a entity + if (inner.getBody() == null) { // head request does not have an entity this.body = ""; } else { this.body = inner.getBodyText(); From 6b9762b17ac109c268b7cfc58ad42e39c4349615 Mon Sep 17 00:00:00 2001 From: lily Date: Fri, 26 Apr 2024 08:41:53 +0900 Subject: [PATCH 194/204] System index permission grants reading access to documents in the index (#4201) Signed-off-by: llilyy Co-authored-by: hyeonyoung-na --- .../SecurityIndexSearcherWrapper.java | 19 ++++++++++++++++++ .../privileges/PrivilegesEvaluator.java | 2 +- .../security/securityconf/ConfigModelV6.java | 20 +++++++++++++++++++ .../security/securityconf/ConfigModelV7.java | 18 +++++++++++++++++ .../security/securityconf/SecurityRoles.java | 2 ++ .../SecurityRolesPermissionsV6Test.java | 11 ++++++++++ .../SystemIndexPermissionEnabledTests.java | 3 ++- 7 files changed, 73 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/opensearch/security/configuration/SecurityIndexSearcherWrapper.java b/src/main/java/org/opensearch/security/configuration/SecurityIndexSearcherWrapper.java index b2008861aa..0602cb0bed 100644 --- a/src/main/java/org/opensearch/security/configuration/SecurityIndexSearcherWrapper.java +++ b/src/main/java/org/opensearch/security/configuration/SecurityIndexSearcherWrapper.java @@ -41,6 +41,7 @@ import org.opensearch.index.IndexService; import org.opensearch.security.privileges.PrivilegesEvaluator; import org.opensearch.security.securityconf.ConfigModel; +import org.opensearch.security.securityconf.SecurityRoles; import org.opensearch.security.support.ConfigConstants; import org.opensearch.security.support.HeaderHelper; import org.opensearch.security.support.WildcardMatcher; @@ -64,6 +65,8 @@ public class SecurityIndexSearcherWrapper implements CheckedFunction mappedRoles = evaluator.mapRoles(user, caller); + final SecurityRoles securityRoles = evaluator.getSecurityRoles(mappedRoles); + return !securityRoles.isPermittedOnSystemIndex(index.getName()); + } return systemIndexMatcher.test(index.getName()); } diff --git a/src/main/java/org/opensearch/security/privileges/PrivilegesEvaluator.java b/src/main/java/org/opensearch/security/privileges/PrivilegesEvaluator.java index 06e41a2aaa..872c2c37f2 100644 --- a/src/main/java/org/opensearch/security/privileges/PrivilegesEvaluator.java +++ b/src/main/java/org/opensearch/security/privileges/PrivilegesEvaluator.java @@ -193,7 +193,7 @@ public void onDynamicConfigModelChanged(DynamicConfigModel dcm) { this.dcm = dcm; } - private SecurityRoles getSecurityRoles(Set roles) { + public SecurityRoles getSecurityRoles(Set roles) { return configModel.getSecurityRoles().filter(roles); } diff --git a/src/main/java/org/opensearch/security/securityconf/ConfigModelV6.java b/src/main/java/org/opensearch/security/securityconf/ConfigModelV6.java index 3650057d63..e35fb40a24 100644 --- a/src/main/java/org/opensearch/security/securityconf/ConfigModelV6.java +++ b/src/main/java/org/opensearch/security/securityconf/ConfigModelV6.java @@ -542,6 +542,26 @@ public boolean impliesTypePermGlobal( roles.stream().forEach(p -> ipatterns.addAll(p.getIpatterns())); return ConfigModelV6.impliesTypePerm(ipatterns, resolved, user, actions, resolver, cs); } + + @Override + public boolean isPermittedOnSystemIndex(String indexName) { + boolean isPatternMatched = false; + boolean isPermitted = false; + for (SecurityRole role : roles) { + for (IndexPattern ip : role.getIpatterns()) { + WildcardMatcher wildcardMatcher = WildcardMatcher.from(ip.indexPattern); + if (wildcardMatcher.test(indexName)) { + isPatternMatched = true; + } + for (TypePerm tp : ip.getTypePerms()) { + if (tp.perms.contains(ConfigConstants.SYSTEM_INDEX_PERMISSION)) { + isPermitted = true; + } + } + } + } + return isPatternMatched && isPermitted; + } } public static class SecurityRole { diff --git a/src/main/java/org/opensearch/security/securityconf/ConfigModelV7.java b/src/main/java/org/opensearch/security/securityconf/ConfigModelV7.java index 473e224538..5c776dffa9 100644 --- a/src/main/java/org/opensearch/security/securityconf/ConfigModelV7.java +++ b/src/main/java/org/opensearch/security/securityconf/ConfigModelV7.java @@ -563,6 +563,24 @@ private boolean containsDlsFlsConfig() { return false; } + + @Override + public boolean isPermittedOnSystemIndex(String indexName) { + boolean isPatternMatched = false; + boolean isPermitted = false; + for (SecurityRole role : roles) { + for (IndexPattern ip : role.getIpatterns()) { + WildcardMatcher wildcardMatcher = WildcardMatcher.from(ip.indexPattern); + if (wildcardMatcher.test(indexName)) { + isPatternMatched = true; + } + if (ip.perms.contains(ConfigConstants.SYSTEM_INDEX_PERMISSION)) { + isPermitted = true; + } + } + } + return isPatternMatched && isPermitted; + } } public static class SecurityRole { diff --git a/src/main/java/org/opensearch/security/securityconf/SecurityRoles.java b/src/main/java/org/opensearch/security/securityconf/SecurityRoles.java index 079853d581..fb25e1a21f 100644 --- a/src/main/java/org/opensearch/security/securityconf/SecurityRoles.java +++ b/src/main/java/org/opensearch/security/securityconf/SecurityRoles.java @@ -96,4 +96,6 @@ Set getAllPermittedIndicesForDashboards( ); SecurityRoles filter(Set roles); + + boolean isPermittedOnSystemIndex(String indexName); } diff --git a/src/test/java/org/opensearch/security/securityconf/SecurityRolesPermissionsV6Test.java b/src/test/java/org/opensearch/security/securityconf/SecurityRolesPermissionsV6Test.java index ace182bcda..530db3211b 100644 --- a/src/test/java/org/opensearch/security/securityconf/SecurityRolesPermissionsV6Test.java +++ b/src/test/java/org/opensearch/security/securityconf/SecurityRolesPermissionsV6Test.java @@ -126,6 +126,17 @@ public void hasExplicitIndexPermission() { ); } + @Test + public void isPermittedOnSystemIndex() { + final SecurityRoles securityRoleWithExplicitAccess = configModel.getSecurityRoles() + .filter(ImmutableSet.of("has_system_index_permission")); + Assert.assertTrue(securityRoleWithExplicitAccess.isPermittedOnSystemIndex(TEST_INDEX)); + + final SecurityRoles securityRoleWithStarAccess = configModel.getSecurityRoles() + .filter(ImmutableSet.of("all_access_without_system_index_permission")); + Assert.assertFalse(securityRoleWithStarAccess.isPermittedOnSystemIndex(TEST_INDEX)); + } + static SecurityDynamicConfiguration createRolesConfig() throws IOException { final ObjectNode rolesNode = DefaultObjectMapper.objectMapper.createObjectNode(); NO_EXPLICIT_SYSTEM_INDEX_PERMISSION.forEach(rolesNode::set); diff --git a/src/test/java/org/opensearch/security/system_indices/SystemIndexPermissionEnabledTests.java b/src/test/java/org/opensearch/security/system_indices/SystemIndexPermissionEnabledTests.java index b44b014d17..7db072761f 100644 --- a/src/test/java/org/opensearch/security/system_indices/SystemIndexPermissionEnabledTests.java +++ b/src/test/java/org/opensearch/security/system_indices/SystemIndexPermissionEnabledTests.java @@ -79,7 +79,8 @@ public void testSearchAsNormalUser() throws Exception { if (index.equals(ACCESSIBLE_ONLY_BY_SUPER_ADMIN) || index.equals(SYSTEM_INDEX_WITH_NO_ASSOCIATED_ROLE_PERMISSIONS)) { validateForbiddenResponse(response, "", normalUser); } else { - validateSearchResponse(response, 0); + // got 1 hits because system index permissions are enabled + validateSearchResponse(response, 1); } } From b3bfbd4752ec24a2d33d3f6dedb9f2deb002a585 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Fri, 26 Apr 2024 11:03:59 -0500 Subject: [PATCH 195/204] Use predictable serialization logic for transport headers (#4264) Signed-off-by: Peter Nied --- .../transport/SecuritySSLRequestHandler.java | 3 +- .../security/support/ConfigConstants.java | 2 -- .../security/support/SerializationFormat.java | 35 +++++++++++++++++++ .../transport/SecurityInterceptor.java | 30 +++++++++------- .../security/support/Base64HelperTest.java | 23 ++++++++++++ .../transport/SecurityInterceptorTests.java | 28 ++++++++------- .../SecuritySSLRequestHandlerTests.java | 24 +++++++++++-- 7 files changed, 115 insertions(+), 30 deletions(-) create mode 100644 src/main/java/org/opensearch/security/support/SerializationFormat.java diff --git a/src/main/java/org/opensearch/security/ssl/transport/SecuritySSLRequestHandler.java b/src/main/java/org/opensearch/security/ssl/transport/SecuritySSLRequestHandler.java index 078c822357..7002171595 100644 --- a/src/main/java/org/opensearch/security/ssl/transport/SecuritySSLRequestHandler.java +++ b/src/main/java/org/opensearch/security/ssl/transport/SecuritySSLRequestHandler.java @@ -34,6 +34,7 @@ import org.opensearch.security.ssl.util.ExceptionUtils; import org.opensearch.security.ssl.util.SSLRequestHelper; import org.opensearch.security.support.ConfigConstants; +import org.opensearch.security.support.SerializationFormat; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportChannel; @@ -92,7 +93,7 @@ public final void messageReceived(T request, TransportChannel channel, Task task threadContext.putTransient( ConfigConstants.USE_JDK_SERIALIZATION, - channel.getVersion().before(ConfigConstants.FIRST_CUSTOM_SERIALIZATION_SUPPORTED_OS_VERSION) + SerializationFormat.determineFormat(channel.getVersion()) == SerializationFormat.JDK ); if (SSLRequestHelper.containsBadHeader(threadContext, "_opendistro_security_ssl_")) { diff --git a/src/main/java/org/opensearch/security/support/ConfigConstants.java b/src/main/java/org/opensearch/security/support/ConfigConstants.java index 5169d02d20..9c671a80f9 100644 --- a/src/main/java/org/opensearch/security/support/ConfigConstants.java +++ b/src/main/java/org/opensearch/security/support/ConfigConstants.java @@ -35,7 +35,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.opensearch.Version; import org.opensearch.common.settings.Settings; import org.opensearch.security.auditlog.impl.AuditCategory; @@ -334,7 +333,6 @@ public enum RolesMappingResolution { public static final String TENANCY_GLOBAL_TENANT_DEFAULT_NAME = ""; public static final String USE_JDK_SERIALIZATION = "plugins.security.use_jdk_serialization"; - public static final Version FIRST_CUSTOM_SERIALIZATION_SUPPORTED_OS_VERSION = Version.V_2_11_0; // On-behalf-of endpoints settings // CS-SUPPRESS-SINGLE: RegexpSingleline get Extensions Settings diff --git a/src/main/java/org/opensearch/security/support/SerializationFormat.java b/src/main/java/org/opensearch/security/support/SerializationFormat.java new file mode 100644 index 0000000000..210a5cf6a5 --- /dev/null +++ b/src/main/java/org/opensearch/security/support/SerializationFormat.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.support; + +import org.opensearch.Version; + +public enum SerializationFormat { + /** Uses Java's native serialization system */ + JDK, + /** Uses a custom serializer built ontop of OpenSearch 2.11 */ + CustomSerializer_2_11; + + private static final Version FIRST_CUSTOM_SERIALIZATION_SUPPORTED_OS_VERSION = Version.V_2_11_0; + private static final Version CUSTOM_SERIALIZATION_NO_LONGER_SUPPORTED_OS_VERSION = Version.V_2_14_0; + + /** + * Determines the format of serialization that should be used from a version identifier + */ + public static SerializationFormat determineFormat(final Version version) { + if (version.onOrAfter(FIRST_CUSTOM_SERIALIZATION_SUPPORTED_OS_VERSION) + && version.before(CUSTOM_SERIALIZATION_NO_LONGER_SUPPORTED_OS_VERSION)) { + return SerializationFormat.CustomSerializer_2_11; + } + return SerializationFormat.JDK; + } +} diff --git a/src/main/java/org/opensearch/security/transport/SecurityInterceptor.java b/src/main/java/org/opensearch/security/transport/SecurityInterceptor.java index f791cd013a..f55d9ac338 100644 --- a/src/main/java/org/opensearch/security/transport/SecurityInterceptor.java +++ b/src/main/java/org/opensearch/security/transport/SecurityInterceptor.java @@ -62,6 +62,7 @@ import org.opensearch.security.support.Base64Helper; import org.opensearch.security.support.ConfigConstants; import org.opensearch.security.support.HeaderHelper; +import org.opensearch.security.support.SerializationFormat; import org.opensearch.security.user.User; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transport.Connection; @@ -150,7 +151,8 @@ public void sendRequestDecorate( final String origCCSTransientMf = getThreadContext().getTransient(ConfigConstants.OPENDISTRO_SECURITY_MASKED_FIELD_CCS); final boolean isDebugEnabled = log.isDebugEnabled(); - final boolean useJDKSerialization = connection.getVersion().before(ConfigConstants.FIRST_CUSTOM_SERIALIZATION_SUPPORTED_OS_VERSION); + + final var serializationFormat = SerializationFormat.determineFormat(connection.getVersion()); final boolean isSameNodeRequest = localNode != null && localNode.equals(connection.getNode()); try (ThreadContext.StoredContext stashedContext = getThreadContext().stashContext()) { @@ -228,17 +230,20 @@ && getThreadContext().getHeader(ConfigConstants.OPENDISTRO_SECURITY_INJECTED_ROL ); } - if (useJDKSerialization) { - Map jdkSerializedHeaders = new HashMap<>(); - HeaderHelper.getAllSerializedHeaderNames() - .stream() - .filter(k -> headerMap.get(k) != null) - .forEach(k -> jdkSerializedHeaders.put(k, Base64Helper.ensureJDKSerialized(headerMap.get(k)))); - headerMap.putAll(jdkSerializedHeaders); + try { + if (serializationFormat == SerializationFormat.JDK) { + Map jdkSerializedHeaders = new HashMap<>(); + HeaderHelper.getAllSerializedHeaderNames() + .stream() + .filter(k -> headerMap.get(k) != null) + .forEach(k -> jdkSerializedHeaders.put(k, Base64Helper.ensureJDKSerialized(headerMap.get(k)))); + headerMap.putAll(jdkSerializedHeaders); + } + getThreadContext().putHeader(headerMap); + } catch (IllegalArgumentException iae) { + log.debug("Failed to add headers information onto on thread context", iae); } - getThreadContext().putHeader(headerMap); - ensureCorrectHeaders( remoteAddress0, user0, @@ -246,7 +251,7 @@ && getThreadContext().getHeader(ConfigConstants.OPENDISTRO_SECURITY_INJECTED_ROL injectedUserString, injectedRolesString, isSameNodeRequest, - useJDKSerialization + serializationFormat ); if (actionTraceEnabled.get()) { @@ -275,7 +280,7 @@ private void ensureCorrectHeaders( final String injectedUserString, final String injectedRolesString, final boolean isSameNodeRequest, - final boolean useJDKSerialization + final SerializationFormat format ) { // keep original address @@ -313,6 +318,7 @@ && getThreadContext().getHeader(ConfigConstants.OPENDISTRO_SECURITY_ORIGIN_HEADE getThreadContext().putTransient(ConfigConstants.OPENDISTRO_SECURITY_INJECTED_USER, injectedUserString); } } else { + final var useJDKSerialization = format == SerializationFormat.JDK; if (transportAddress != null) { getThreadContext().putHeader( ConfigConstants.OPENDISTRO_SECURITY_REMOTE_ADDRESS_HEADER, diff --git a/src/test/java/org/opensearch/security/support/Base64HelperTest.java b/src/test/java/org/opensearch/security/support/Base64HelperTest.java index 3bc81aaebc..32d96767d8 100644 --- a/src/test/java/org/opensearch/security/support/Base64HelperTest.java +++ b/src/test/java/org/opensearch/security/support/Base64HelperTest.java @@ -11,12 +11,17 @@ package org.opensearch.security.support; import java.io.Serializable; +import java.util.HashMap; +import java.util.stream.IntStream; import org.junit.Assert; import org.junit.Test; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; import static org.opensearch.security.support.Base64Helper.deserializeObject; import static org.opensearch.security.support.Base64Helper.serializeObject; +import static org.junit.Assert.assertThat; public class Base64HelperTest { @@ -48,4 +53,22 @@ public void testEnsureJDKSerialized() { Assert.assertEquals(jdkSerialized, Base64Helper.ensureJDKSerialized(jdkSerialized)); Assert.assertEquals(jdkSerialized, Base64Helper.ensureJDKSerialized(customSerialized)); } + + @Test + public void testDuplicatedItemSizes() { + var largeObject = new HashMap(); + var hm = new HashMap<>(); + IntStream.range(0, 100).forEach(i -> { hm.put("c" + i, "cvalue" + i); }); + IntStream.range(0, 100).forEach(i -> { largeObject.put("b" + i, hm); }); + + final var jdkSerialized = Base64Helper.serializeObject(largeObject, true); + final var customSerialized = Base64Helper.serializeObject(largeObject, false); + final var customSerializedOnlyHashMap = Base64Helper.serializeObject(hm, false); + + assertThat(jdkSerialized.length(), equalTo(3832)); + // The custom serializer is ~50x larger than the jdk serialized version + assertThat(customSerialized.length(), equalTo(184792)); + // Show that the majority of the size of the custom serialized large object is the map duplicated ~100 times + assertThat((double) customSerializedOnlyHashMap.length(), closeTo(customSerialized.length() / 100, 70d)); + } } diff --git a/src/test/java/org/opensearch/security/transport/SecurityInterceptorTests.java b/src/test/java/org/opensearch/security/transport/SecurityInterceptorTests.java index 4b3636a000..8d902ed498 100644 --- a/src/test/java/org/opensearch/security/transport/SecurityInterceptorTests.java +++ b/src/test/java/org/opensearch/security/transport/SecurityInterceptorTests.java @@ -12,8 +12,9 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.junit.Before; import org.junit.Test; @@ -120,7 +121,7 @@ public class SecurityInterceptorTests { private AsyncSender sender; private AsyncSender serializedSender; - private AsyncSender nullSender; + private AtomicReference senderLatch = new AtomicReference<>(new CountDownLatch(1)); @Before public void setup() { @@ -208,6 +209,7 @@ public void sendRequest( ) { String serializedUserHeader = threadPool.getThreadContext().getHeader(ConfigConstants.OPENDISTRO_SECURITY_USER_HEADER); assertEquals(serializedUserHeader, Base64Helper.serializeObject(user, true)); + senderLatch.get().countDown(); } }; @@ -222,6 +224,7 @@ public void sendRequest( ) { User transientUser = threadPool.getThreadContext().getTransient(ConfigConstants.OPENDISTRO_SECURITY_USER); assertEquals(transientUser, user); + senderLatch.get().countDown(); } }; @@ -249,17 +252,16 @@ final void completableRequestDecorate( TransportResponseHandler handler, DiscoveryNode localNode ) { + securityInterceptor.sendRequestDecorate(sender, connection, action, request, options, handler, localNode); + verifyOriginalContext(user); + try { + senderLatch.get().await(1, TimeUnit.SECONDS); + } catch (final InterruptedException e) { + throw new RuntimeException(e); + } - ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); - - singleThreadExecutor.execute(() -> { - try { - securityInterceptor.sendRequestDecorate(sender, connection, action, request, options, handler, localNode); - verifyOriginalContext(user); - } finally { - singleThreadExecutor.shutdown(); - } - }); + // Reset the latch so another request can be processed + senderLatch.set(new CountDownLatch(1)); } @Test diff --git a/src/test/java/org/opensearch/security/transport/SecuritySSLRequestHandlerTests.java b/src/test/java/org/opensearch/security/transport/SecuritySSLRequestHandlerTests.java index 2d10b6f84f..c63c8d26ae 100644 --- a/src/test/java/org/opensearch/security/transport/SecuritySSLRequestHandlerTests.java +++ b/src/test/java/org/opensearch/security/transport/SecuritySSLRequestHandlerTests.java @@ -94,9 +94,19 @@ public void testUseJDKSerializationHeaderIsSetOnMessageReceived() throws Excepti Assert.assertFalse(threadPool.getThreadContext().getTransient(ConfigConstants.USE_JDK_SERIALIZATION)); threadPool.getThreadContext().stashContext(); - when(transportChannel.getVersion()).thenReturn(Version.V_3_0_0); + when(transportChannel.getVersion()).thenReturn(Version.V_2_13_0); Assert.assertThrows(Exception.class, () -> securitySSLRequestHandler.messageReceived(transportRequest, transportChannel, task)); Assert.assertFalse(threadPool.getThreadContext().getTransient(ConfigConstants.USE_JDK_SERIALIZATION)); + + threadPool.getThreadContext().stashContext(); + when(transportChannel.getVersion()).thenReturn(Version.V_2_14_0); + Assert.assertThrows(Exception.class, () -> securitySSLRequestHandler.messageReceived(transportRequest, transportChannel, task)); + Assert.assertTrue(threadPool.getThreadContext().getTransient(ConfigConstants.USE_JDK_SERIALIZATION)); + + threadPool.getThreadContext().stashContext(); + when(transportChannel.getVersion()).thenReturn(Version.V_3_0_0); + Assert.assertThrows(Exception.class, () -> securitySSLRequestHandler.messageReceived(transportRequest, transportChannel, task)); + Assert.assertTrue(threadPool.getThreadContext().getTransient(ConfigConstants.USE_JDK_SERIALIZATION)); } @Test @@ -118,9 +128,19 @@ public void testUseJDKSerializationHeaderIsSetWithWrapperChannel() throws Except Assert.assertFalse(threadPool.getThreadContext().getTransient(ConfigConstants.USE_JDK_SERIALIZATION)); threadPool.getThreadContext().stashContext(); - when(transportChannel.getVersion()).thenReturn(Version.V_3_0_0); + when(transportChannel.getVersion()).thenReturn(Version.V_2_13_0); Assert.assertThrows(Exception.class, () -> securitySSLRequestHandler.messageReceived(transportRequest, wrappedChannel, task)); Assert.assertFalse(threadPool.getThreadContext().getTransient(ConfigConstants.USE_JDK_SERIALIZATION)); + + threadPool.getThreadContext().stashContext(); + when(transportChannel.getVersion()).thenReturn(Version.V_2_14_0); + Assert.assertThrows(Exception.class, () -> securitySSLRequestHandler.messageReceived(transportRequest, wrappedChannel, task)); + Assert.assertTrue(threadPool.getThreadContext().getTransient(ConfigConstants.USE_JDK_SERIALIZATION)); + + threadPool.getThreadContext().stashContext(); + when(transportChannel.getVersion()).thenReturn(Version.V_3_0_0); + Assert.assertThrows(Exception.class, () -> securitySSLRequestHandler.messageReceived(transportRequest, wrappedChannel, task)); + Assert.assertTrue(threadPool.getThreadContext().getTransient(ConfigConstants.USE_JDK_SERIALIZATION)); } @Test From fe61282ca82c9529b7c49e160b55cb37326a813b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 07:12:32 -0400 Subject: [PATCH 196/204] Bump derek-ho/start-opensearch from 3 to 4 (#4292) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/plugin_install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/plugin_install.yml b/.github/workflows/plugin_install.yml index 6fa8c74beb..8cfcd156ae 100644 --- a/.github/workflows/plugin_install.yml +++ b/.github/workflows/plugin_install.yml @@ -40,7 +40,7 @@ jobs: shell: bash - name: Run Opensearch with A Single Plugin - uses: derek-ho/start-opensearch@v3 + uses: derek-ho/start-opensearch@v4 with: opensearch-version: ${{ env.OPENSEARCH_VERSION }} plugins: "file:$(pwd)/${{ env.PLUGIN_NAME }}.zip" From 14281a20f59916177488bd27a7d13d8564ad6f4a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 07:57:36 -0400 Subject: [PATCH 197/204] Bump com.google.errorprone:error_prone_annotations from 2.26.1 to 2.27.0 (#4294) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index ad0a58313f..a2e3f70378 100644 --- a/build.gradle +++ b/build.gradle @@ -494,7 +494,7 @@ configurations { // For integrationTest force "org.apache.httpcomponents:httpclient:4.5.14" force "org.apache.httpcomponents:httpcore:4.4.16" - force "com.google.errorprone:error_prone_annotations:2.26.1" + force "com.google.errorprone:error_prone_annotations:2.27.0" force "org.checkerframework:checker-qual:3.42.0" force "ch.qos.logback:logback-classic:1.5.6" } @@ -605,7 +605,7 @@ dependencies { runtimeOnly 'com.eclipsesource.minimal-json:minimal-json:0.9.5' runtimeOnly 'commons-codec:commons-codec:1.16.1' runtimeOnly 'org.cryptacular:cryptacular:1.2.6' - compileOnly 'com.google.errorprone:error_prone_annotations:2.26.1' + compileOnly 'com.google.errorprone:error_prone_annotations:2.27.0' runtimeOnly 'com.sun.istack:istack-commons-runtime:4.2.0' runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.2' runtimeOnly 'org.ow2.asm:asm:9.7' From b396a31581918b01eecd2ea3dfa9fc046ee6d5f4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 08:08:23 -0400 Subject: [PATCH 198/204] Bump com.netflix.nebula.ospackage from 11.8.1 to 11.9.0 (#4296) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a2e3f70378..134c9d5386 100644 --- a/build.gradle +++ b/build.gradle @@ -64,7 +64,7 @@ plugins { id 'maven-publish' id 'com.diffplug.spotless' version '6.25.0' id 'checkstyle' - id 'com.netflix.nebula.ospackage' version "11.8.1" + id 'com.netflix.nebula.ospackage' version "11.9.0" id "org.gradle.test-retry" version "1.5.8" id 'eclipse' id "com.github.spotbugs" version "5.2.5" From 7ccc09ec2f61189cc0c5f4dc50a410b7373062b9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 08:58:04 -0400 Subject: [PATCH 199/204] Bump org.gradle.test-retry from 1.5.8 to 1.5.9 (#4295) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 134c9d5386..70bf7ff601 100644 --- a/build.gradle +++ b/build.gradle @@ -65,7 +65,7 @@ plugins { id 'com.diffplug.spotless' version '6.25.0' id 'checkstyle' id 'com.netflix.nebula.ospackage' version "11.9.0" - id "org.gradle.test-retry" version "1.5.8" + id "org.gradle.test-retry" version "1.5.9" id 'eclipse' id "com.github.spotbugs" version "5.2.5" id "com.google.osdetector" version "1.7.3" From b525839171954f1bc0568528b386eaab363b8c3c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 09:43:21 -0400 Subject: [PATCH 200/204] Bump commons-codec:commons-codec from 1.16.1 to 1.17.0 (#4293) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 70bf7ff601..4fe975e00b 100644 --- a/build.gradle +++ b/build.gradle @@ -469,7 +469,7 @@ bundlePlugin { configurations { all { resolutionStrategy { - force 'commons-codec:commons-codec:1.16.1' + force 'commons-codec:commons-codec:1.17.0' force 'org.slf4j:slf4j-api:1.7.36' force 'org.scala-lang:scala-library:2.13.13' force "com.fasterxml.jackson:jackson-bom:${versions.jackson}" @@ -603,7 +603,7 @@ dependencies { runtimeOnly 'com.sun.activation:jakarta.activation:1.2.2' runtimeOnly 'com.eclipsesource.minimal-json:minimal-json:0.9.5' - runtimeOnly 'commons-codec:commons-codec:1.16.1' + runtimeOnly 'commons-codec:commons-codec:1.17.0' runtimeOnly 'org.cryptacular:cryptacular:1.2.6' compileOnly 'com.google.errorprone:error_prone_annotations:2.27.0' runtimeOnly 'com.sun.istack:istack-commons-runtime:4.2.0' From 6d35a0af9de48620f9c6d5b2c67b094713ca8c3e Mon Sep 17 00:00:00 2001 From: Andrey Pleskach Date: Mon, 29 Apr 2024 17:48:58 +0200 Subject: [PATCH 201/204] REST API tests refactoring (Part 3) (#4253) Signed-off-by: Andrey Pleskach --- .../rest/api/SecurityHealthActionTest.java | 48 ------------------- .../LegacySecurityHealthActionTests.java | 23 --------- 2 files changed, 71 deletions(-) delete mode 100644 src/test/java/org/opensearch/security/dlic/rest/api/SecurityHealthActionTest.java delete mode 100644 src/test/java/org/opensearch/security/dlic/rest/api/legacy/LegacySecurityHealthActionTests.java diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/SecurityHealthActionTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/SecurityHealthActionTest.java deleted file mode 100644 index e239050612..0000000000 --- a/src/test/java/org/opensearch/security/dlic/rest/api/SecurityHealthActionTest.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.security.dlic.rest.api; - -import org.apache.http.HttpStatus; -import org.junit.Assert; -import org.junit.Test; - -import org.opensearch.common.settings.Settings; -import org.opensearch.security.support.ConfigConstants; -import org.opensearch.security.test.helper.rest.RestHelper; - -import static org.opensearch.security.OpenSearchSecurityPlugin.PLUGINS_PREFIX; - -public class SecurityHealthActionTest extends AbstractRestApiUnitTest { - private final String ENDPOINT; - - protected String getEndpointPrefix() { - return PLUGINS_PREFIX; - } - - public SecurityHealthActionTest() { - ENDPOINT = getEndpointPrefix(); - } - - @Test - public void testSecurityHealthAPI() throws Exception { - Settings settings = Settings.builder() - .put(ConfigConstants.SECURITY_UNSUPPORTED_RESTAPI_ALLOW_SECURITYCONFIG_MODIFICATION, true) - .build(); - setup(settings); - - rh.keystore = "restapi/kirk-keystore.jks"; - rh.sendAdminCertificate = true; - - RestHelper.HttpResponse response = rh.executeGetRequest(ENDPOINT + "/health"); - Assert.assertEquals(HttpStatus.SC_OK, response.getStatusCode()); - } -} diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/legacy/LegacySecurityHealthActionTests.java b/src/test/java/org/opensearch/security/dlic/rest/api/legacy/LegacySecurityHealthActionTests.java deleted file mode 100644 index 99fa4a99ae..0000000000 --- a/src/test/java/org/opensearch/security/dlic/rest/api/legacy/LegacySecurityHealthActionTests.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.security.dlic.rest.api.legacy; - -import org.opensearch.security.dlic.rest.api.SecurityHealthActionTest; - -import static org.opensearch.security.OpenSearchSecurityPlugin.LEGACY_OPENDISTRO_PREFIX; - -public class LegacySecurityHealthActionTests extends SecurityHealthActionTest { - @Override - protected String getEndpointPrefix() { - return LEGACY_OPENDISTRO_PREFIX; - } -} From af358d0e41b9eb28b9cadc6eb7679e83bb083b84 Mon Sep 17 00:00:00 2001 From: Andrey Pleskach Date: Wed, 1 May 2024 13:21:31 +0200 Subject: [PATCH 202/204] Fix spelling (#4305) Signed-off-by: Andrey Pleskach --- .../security/dlic/rest/api/AbstractApiAction.java | 6 +++--- .../security/dlic/rest/api/FlushCacheApiAction.java | 6 +++--- .../security/dlic/rest/api/MigrateApiAction.java | 12 ++++++------ .../opensearch/security/dlic/rest/api/Responses.java | 2 +- .../security/dlic/rest/api/ValidateApiAction.java | 4 ++-- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/AbstractApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/AbstractApiAction.java index 157d44da73..bee0fe5579 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/AbstractApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/AbstractApiAction.java @@ -74,7 +74,7 @@ import static org.opensearch.security.dlic.rest.api.Responses.conflict; import static org.opensearch.security.dlic.rest.api.Responses.forbidden; import static org.opensearch.security.dlic.rest.api.Responses.forbiddenMessage; -import static org.opensearch.security.dlic.rest.api.Responses.internalSeverError; +import static org.opensearch.security.dlic.rest.api.Responses.internalServerError; import static org.opensearch.security.dlic.rest.api.Responses.payload; import static org.opensearch.security.dlic.rest.support.Utils.withIOException; @@ -482,7 +482,7 @@ public final void onFailure(Exception e) { if (ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException) { conflict(channel, e.getMessage()); } else { - internalSeverError(channel, "Error " + e.getMessage()); + internalServerError(channel, "Error " + e.getMessage()); } } @@ -579,7 +579,7 @@ protected final RestChannelConsumer prepareRequest(RestRequest request, NodeClie // check if .opendistro_security index has been initialized if (!ensureIndexExists()) { - return channel -> internalSeverError(channel, RequestContentValidator.ValidationError.SECURITY_NOT_INITIALIZED.message()); + return channel -> internalServerError(channel, RequestContentValidator.ValidationError.SECURITY_NOT_INITIALIZED.message()); } // check if request is authorized diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/FlushCacheApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/FlushCacheApiAction.java index d6f5e24d7d..2f579ecbd9 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/FlushCacheApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/FlushCacheApiAction.java @@ -28,7 +28,7 @@ import org.opensearch.security.securityconf.impl.CType; import org.opensearch.threadpool.ThreadPool; -import static org.opensearch.security.dlic.rest.api.Responses.internalSeverError; +import static org.opensearch.security.dlic.rest.api.Responses.internalServerError; import static org.opensearch.security.dlic.rest.api.Responses.ok; import static org.opensearch.security.dlic.rest.support.Utils.addRoutesPrefix; @@ -73,7 +73,7 @@ private void flushCacheApiRequestHandlers(RequestHandler.RequestHandlersBuilder public void onResponse(ConfigUpdateResponse configUpdateResponse) { if (configUpdateResponse.hasFailures()) { LOGGER.error("Cannot flush cache due to", configUpdateResponse.failures().get(0)); - internalSeverError( + internalServerError( channel, "Cannot flush cache due to " + configUpdateResponse.failures().get(0).getMessage() + "." ); @@ -86,7 +86,7 @@ public void onResponse(ConfigUpdateResponse configUpdateResponse) { @Override public void onFailure(final Exception e) { LOGGER.error("Cannot flush cache due to", e); - internalSeverError(channel, "Cannot flush cache due to " + e.getMessage() + "."); + internalServerError(channel, "Cannot flush cache due to " + e.getMessage() + "."); } } diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/MigrateApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/MigrateApiAction.java index 7f1adecd3e..b66ff0d5f3 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/MigrateApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/MigrateApiAction.java @@ -61,7 +61,7 @@ import org.opensearch.threadpool.ThreadPool; import static org.opensearch.security.dlic.rest.api.Responses.badRequest; -import static org.opensearch.security.dlic.rest.api.Responses.internalSeverError; +import static org.opensearch.security.dlic.rest.api.Responses.internalServerError; import static org.opensearch.security.dlic.rest.api.Responses.ok; import static org.opensearch.security.dlic.rest.support.Utils.addRoutesPrefix; // CS-ENFORCE-SINGLE @@ -209,7 +209,7 @@ public void onResponse(CreateIndexResponse response) { } } catch (final IOException e1) { LOGGER.error("Unable to create bulk request " + e1, e1); - internalSeverError(channel, "Unable to create bulk request."); + internalServerError(channel, "Unable to create bulk request."); return; } @@ -226,7 +226,7 @@ public void onResponse(BulkResponse response) { "Unable to upload migrated configuration because of " + response.buildFailureMessage() ); - internalSeverError( + internalServerError( channel, "Unable to upload migrated configuration (bulk index failed)." ); @@ -240,7 +240,7 @@ public void onResponse(BulkResponse response) { @Override public void onFailure(Exception e) { LOGGER.error("Unable to upload migrated configuration because of " + e, e); - internalSeverError(channel, "Unable to upload migrated configuration."); + internalServerError(channel, "Unable to upload migrated configuration."); } } ) @@ -251,7 +251,7 @@ public void onFailure(Exception e) { @Override public void onFailure(Exception e) { LOGGER.error("Unable to create opendistro_security index because of " + e, e); - internalSeverError(channel, "Unable to create opendistro_security index."); + internalServerError(channel, "Unable to create opendistro_security index."); } }); @@ -263,7 +263,7 @@ public void onFailure(Exception e) { @Override public void onFailure(Exception e) { LOGGER.error("Unable to delete opendistro_security index because of " + e, e); - internalSeverError(channel, "Unable to delete opendistro_security index."); + internalServerError(channel, "Unable to delete opendistro_security index."); } }); diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/Responses.java b/src/main/java/org/opensearch/security/dlic/rest/api/Responses.java index 4f895d1a91..f0d90af6a0 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/Responses.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/Responses.java @@ -50,7 +50,7 @@ public static void conflict(final RestChannel channel, final String message) { response(channel, RestStatus.CONFLICT, message); } - public static void internalSeverError(final RestChannel channel, final String message) { + public static void internalServerError(final RestChannel channel, final String message) { response(channel, RestStatus.INTERNAL_SERVER_ERROR, message); } diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/ValidateApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/ValidateApiAction.java index 93f1cd35c3..1d56ed80f9 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/ValidateApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/ValidateApiAction.java @@ -40,7 +40,7 @@ import org.opensearch.threadpool.ThreadPool; import static org.opensearch.security.dlic.rest.api.Responses.badRequest; -import static org.opensearch.security.dlic.rest.api.Responses.internalSeverError; +import static org.opensearch.security.dlic.rest.api.Responses.internalServerError; import static org.opensearch.security.dlic.rest.api.Responses.ok; import static org.opensearch.security.dlic.rest.support.Utils.addRoutesPrefix; @@ -125,7 +125,7 @@ protected void validate(RestChannel channel, RestRequest request) throws IOExcep ok(channel, "OK."); } catch (Exception e) { - internalSeverError(channel, "Configuration is not valid."); + internalServerError(channel, "Configuration is not valid."); } } From 753e318013a252211eddf6b9112e0e821a9ae74d Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 1 May 2024 09:55:01 -0400 Subject: [PATCH 203/204] Release notes for 2.14 (#4307) Signed-off-by: Peter Nied Signed-off-by: Craig Perkins Co-authored-by: Peter Nied --- ...nsearch-security.release-notes-2.14.0.0.md | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 release-notes/opensearch-security.release-notes-2.14.0.0.md diff --git a/release-notes/opensearch-security.release-notes-2.14.0.0.md b/release-notes/opensearch-security.release-notes-2.14.0.0.md new file mode 100644 index 0000000000..2d089866f9 --- /dev/null +++ b/release-notes/opensearch-security.release-notes-2.14.0.0.md @@ -0,0 +1,39 @@ +## Version 2.14.0.0 + +Compatible with OpenSearch 2.14.0 + +### Enhancements +* Check for and perform upgrades on security configurations ([#4251](https://github.com/opensearch-project/security/pull/4251)) +* Replace bouncy castle blake2b ([#4284](https://github.com/opensearch-project/security/pull/4284)) +* Adds saml auth header to differentiate saml requests and prevents auto login as anonymous user when basic authentication fails ([#4228](https://github.com/opensearch-project/security/pull/4228)) +* Dynamic sign in options ([#4137](https://github.com/opensearch-project/security/pull/4137)) +* Add index permissions for query insights exporters ([#4231](https://github.com/opensearch-project/security/pull/4231)) +* Add new stop words system index ([#4181](https://github.com/opensearch-project/security/pull/4181)) +* Switch to built-in security transports from core ([#4119](https://github.com/opensearch-project/security/pull/4119)) ([#4174](https://github.com/opensearch-project/security/pull/4174)) ([#4187](https://github.com/opensearch-project/security/pull/4187)) +* System index permission grants reading access to documents in the index ([#4291](https://github.com/opensearch-project/security/pull/4291)) +* Improve cluster initialization reliability ([#4002](https://github.com/opensearch-project/security/pull/4002)) ([#4256](https://github.com/opensearch-project/security/pull/4256)) + +### Bug Fixes +* Ensure that challenge response contains body ([#4268](https://github.com/opensearch-project/security/pull/4268)) +* Add logging for audit log that are unable to saving the request body ([#4272](https://github.com/opensearch-project/security/pull/4272)) +* Use predictable serialization logic for transport headers ([#4288](https://github.com/opensearch-project/security/pull/4288)) +* Update Log4JSink Default from sgaudit to audit and add test for default values ([#4155](https://github.com/opensearch-project/security/pull/4155)) +* Remove Pom task dependencies rewrite ([#4178](https://github.com/opensearch-project/security/pull/4178)) ([#4186](https://github.com/opensearch-project/security/pull/4186)) +* Misc changes for tests ([#4184](https://github.com/opensearch-project/security/pull/4184)) +* Add simple roles mapping integ test to test mapping of backend role to role ([#4176](https://github.com/opensearch-project/security/pull/4176)) + +### Maintenance +* Add getProperty.org.bouncycastle.ec.max_f2m_field_size to plugin-security.policy ([#4270](https://github.com/opensearch-project/security/pull/4270)) +* Add getProperty.org.bouncycastle.pkcs12.default to plugin-security.policy ([#4266](https://github.com/opensearch-project/security/pull/4266)) +* Bump apache_cxf_version from 4.0.3 to 4.0.4 ([#4287](https://github.com/opensearch-project/security/pull/4287)) +* Bump ch.qos.logback:logback-classic from 1.5.3 to 1.5.5 ([#4248](https://github.com/opensearch-project/security/pull/4248)) +* Bump codecov/codecov-action from v3 to v4 ([#4237](https://github.com/opensearch-project/security/pull/4237)) +* Bump com.fasterxml.woodstox:woodstox-core from 6.6.1 to 6.6.2 ([#4195](https://github.com/opensearch-project/security/pull/4195)) +* Bump com.google.googlejavaformat:google-java-format from 1.21.0 to 1.22.0 ([#4220](https://github.com/opensearch-project/security/pull/4220)) +* Bump commons-io:commons-io from 2.15.1 to 2.16.1 ([#4196](https://github.com/opensearch-project/security/pull/4196)) ([#4246](https://github.com/opensearch-project/security/pull/4246)) +* Bump com.nulab-inc:zxcvbn from 1.8.2 to 1.9.0 ([#4219](https://github.com/opensearch-project/security/pull/4219)) +* Bump io.dropwizard.metrics:metrics-core from 4.2.15 to 4.2.25 ([#4193](https://github.com/opensearch-project/security/pull/4193)) ([#4197](https://github.com/opensearch-project/security/pull/4197)) +* Bump net.shibboleth.utilities:java-support from 8.4.1 to 8.4.2 ([#4245](https://github.com/opensearch-project/security/pull/4245)) +* Bump spring_version from 5.3.33 to 5.3.34 ([#4250](https://github.com/opensearch-project/security/pull/4250)) +* Bump Wandalen/wretry.action from 1.4.10 to 3.3.0 ([#4167](https://github.com/opensearch-project/security/pull/4167)) ([#4198](https://github.com/opensearch-project/security/pull/4198)) ([#4221](https://github.com/opensearch-project/security/pull/4221)) ([#4247](https://github.com/opensearch-project/security/pull/4247)) +* Bump open_saml_version from 4.3.0 to 4.3.2 ([#4303](https://github.com/opensearch-project/security/pull/4303)) ([#4239](https://github.com/opensearch-project/security/pull/4239)) From f130915daa969643c3996e0500e2bdbf3499c674 Mon Sep 17 00:00:00 2001 From: Andrey Pleskach Date: Wed, 1 May 2024 16:19:24 +0200 Subject: [PATCH 204/204] REST API tests refactoring (Part 2) (#4252) Signed-off-by: Andrey Pleskach --- .../api/SslCertsRestApiIntegrationTest.java | 79 ++++++++ .../certificate/TestCertificates.java | 32 ++-- .../test/framework/cluster/LocalCluster.java | 3 +- .../cluster/LocalOpenSearchCluster.java | 14 +- .../ssl/OpenSearchSecuritySSLPlugin.java | 4 - .../security/ssl/SecurityKeyStore.java | 22 +-- .../dlic/rest/api/SslCertsApiTest.java | 179 ------------------ .../api/legacy/LegacySslCertsApiTest.java | 29 --- 8 files changed, 119 insertions(+), 243 deletions(-) create mode 100644 src/integrationTest/java/org/opensearch/security/api/SslCertsRestApiIntegrationTest.java delete mode 100644 src/test/java/org/opensearch/security/dlic/rest/api/SslCertsApiTest.java delete mode 100644 src/test/java/org/opensearch/security/dlic/rest/api/legacy/LegacySslCertsApiTest.java diff --git a/src/integrationTest/java/org/opensearch/security/api/SslCertsRestApiIntegrationTest.java b/src/integrationTest/java/org/opensearch/security/api/SslCertsRestApiIntegrationTest.java new file mode 100644 index 0000000000..61085d3f8a --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/api/SslCertsRestApiIntegrationTest.java @@ -0,0 +1,79 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ +package org.opensearch.security.api; + +import com.fasterxml.jackson.databind.JsonNode; +import org.junit.Test; + +import org.opensearch.security.dlic.rest.api.Endpoint; +import org.opensearch.test.framework.cluster.TestRestClient; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.opensearch.security.dlic.rest.api.RestApiAdminPrivilegesEvaluator.CERTS_INFO_ACTION; +import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_ADMIN_ENABLED; + +public class SslCertsRestApiIntegrationTest extends AbstractApiIntegrationTest { + + final static String REST_API_ADMIN_SSL_INFO = "rest-api-admin-ssl-info"; + + static { + clusterSettings.put(SECURITY_RESTAPI_ADMIN_ENABLED, true); + testSecurityConfig.withRestAdminUser(REST_ADMIN_USER, allRestAdminPermissions()) + .withRestAdminUser(REST_API_ADMIN_SSL_INFO, restAdminPermission(Endpoint.SSL, CERTS_INFO_ACTION)); + } + + protected String sslCertsPath() { + return super.apiPath("ssl", "certs"); + } + + @Test + public void certsInfoForbiddenForRegularUser() throws Exception { + withUser(NEW_USER, client -> forbidden(() -> client.get(sslCertsPath()))); + } + + @Test + public void certsInfoForbiddenForAdminUser() throws Exception { + withUser(NEW_USER, client -> forbidden(() -> client.get(sslCertsPath()))); + } + + @Test + public void certsInfoAvailableForTlsAdmin() throws Exception { + withUser(ADMIN_USER_NAME, localCluster.getAdminCertificate(), this::verifySSLCertsInfo); + } + + @Test + public void certsInfoAvailableForRestAdmin() throws Exception { + withUser(REST_ADMIN_USER, this::verifySSLCertsInfo); + withUser(REST_API_ADMIN_SSL_INFO, this::verifySSLCertsInfo); + } + + private void verifySSLCertsInfo(final TestRestClient client) throws Exception { + final var response = ok(() -> client.get(sslCertsPath())); + + final var body = response.bodyAsJsonNode(); + assertThat(response.getBody(), body.has("http_certificates_list")); + assertThat(response.getBody(), body.get("http_certificates_list").isArray()); + verifyCertsJson(body.get("http_certificates_list").get(0)); + assertThat(response.getBody(), body.has("transport_certificates_list")); + assertThat(response.getBody(), body.get("transport_certificates_list").isArray()); + verifyCertsJson(body.get("transport_certificates_list").get(0)); + } + + private void verifyCertsJson(final JsonNode jsonNode) { + assertThat(jsonNode.toPrettyString(), jsonNode.has("issuer_dn")); + assertThat(jsonNode.toPrettyString(), jsonNode.has("subject_dn")); + assertThat(jsonNode.toPrettyString(), jsonNode.get("subject_dn").asText().matches(".*node-\\d.example.com+")); + assertThat(jsonNode.toPrettyString(), jsonNode.get("san").asText().matches(".*node-\\d.example.com.*")); + assertThat(jsonNode.toPrettyString(), jsonNode.has("not_before")); + assertThat(jsonNode.toPrettyString(), jsonNode.has("not_after")); + } + +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java b/src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java index 2dd1dd5eea..f5a936ce7b 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java +++ b/src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java @@ -55,9 +55,13 @@ public class TestCertificates { private static final Logger log = LogManager.getLogger(TestCertificates.class); - public static final Integer MAX_NUMBER_OF_NODE_CERTIFICATES = 3; + public static final Integer DEFAULT_NUMBER_OF_NODE_CERTIFICATES = 3; + + public static final String CA_SUBJECT = "DC=com,DC=example,O=Example Com Inc.,OU=Example Com Inc. Root CA,CN=Example Com Inc. Root CA"; + + public static final String LDAP_SUBJECT = "DC=de,L=test,O=node,OU=node,CN=ldap.example.com"; + public static final String NODE_SUBJECT_PATTERN = "DC=de,L=test,O=node,OU=node,CN=node-%d.example.com"; - private static final String CA_SUBJECT = "DC=com,DC=example,O=Example Com Inc.,OU=Example Com Inc. Root CA,CN=Example Com Inc. Root CA"; private static final String ADMIN_DN = "CN=kirk,OU=client,O=client,L=test,C=de"; private static final int CERTIFICATE_VALIDITY_DAYS = 365; private static final String CERTIFICATE_FILE_EXT = ".cert"; @@ -66,13 +70,18 @@ public class TestCertificates { private final CertificateData adminCertificate; private final List nodeCertificates; + private final int numberOfNodes; + private final CertificateData ldapCertificate; public TestCertificates() { + this(DEFAULT_NUMBER_OF_NODE_CERTIFICATES); + } + + public TestCertificates(final int numberOfNodes) { this.caCertificate = createCaCertificate(); - this.nodeCertificates = IntStream.range(0, MAX_NUMBER_OF_NODE_CERTIFICATES) - .mapToObj(this::createNodeCertificate) - .collect(Collectors.toList()); + this.numberOfNodes = numberOfNodes; + this.nodeCertificates = IntStream.range(0, this.numberOfNodes).mapToObj(this::createNodeCertificate).collect(Collectors.toList()); this.ldapCertificate = createLdapCertificate(); this.adminCertificate = createAdminCertificate(ADMIN_DN); log.info("Test certificates successfully generated"); @@ -109,7 +118,7 @@ public CertificateData getRootCertificateData() { /** * Certificate for Open Search node. The certificate is derived from root certificate, returned by method {@link #getRootCertificate()} - * @param node is a node index. It has to be less than {@link #MAX_NUMBER_OF_NODE_CERTIFICATES} + * @param node is a node index. It has to be less than {@link #DEFAULT_NUMBER_OF_NODE_CERTIFICATES} * @return file which contains certificate in PEM format, defined by RFC 1421 */ public File getNodeCertificate(int node) { @@ -123,18 +132,18 @@ public CertificateData getNodeCertificateData(int node) { } private void isCorrectNodeNumber(int node) { - if (node >= MAX_NUMBER_OF_NODE_CERTIFICATES) { + if (node >= numberOfNodes) { String message = String.format( "Cannot get certificate for node %d, number of created certificates for nodes is %d", node, - MAX_NUMBER_OF_NODE_CERTIFICATES + numberOfNodes ); throw new RuntimeException(message); } } private CertificateData createNodeCertificate(Integer node) { - String subject = String.format("DC=de,L=test,O=node,OU=node,CN=node-%d.example.com", node); + final var subject = String.format(NODE_SUBJECT_PATTERN, node); String domain = String.format("node-%d.example.com", node); CertificateMetadata metadata = CertificateMetadata.basicMetadata(subject, CERTIFICATE_VALIDITY_DAYS) .withKeyUsage(false, DIGITAL_SIGNATURE, NON_REPUDIATION, KEY_ENCIPHERMENT, CLIENT_AUTH, SERVER_AUTH) @@ -150,8 +159,7 @@ public CertificateData issueUserCertificate(String organizationUnit, String user } private CertificateData createLdapCertificate() { - String subject = "DC=de,L=test,O=node,OU=node,CN=ldap.example.com"; - CertificateMetadata metadata = CertificateMetadata.basicMetadata(subject, CERTIFICATE_VALIDITY_DAYS) + CertificateMetadata metadata = CertificateMetadata.basicMetadata(LDAP_SUBJECT, CERTIFICATE_VALIDITY_DAYS) .withKeyUsage(false, DIGITAL_SIGNATURE, NON_REPUDIATION, KEY_ENCIPHERMENT, CLIENT_AUTH, SERVER_AUTH) .withSubjectAlternativeName(null, List.of("localhost"), "127.0.0.1"); return CertificatesIssuerFactory.rsaBaseCertificateIssuer().issueSignedCertificate(metadata, caCertificate); @@ -164,7 +172,7 @@ public CertificateData getLdapCertificateData() { /** * It returns private key associated with node certificate returned by method {@link #getNodeCertificate(int)} * - * @param node is a node index. It has to be less than {@link #MAX_NUMBER_OF_NODE_CERTIFICATES} + * @param node is a node index. It has to be less than {@link #DEFAULT_NUMBER_OF_NODE_CERTIFICATES} * @param privateKeyPassword is a password used to encode private key, can be null to retrieve unencrypted key. * @return file which contains private key encoded in PEM format, defined * by RFC 1421 diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index 135f1fb481..894bb5baa9 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -141,7 +141,6 @@ public void before() { } for (Map.Entry entry : remotes.entrySet()) { - @SuppressWarnings("resource") InetSocketAddress transportAddress = entry.getValue().localOpenSearchCluster.clusterManagerNode().getTransportAddress(); String key = "cluster.remote." + entry.getKey() + ".seeds"; String value = transportAddress.getHostString() + ":" + transportAddress.getPort(); @@ -509,7 +508,7 @@ public Builder defaultConfigurationInitDirectory(String defaultConfigurationInit public LocalCluster build() { try { if (testCertificates == null) { - testCertificates = new TestCertificates(); + testCertificates = new TestCertificates(clusterManager.getNodes()); } clusterName += "_" + num.incrementAndGet(); Settings settings = nodeOverrideSettingsBuilder.build(); diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java index 5e9fd75326..8a14daeb2d 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -105,6 +105,8 @@ public class LocalOpenSearchCluster { private File snapshotDir; + private int nodeCounter = 0; + public LocalOpenSearchCluster( String clusterName, ClusterManager clusterManager, @@ -163,7 +165,6 @@ public void start() throws Exception { this.initialClusterManagerHosts = toHostList(clusterManagerPorts); started = true; - CompletableFuture clusterManagerNodeFuture = startNodes( clusterManager.getClusterManagerNodeSettings(), clusterManagerNodeTransportPorts, @@ -195,7 +196,6 @@ public void start() throws Exception { log.info("Startup finished. Waiting for GREEN"); waitForCluster(ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(10), nodes.size()); - log.info("Started: {}", this); } @@ -303,10 +303,10 @@ private CompletableFuture startNodes( List> futures = new ArrayList<>(); for (NodeSettings nodeSettings : nodeSettingList) { - Node node = new Node(nodeSettings, transportPortIterator.next(), httpPortIterator.next()); + Node node = new Node(nodeCounter, nodeSettings, transportPortIterator.next(), httpPortIterator.next()); futures.add(node.start()); + nodeCounter += 1; } - return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); } @@ -386,8 +386,10 @@ public class Node implements OpenSearchClientProvider { private PluginAwareNode node; private boolean running = false; private boolean portCollision = false; + private final int nodeNumber; - Node(NodeSettings nodeSettings, int transportPort, int httpPort) { + Node(int nodeNumber, NodeSettings nodeSettings, int transportPort, int httpPort) { + this.nodeNumber = nodeNumber; this.nodeName = createNextNodeName(requireNonNull(nodeSettings, "Node settings are required.")); this.nodeSettings = nodeSettings; this.nodeHomeDir = new File(clusterHomeDir, nodeName); @@ -517,7 +519,7 @@ private Settings getOpenSearchSettings() { if (nodeSettingsSupplier != null) { // TODO node number - return Settings.builder().put(settings).put(nodeSettingsSupplier.get(0)).build(); + return Settings.builder().put(settings).put(nodeSettingsSupplier.get(nodeNumber)).build(); } return settings; } diff --git a/src/main/java/org/opensearch/security/ssl/OpenSearchSecuritySSLPlugin.java b/src/main/java/org/opensearch/security/ssl/OpenSearchSecuritySSLPlugin.java index 073193e9d4..e6a1b47888 100644 --- a/src/main/java/org/opensearch/security/ssl/OpenSearchSecuritySSLPlugin.java +++ b/src/main/java/org/opensearch/security/ssl/OpenSearchSecuritySSLPlugin.java @@ -134,10 +134,6 @@ public class OpenSearchSecuritySSLPlugin extends Plugin implements SystemIndexPl protected final SSLConfig SSLConfig; protected volatile ThreadPool threadPool; - // public OpenSearchSecuritySSLPlugin(final Settings settings, final Path configPath) { - // this(settings, configPath, false); - // } - @SuppressWarnings("removal") protected OpenSearchSecuritySSLPlugin(final Settings settings, final Path configPath, boolean disabled) { diff --git a/src/main/java/org/opensearch/security/ssl/SecurityKeyStore.java b/src/main/java/org/opensearch/security/ssl/SecurityKeyStore.java index 03b5df2100..29083d6d6b 100644 --- a/src/main/java/org/opensearch/security/ssl/SecurityKeyStore.java +++ b/src/main/java/org/opensearch/security/ssl/SecurityKeyStore.java @@ -23,25 +23,25 @@ public interface SecurityKeyStore { - public SSLEngine createHTTPSSLEngine() throws SSLException; + SSLEngine createHTTPSSLEngine() throws SSLException; - public SSLEngine createServerTransportSSLEngine() throws SSLException; + SSLEngine createServerTransportSSLEngine() throws SSLException; - public SSLEngine createClientTransportSSLEngine(String peerHost, int peerPort) throws SSLException; + SSLEngine createClientTransportSSLEngine(String peerHost, int peerPort) throws SSLException; - public String getHTTPProviderName(); + String getHTTPProviderName(); - public String getTransportServerProviderName(); + String getTransportServerProviderName(); - public String getTransportClientProviderName(); + String getTransportClientProviderName(); - public String getSubjectAlternativeNames(X509Certificate cert); + String getSubjectAlternativeNames(X509Certificate cert); - public void initHttpSSLConfig(); + void initHttpSSLConfig(); - public void initTransportSSLConfig(); + void initTransportSSLConfig(); - public X509Certificate[] getTransportCerts(); + X509Certificate[] getTransportCerts(); - public X509Certificate[] getHttpCerts(); + X509Certificate[] getHttpCerts(); } diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/SslCertsApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/SslCertsApiTest.java deleted file mode 100644 index 8617555925..0000000000 --- a/src/test/java/org/opensearch/security/dlic/rest/api/SslCertsApiTest.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.security.dlic.rest.api; - -import java.util.List; -import java.util.Map; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.fasterxml.jackson.core.JsonProcessingException; -import org.apache.hc.core5.http.Header; -import org.apache.http.HttpStatus; -import org.junit.Assert; -import org.junit.Test; - -import org.opensearch.common.settings.Settings; -import org.opensearch.security.DefaultObjectMapper; -import org.opensearch.security.support.ConfigConstants; -import org.opensearch.security.test.helper.rest.RestHelper.HttpResponse; - -import static org.opensearch.security.OpenSearchSecurityPlugin.PLUGINS_PREFIX; -import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_ADMIN_ENABLED; - -public class SslCertsApiTest extends AbstractRestApiUnitTest { - - static final String HTTP_CERTS = "http"; - - static final String TRANSPORT_CERTS = "transport"; - - private final static List> EXPECTED_CERTIFICATES = ImmutableList.of( - ImmutableMap.of( - "issuer_dn", - "CN=Example Com Inc. Signing CA,OU=Example Com Inc. Signing CA,O=Example Com Inc.,DC=example,DC=com", - "subject_dn", - "CN=node-0.example.com,OU=SSL,O=Test,L=Test,C=DE", - "san", - "[[2, node-0.example.com], [2, localhost], [7, 127.0.0.1], [8, 1.2.3.4.5.5]]", - "not_before", - "2018-05-05T14:37:09Z", - "not_after", - "2028-05-02T14:37:09Z" - ), - ImmutableMap.of( - "issuer_dn", - "CN=Example Com Inc. Root CA,OU=Example Com Inc. Root CA,O=Example Com Inc.,DC=example,DC=com", - "subject_dn", - "CN=Example Com Inc. Signing CA,OU=Example Com Inc. Signing CA,O=Example Com Inc.,DC=example,DC=com", - "san", - "", - "not_before", - "2018-05-05T14:37:08Z", - "not_after", - "2028-05-04T14:37:08Z" - ) - ); - - private final static String EXPECTED_CERTIFICATES_BY_TYPE; - static { - try { - EXPECTED_CERTIFICATES_BY_TYPE = DefaultObjectMapper.objectMapper.writeValueAsString( - ImmutableMap.of("http_certificates_list", EXPECTED_CERTIFICATES, "transport_certificates_list", EXPECTED_CERTIFICATES) - ); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - private final Header restApiAdminHeader = encodeBasicHeader("rest_api_admin_user", "rest_api_admin_user"); - private final Header restApiCertsInfoAdminHeader = encodeBasicHeader("rest_api_admin_ssl_info", "rest_api_admin_ssl_info"); - - private final Header restApiReloadCertsAdminHeader = encodeBasicHeader( - "rest_api_admin_ssl_reloadcerts", - "rest_api_admin_ssl_reloadcerts" - ); - - private final Header restApiHeader = encodeBasicHeader("test", "test"); - - public String certsInfoEndpoint() { - return PLUGINS_PREFIX + "/api/ssl/certs"; - } - - public String certsReloadEndpoint(final String certType) { - return String.format("%s/api/ssl/%s/reloadcerts", PLUGINS_PREFIX, certType); - } - - private void verifyHasNoAccess() throws Exception { - final Header adminCredsHeader = encodeBasicHeader("admin", "admin"); - // No creds, no admin certificate - UNAUTHORIZED - rh.sendAdminCertificate = false; - HttpResponse response = rh.executeGetRequest(certsInfoEndpoint()); - Assert.assertEquals(response.getBody(), HttpStatus.SC_UNAUTHORIZED, response.getStatusCode()); - - rh.sendAdminCertificate = false; - response = rh.executeGetRequest(certsInfoEndpoint(), adminCredsHeader); - Assert.assertEquals(response.getBody(), HttpStatus.SC_FORBIDDEN, response.getStatusCode()); - - response = rh.executeGetRequest(certsInfoEndpoint(), restApiHeader); - Assert.assertEquals(response.getBody(), HttpStatus.SC_FORBIDDEN, response.getStatusCode()); - } - - @Test - public void testCertsInfo() throws Exception { - setup(); - verifyHasNoAccess(); - sendAdminCert(); - HttpResponse response = rh.executeGetRequest(certsInfoEndpoint()); - Assert.assertEquals(response.getBody(), HttpStatus.SC_OK, response.getStatusCode()); - Assert.assertEquals(EXPECTED_CERTIFICATES_BY_TYPE, response.getBody()); - - } - - @Test - public void testCertsInfoRestAdmin() throws Exception { - setupWithRestRoles(Settings.builder().put(SECURITY_RESTAPI_ADMIN_ENABLED, true).build()); - verifyHasNoAccess(); - rh.sendAdminCertificate = false; - Assert.assertEquals(EXPECTED_CERTIFICATES_BY_TYPE, loadCerts(restApiAdminHeader)); - Assert.assertEquals(EXPECTED_CERTIFICATES_BY_TYPE, loadCerts(restApiCertsInfoAdminHeader)); - } - - private String loadCerts(final Header... header) throws Exception { - HttpResponse response = rh.executeGetRequest(certsInfoEndpoint(), restApiAdminHeader); - Assert.assertEquals(response.getBody(), HttpStatus.SC_OK, response.getStatusCode()); - return response.getBody(); - } - - @Test - public void testReloadCertsNotAvailableByDefault() throws Exception { - setupWithRestRoles(); - - sendAdminCert(); - verifyReloadCertsNotAvailable(HttpStatus.SC_BAD_REQUEST); - - rh.sendAdminCertificate = false; - verifyReloadCertsNotAvailable(HttpStatus.SC_FORBIDDEN, restApiAdminHeader); - verifyReloadCertsNotAvailable(HttpStatus.SC_FORBIDDEN, restApiReloadCertsAdminHeader); - } - - private void verifyReloadCertsNotAvailable(final int expectedStatus, final Header... header) { - HttpResponse response = rh.executePutRequest(certsReloadEndpoint(HTTP_CERTS), "{}", header); - Assert.assertEquals(response.getBody(), expectedStatus, response.getStatusCode()); - response = rh.executePutRequest(certsReloadEndpoint(TRANSPORT_CERTS), "{}", header); - Assert.assertEquals(response.getBody(), expectedStatus, response.getStatusCode()); - } - - @Test - public void testReloadCertsWrongCertsType() throws Exception { - setupWithRestRoles(reloadEnabled()); - sendAdminCert(); - HttpResponse response = rh.executePutRequest(certsReloadEndpoint("aaaaa"), "{}"); - Assert.assertEquals(response.getBody(), HttpStatus.SC_FORBIDDEN, response.getStatusCode()); - - rh.sendAdminCertificate = false; - response = rh.executePutRequest(certsReloadEndpoint("bbbb"), "{}", restApiAdminHeader); - Assert.assertEquals(response.getBody(), HttpStatus.SC_FORBIDDEN, response.getStatusCode()); - response = rh.executePutRequest(certsReloadEndpoint("cccc"), "{}", restApiReloadCertsAdminHeader); - Assert.assertEquals(response.getBody(), HttpStatus.SC_FORBIDDEN, response.getStatusCode()); - - } - - private void sendAdminCert() { - rh.keystore = "restapi/kirk-keystore.jks"; - rh.sendAdminCertificate = true; - } - - Settings reloadEnabled() { - return Settings.builder().put(ConfigConstants.SECURITY_SSL_CERT_RELOAD_ENABLED, true).build(); - } - -} diff --git a/src/test/java/org/opensearch/security/dlic/rest/api/legacy/LegacySslCertsApiTest.java b/src/test/java/org/opensearch/security/dlic/rest/api/legacy/LegacySslCertsApiTest.java deleted file mode 100644 index 5d1c3ae538..0000000000 --- a/src/test/java/org/opensearch/security/dlic/rest/api/legacy/LegacySslCertsApiTest.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.security.dlic.rest.api.legacy; - -import org.opensearch.security.dlic.rest.api.SslCertsApiTest; - -import static org.opensearch.security.OpenSearchSecurityPlugin.LEGACY_OPENDISTRO_PREFIX; - -public class LegacySslCertsApiTest extends SslCertsApiTest { - - @Override - public String certsInfoEndpoint() { - return LEGACY_OPENDISTRO_PREFIX + "/api/ssl/certs"; - } - - @Override - public String certsReloadEndpoint(String certType) { - return String.format("%s/api/ssl/%s/reloadcerts", LEGACY_OPENDISTRO_PREFIX, certType); - } -}