From 3181774b4c34dfd40e1d5e330da8c308bcf689fd Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Mon, 8 Jan 2024 15:16:17 -0800 Subject: [PATCH 1/8] Run Integ test in secure opensearch cluster Signed-off-by: Varun Jain --- ...backwards_compatibility_tests_workflow.yml | 4 +- .github/workflows/test_security.yml | 50 +++++++++++++++++++ DEVELOPER_GUIDE.md | 37 ++++++++++++++ 3 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/test_security.yml diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index f0490155c..32a71c5d0 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -24,7 +24,7 @@ jobs: BWC_VERSION_RESTART_UPGRADE: ${{ matrix.bwc_version }} steps: - - name: Checkout NeuralSearch + - name: Checkout neural-search uses: actions/checkout@v1 - name: Setup Java ${{ matrix.java }} @@ -51,7 +51,7 @@ jobs: BWC_VERSION_ROLLING_UPGRADE: ${{ matrix.bwc_version }} steps: - - name: Checkout NeuralSearch + - name: Checkout neural-search uses: actions/checkout@v1 - name: Setup Java ${{ matrix.java }} diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml new file mode 100644 index 000000000..a0eb107a1 --- /dev/null +++ b/.github/workflows/test_security.yml @@ -0,0 +1,50 @@ +name: Test neural-search on Secure Cluster +on: + schedule: + - cron: '0 0 * * *' # every night + push: + branches: + - "*" + - "feature/**" + pull_request: + branches: + - "*" + - "feature/**" + +jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + integ-test-with-security-linux: + strategy: + matrix: + java: [11, 17, 21] + + name: Run Integration Tests on Linux + runs-on: ubuntu-latest + needs: Get-CI-Image-Tag + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + + steps: + - name: Checkout neural-search + uses: actions/checkout@v1 + with: + submodules: true + + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: Run build + # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true" diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 3af2dd659..51355e017 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -131,6 +131,38 @@ curl localhost:9200 "tagline" : "The OpenSearch Project: https://opensearch.org/" } ``` + +Additionally, it is also possible to run a cluster with security enabled: +```shell script +./gradlew run -Dsecurity.enabled=true -Dhttps=true -Duser=admin -Dpassword=admin +``` + +By default, if `-Dsecurity.enabled=true` is passed the following defaults will be used: `https=true`, `user=admin` and +`password=admin`. + +Then, to access the cluster, we can run +```bash +curl https://localhost:9200 --insecure -u admin:admin + +{ + "name" : "integTest-0", + "cluster_name" : "integTest", + "cluster_uuid" : "kLsNk4JDTMyp1yQRqog-3g", + "version" : { + "distribution" : "opensearch", + "number" : "3.0.0-SNAPSHOT", + "build_type" : "tar", + "build_hash" : "9d85e566894ef53e5f2093618b3d455e4d0a04ce", + "build_date" : "2023-10-30T18:34:06.996519Z", + "build_snapshot" : true, + "lucene_version" : "9.8.0", + "minimum_wire_compatibility_version" : "2.12.0", + "minimum_index_compatibility_version" : "2.0.0" + }, + "tagline" : "The OpenSearch Project: https://opensearch.org/" +} +``` + ### Run Multi-node Cluster Locally It can be useful to test and debug on a multi-node cluster. In order to launch a 3 node cluster with the neural-search plugin installed, run the following command: @@ -145,6 +177,11 @@ In order to run the integration tests with a 3 node cluster, run this command: ./gradlew :integTest -PnumNodes=3 ``` +Additionally, to run integration tests with security enabled, run +``` +./gradlew :integTest -Dsecurity.enabled=true -PnumNodes=3 +``` + Integration tests can be run with remote cluster. For that run the following command and replace host/port/cluster name values with ones for the target cluster: ``` From fd8cebe364d4ede7e40b75ad548649ebc9262ed0 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Mon, 8 Jan 2024 15:31:42 -0800 Subject: [PATCH 2/8] Add change in Changelog.md Signed-off-by: Varun Jain --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ae6baab7..9f48d8ba7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fixing multiple issues reported in #497 ([#524](https://github.com/opensearch-project/neural-search/pull/524)) ### Infrastructure - BWC tests for Neural Search ([#515](https://github.com/opensearch-project/neural-search/pull/515)) +- Github action to run integ tests in secure opensearch cluster ([#535](https://github.com/opensearch-project/neural-search/pull/535)) ### Documentation ### Maintenance ### Refactoring From a8f250778a26695502cdc7bff18f77eb8a42badb Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 16 Jan 2024 10:38:47 -0800 Subject: [PATCH 3/8] Adding Configuration to install security plugin Signed-off-by: Varun Jain --- build.gradle | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index dd48066db..93cd8300f 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,9 @@ */ import org.opensearch.gradle.test.RestIntegTestTask +import org.opensearch.gradle.testclusters.OpenSearchCluster +import java.nio.file.Paths import java.util.concurrent.Callable apply plugin: 'java' @@ -118,6 +120,71 @@ buildscript { ext { isSnapshot = "true" == System.getProperty("build.snapshot", "true") + projectSubstitutions = [:] + + configureSecurityPlugin = { OpenSearchCluster cluster -> + configurations.zipArchive.asFileTree.each { + cluster.plugin(provider(new Callable() { + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return it + } + } + } + })) + } + + cluster.getNodes().forEach { node -> + var creds = node.getCredentials() + if (creds.isEmpty()) { + creds.add(Map.of('username', 'admin', 'password', 'admin')) + } else { + creds.get(0).putAll(Map.of('username', 'admin', 'password', 'admin')) + } + } + + // Config below including files are copied from security demo configuration + ['esnode.pem', 'esnode-key.pem', 'root-ca.pem'].forEach { file -> + File local = Paths.get(opensearch_tmp_dir.absolutePath, file).toFile() + download.run { + src "https://raw.githubusercontent.com/opensearch-project/security/main/bwc-test/src/test/resources/security/" + file + dest local + overwrite false + } + cluster.extraConfigFile(file, local) + } + + // This configuration is copied from the security plugins demo install: + // https://github.com/opensearch-project/security/blob/2.11.1.0/tools/install_demo_configuration.sh#L365-L388 + cluster.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem") + cluster.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem") + cluster.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem") + cluster.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false") + cluster.setting("plugins.security.ssl.http.enabled", "true") + cluster.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem") + cluster.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem") + cluster.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem") + cluster.setting("plugins.security.allow_unsafe_democertificates", "true") + cluster.setting("plugins.security.allow_default_init_securityindex", "true") + cluster.setting("plugins.security.unsupported.inject_user.enabled", "true") + + cluster.setting("plugins.security.authcz.admin_dn", "\n- CN=kirk,OU=client,O=client,L=test, C=de") + cluster.setting('plugins.security.restapi.roles_enabled', '["all_access", "security_rest_api_access"]') + cluster.setting('plugins.security.system_indices.enabled', "true") + cluster.setting('plugins.security.system_indices.indices', '[' + + '".plugins-ml-config", ' + + '".plugins-ml-connector", ' + + '".plugins-ml-model-group", ' + + '".plugins-ml-model", ".plugins-ml-task", ' + + '".plugins-ml-conversation-meta", ' + + '".plugins-ml-conversation-interactions", ' + + ']' + ) + cluster.setSecure(true) + } } allprojects { @@ -163,6 +230,7 @@ dependencies { api "org.opensearch:opensearch:${opensearch_version}" zipArchive group: 'org.opensearch.plugin', name:'opensearch-knn', version: "${opensearch_build}" zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-security', version: "${opensearch_build}" compileOnly fileTree(dir: knnJarDirectory, include: '*.jar') api group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}" testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}" @@ -227,9 +295,18 @@ integTest { // allows integration test classes to access test resource from project root path systemProperty('project.root', project.rootDir.absolutePath) - systemProperty "https", System.getProperty("https") - systemProperty "user", System.getProperty("user") - systemProperty "password", System.getProperty("password") + var is_https = System.getProperty("https") + var user = System.getProperty("user") + var password = System.getProperty("password") + if (System.getProperty("security.enabled") != null) { + // If security is enabled, set is_https/user/password defaults + is_https = is_https == null ? "true" : is_https + user = user == null ? "admin" : user + password = password == null ? "admin" : password + } + systemProperty("https", is_https) + systemProperty("user", user) + systemProperty("password", password) doFirst { // Tell the test JVM if the cluster JVM is running under a debugger so that tests can @@ -253,6 +330,12 @@ integTest { testClusters.integTest { testDistribution = "ARCHIVE" + + // Optionally install security + if (System.getProperty("security.enabled") != null) { + configureSecurityPlugin(testClusters.integTest) + } + // Install K-NN/ml-commons plugins on the integTest cluster nodes configurations.zipArchive.asFileTree.each { plugin(provider(new Callable(){ From 61330e737410d877d3f79787e2bbbbb9624b2436 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 16 Jan 2024 10:55:56 -0800 Subject: [PATCH 4/8] Formatting build.gradle Signed-off-by: Varun Jain --- build.gradle | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 93cd8300f..b70fdba2f 100644 --- a/build.gradle +++ b/build.gradle @@ -118,6 +118,9 @@ buildscript { } } +def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile +opensearch_tmp_dir.mkdirs() + ext { isSnapshot = "true" == System.getProperty("build.snapshot", "true") projectSubstitutions = [:] @@ -272,8 +275,6 @@ compileTestFixturesJava { options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) } -def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile -opensearch_tmp_dir.mkdirs() def _numNodes = findProperty('numNodes') as Integer ?: 1 test { From e5d9a34d960ad2649f239c97a2e5c70841297ad2 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 16 Jan 2024 13:13:33 -0800 Subject: [PATCH 5/8] Fixing Bug Signed-off-by: Varun Jain --- build.gradle | 204 +++++++++++++++++++++++++-------------------------- 1 file changed, 101 insertions(+), 103 deletions(-) diff --git a/build.gradle b/build.gradle index b70fdba2f..c4228d8be 100644 --- a/build.gradle +++ b/build.gradle @@ -12,79 +12,6 @@ import org.opensearch.gradle.testclusters.OpenSearchCluster import java.nio.file.Paths import java.util.concurrent.Callable -apply plugin: 'java' -apply plugin: 'java-test-fixtures' -apply plugin: 'idea' -apply plugin: 'opensearch.opensearchplugin' -apply plugin: 'opensearch.pluginzip' -apply plugin: 'jacoco' -apply plugin: "com.diffplug.spotless" -apply plugin: 'io.freefair.lombok' -apply from: 'gradle/formatting.gradle' - -def pluginName = 'opensearch-neural-search' -def pluginDescription = 'A plugin that adds dense neural retrieval into the OpenSearch ecosytem' -def projectPath = 'org.opensearch' -def pathToPlugin = 'neuralsearch.plugin' -def pluginClassName = 'NeuralSearch' - -tasks.register("preparePluginPathDirs") { - mustRunAfter clean - doLast { - def newPath = pathToPlugin.replace(".", "/") - mkdir "src/main/java/org/opensearch/$newPath" - mkdir "src/test/java/org/opensearch/$newPath" - } -} - -publishing { - repositories { - maven { - name = "Snapshots" - url = "https://aws.oss.sonatype.org/content/repositories/snapshots" - credentials { - username "$System.env.SONATYPE_USERNAME" - password "$System.env.SONATYPE_PASSWORD" - } - } - } - publications { - pluginZip(MavenPublication) { publication -> - pom { - name = pluginName - description = pluginDescription - groupId = "org.opensearch.plugin" - licenses { - license { - name = "The Apache License, Version 2.0" - url = "http://www.apache.org/licenses/LICENSE-2.0.txt" - } - } - developers { - developer { - name = "OpenSearch" - url = "https://github.com/opensearch-project/neural-search" - } - } - } - } - } -} -opensearchplugin { - name pluginName - description pluginDescription - classname "${projectPath}.${pathToPlugin}.${pluginClassName}" - licenseFile rootProject.file('LICENSE') - noticeFile rootProject.file('NOTICE') - extendedPlugins = ['opensearch-knn'] -} - -dependencyLicenses.enabled = false -thirdPartyAudit.enabled = false -loggerUsageCheck.enabled = false -// No need to validate pom, as we do not upload to maven/sonatype -validateNebulaPom.enabled = false - buildscript { ext { opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") @@ -118,6 +45,19 @@ buildscript { } } +plugins{ + id "de.undercouch.download" version "5.3.0" +} +apply plugin: 'java' +apply plugin: 'java-test-fixtures' +apply plugin: 'idea' +apply plugin: 'opensearch.opensearchplugin' +apply plugin: 'opensearch.pluginzip' +apply plugin: 'jacoco' +apply plugin: "com.diffplug.spotless" +apply plugin: 'io.freefair.lombok' +apply from: 'gradle/formatting.gradle' + def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile opensearch_tmp_dir.mkdirs() @@ -127,17 +67,19 @@ ext { configureSecurityPlugin = { OpenSearchCluster cluster -> configurations.zipArchive.asFileTree.each { - cluster.plugin(provider(new Callable() { - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - return it + if(it.name.contains("opensearch-security")){ + cluster.plugin(provider(new Callable() { + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return it + } } } - } - })) + })) + } } cluster.getNodes().forEach { node -> @@ -216,17 +158,73 @@ allprojects { } } -repositories { - mavenLocal() - maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } - mavenCentral() - maven { url "https://plugins.gradle.org/m2/" } -} - configurations { zipArchive } +tasks.register("preparePluginPathDirs") { + mustRunAfter clean + doLast { + def newPath = pathToPlugin.replace(".", "/") + mkdir "src/main/java/org/opensearch/$newPath" + mkdir "src/test/java/org/opensearch/$newPath" + } +} + +def pluginName = 'opensearch-neural-search' +def pluginDescription = 'A plugin that adds dense neural retrieval into the OpenSearch ecosytem' +def projectPath = 'org.opensearch' +def pathToPlugin = 'neuralsearch.plugin' +def pluginClassName = 'NeuralSearch' + +publishing { + repositories { + maven { + name = "Snapshots" + url = "https://aws.oss.sonatype.org/content/repositories/snapshots" + credentials { + username "$System.env.SONATYPE_USERNAME" + password "$System.env.SONATYPE_PASSWORD" + } + } + } + publications { + pluginZip(MavenPublication) { publication -> + pom { + name = pluginName + description = pluginDescription + groupId = "org.opensearch.plugin" + licenses { + license { + name = "The Apache License, Version 2.0" + url = "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + } + developers { + developer { + name = "OpenSearch" + url = "https://github.com/opensearch-project/neural-search" + } + } + } + } + } +} +opensearchplugin { + name pluginName + description pluginDescription + classname "${projectPath}.${pathToPlugin}.${pluginClassName}" + licenseFile rootProject.file('LICENSE') + noticeFile rootProject.file('NOTICE') + extendedPlugins = ['opensearch-knn'] +} + +dependencyLicenses.enabled = false +thirdPartyAudit.enabled = false +loggerUsageCheck.enabled = false +// No need to validate pom, as we do not upload to maven/sonatype +validateNebulaPom.enabled = false + def knnJarDirectory = "$buildDir/dependencies/opensearch-knn" dependencies { @@ -295,7 +293,7 @@ integTest { systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath // allows integration test classes to access test resource from project root path systemProperty('project.root', project.rootDir.absolutePath) - + systemProperty 'security.enabled', System.getProperty('security.enabled') var is_https = System.getProperty("https") var user = System.getProperty("user") var password = System.getProperty("password") @@ -333,23 +331,25 @@ testClusters.integTest { testDistribution = "ARCHIVE" // Optionally install security - if (System.getProperty("security.enabled") != null) { + if (System.getProperty("security.enabled") != null && System.getProperty("security.enabled") == "true") { configureSecurityPlugin(testClusters.integTest) } - // Install K-NN/ml-commons plugins on the integTest cluster nodes + // Install K-NN/ml-commons plugins on the integTest cluster nodes except security configurations.zipArchive.asFileTree.each { - plugin(provider(new Callable(){ - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - return it + if(!it.name.contains("opensearch-security")) { + plugin(provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return it + } } } - } - })) + })) + } } // This installs our neural-search plugin into the testClusters @@ -393,7 +393,6 @@ task integTestRemote(type: RestIntegTestTask) { } } - run { useCluster testClusters.integTest } @@ -410,7 +409,6 @@ check.dependsOn spotlessCheck check.dependsOn jacocoTestCoverageVerification jacocoTestCoverageVerification.dependsOn jacocoTestReport - // updateVersion: Task to auto update version to the next development iteration task updateVersion { onlyIf { System.getProperty('newVersion') } From c22c46d40292528a6d342926cbe1edd34c6a0d53 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 16 Jan 2024 13:40:08 -0800 Subject: [PATCH 6/8] Update .github/workflows/test_security.yml Co-authored-by: Owais Kazi Signed-off-by: Varun Jain --- .github/workflows/test_security.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index a0eb107a1..5b7f9d210 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -43,7 +43,7 @@ jobs: with: java-version: ${{ matrix.java }} - - name: Run build + - name: Run tests # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. run: | chown -R 1000:1000 `pwd` From 3fd415addbb150008c2d78051733ce4ad841c4c9 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 16 Jan 2024 14:21:57 -0800 Subject: [PATCH 7/8] Update DEVELOPER_GUIDE.md Co-authored-by: Owais Kazi Signed-off-by: Varun Jain --- DEVELOPER_GUIDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 51355e017..4f2cb0e78 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -177,7 +177,7 @@ In order to run the integration tests with a 3 node cluster, run this command: ./gradlew :integTest -PnumNodes=3 ``` -Additionally, to run integration tests with security enabled, run +Additionally, to run integration tests on multi nodes with security enabled, run ``` ./gradlew :integTest -Dsecurity.enabled=true -PnumNodes=3 ``` From a3bf2dfccfd00a1333dfd77ecf1aee49e087f3ab Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 16 Jan 2024 14:22:06 -0800 Subject: [PATCH 8/8] Update DEVELOPER_GUIDE.md Co-authored-by: Owais Kazi Signed-off-by: Varun Jain --- DEVELOPER_GUIDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 4f2cb0e78..f19f178c4 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -134,7 +134,7 @@ curl localhost:9200 Additionally, it is also possible to run a cluster with security enabled: ```shell script -./gradlew run -Dsecurity.enabled=true -Dhttps=true -Duser=admin -Dpassword=admin +./gradlew run -Dsecurity.enabled=true ``` By default, if `-Dsecurity.enabled=true` is passed the following defaults will be used: `https=true`, `user=admin` and