Skip to content

Commit

Permalink
Optimizing pulling of security files
Browse files Browse the repository at this point in the history 
Signed-off-by: Varun Jain <[email protected]>
  • Loading branch information
@vibrantvarun
vibrantvarun committed Feb 1, 2024
1 parent 8c1a438 commit 57b5616
Show file tree
Hide file tree
Showing 9 changed files with 59 additions and 218 deletions.
33 changes: 21 additions & 12 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,26 @@ apply plugin: "com.diffplug.spotless"
apply plugin: 'io.freefair.lombok'
apply from: 'gradle/formatting.gradle'

def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile
ext.opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile
opensearch_tmp_dir.mkdirs()

ext {
default_bwc_version = System.getProperty("bwc.version")
default_bwc_bundle_version= System.getProperty("bwc.bundle.version")
bwcBundleTest = (project.findProperty('customDistributionDownloadType') != null && project.properties['customDistributionDownloadType'] == "bundle")
neural_search_bwc_version = bwcBundleTest ? System.getProperty("tests.bwc.bundle.version",rootProject.ext.default_bwc_bundle_version): System.getProperty("tests.bwc.version", rootProject.ext.default_bwc_version)
currentBundleVersion = opensearch_version.replace("-SNAPSHOT","")

// Config below including files are copied from security demo configuration
['esnode.pem', 'esnode-key.pem', 'root-ca.pem','kirk.pem','kirk-key.pem'].forEach { file ->
File local = Paths.get(opensearch_tmp_dir.absolutePath, file).toFile()
download.run {
src "https://raw.githubusercontent.com/opensearch-project/security/main/bwc-test/src/test/resources/security/" + file
dest local
overwrite false
}
}

isSnapshot = "true" == System.getProperty("build.snapshot", "true")
projectSubstitutions = [:]

Expand Down Expand Up @@ -91,17 +107,10 @@ ext {
}
}

// Config below including files are copied from security demo configuration
['esnode.pem', 'esnode-key.pem', 'root-ca.pem'].forEach { file ->
File local = Paths.get(opensearch_tmp_dir.absolutePath, file).toFile()
download.run {
src "https://raw.githubusercontent.com/opensearch-project/security/main/bwc-test/src/test/resources/security/" + file
dest local
overwrite false
}
cluster.extraConfigFile(file, local)
}

// // Config below including files are copied from security demo configuration
cluster.extraConfigFile("esnode.pem", file("$opensearch_tmp_dir/esnode.pem"))
cluster.extraConfigFile("esnode-key.pem", file("$opensearch_tmp_dir/esnode-key.pem"))
cluster.extraConfigFile("root-ca.pem", file("$opensearch_tmp_dir/root-ca.pem"))
// This configuration is copied from the security plugins demo install:
// https://github.com/opensearch-project/security/blob/2.11.1.0/tools/install_demo_configuration.sh#L365-L388
cluster.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
Expand Down
35 changes: 16 additions & 19 deletions qa/restart-upgrade/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,7 @@ import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask

apply from : "$rootDir/qa/build.gradle"

String default_bwc_version = System.getProperty("bwc.version")
Boolean bwcBundleTest = (project.findProperty('customDistributionDownloadType') != null &&
project.properties['customDistributionDownloadType'] == "bundle")
String neural_search_bwc_version = bwcBundleTest ? System.getProperty("bwc.bundle.version"): System.getProperty("tests.bwc.version", default_bwc_version)
String currentBundleVersion = opensearch_version.replace("-SNAPSHOT","")
def ext=rootProject.ext
String baseName = "neuralSearchBwcCluster-restart"

// Creates a test cluster of previous version and loads k-NN plugin of bwcVersion
Expand All @@ -20,14 +16,15 @@ testClusters {
testDistribution = "ARCHIVE"
numberOfNodes = 3
jvmArgs("-Xms1g", "-Xmx4g")
if(bwcBundleTest){
versions = [neural_search_bwc_version, currentBundleVersion]
if(ext.bwcBundleTest){
versions = [ext.neural_search_bwc_version, ext.currentBundleVersion]
def path=ext.opensearch_tmp_dir
nodes.each { node ->
node.extraConfigFile("kirk.pem", file("$rootDir/src/test/resources/security/kirk.pem"))
node.extraConfigFile("kirk-key.pem", file("$rootDir/src/test/resources/security/kirk-key.pem"))
node.extraConfigFile("esnode.pem", file("$rootDir/src/test/resources/security/esnode.pem"))
node.extraConfigFile("esnode-key.pem", file("$rootDir/src/test/resources/security/esnode-key.pem"))
node.extraConfigFile("root-ca.pem", file("$rootDir/src/test/resources/security/root-ca.pem"))
node.extraConfigFile("kirk.pem", file("$path/kirk.pem"))
node.extraConfigFile("kirk-key.pem", file("$path/kirk-key.pem"))
node.extraConfigFile("esnode.pem", file("$path/esnode.pem"))
node.extraConfigFile("esnode-key.pem", file("$path/esnode-key.pem"))
node.extraConfigFile("root-ca.pem", file("$path/root-ca.pem"))
node.setting("plugins.security.disabled", "true")
node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
Expand All @@ -47,7 +44,7 @@ testClusters {
node.setting("plugins.security.system_indices.enabled", "true")
}
}else{
versions = [neural_search_bwc_version, opensearch_version]
versions = [ext.neural_search_bwc_version, opensearch_version]
plugin(project.tasks.unZipBwcMlCommonsPlugin.archiveFile)
plugin(project.tasks.unZipBwcKnnPlugin.archiveFile)
plugin(project.tasks.unZipBwcPlugin.archiveFile)
Expand All @@ -59,17 +56,17 @@ testClusters {

// Task to run BWC tests against the old cluster
task testAgainstOldCluster(type: StandaloneRestIntegTestTask) {
if(!bwcBundleTest){
if(!ext.bwcBundleTest){
dependsOn "unZipBwcPlugin"
}
useCluster testClusters."${baseName}"
systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster'
systemProperty 'tests.is_old_cluster', 'true'
systemProperty 'tests.skip_delete_model_index', 'true'
systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version
systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version

//Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version.
if (neural_search_bwc_version.startsWith("2.9") || neural_search_bwc_version.startsWith("2.10")){
if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){
filter {
excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*"
excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*"
Expand All @@ -87,7 +84,7 @@ task testAgainstNewCluster(type: StandaloneRestIntegTestTask) {
dependsOn "testAgainstOldCluster"
dependsOn rootProject.tasks.assemble
useCluster testClusters."${baseName}"
if(bwcBundleTest){
if(ext.bwcBundleTest){
doFirst {
testClusters."${baseName}".goToNextVersion()
}
Expand All @@ -99,10 +96,10 @@ task testAgainstNewCluster(type: StandaloneRestIntegTestTask) {
systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster'
systemProperty 'tests.skip_delete_model_index', 'true'
systemProperty 'tests.is_old_cluster', 'false'
systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version
systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version

//Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version.
if (neural_search_bwc_version.startsWith("2.9") || neural_search_bwc_version.startsWith("2.10")){
if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){
filter {
excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*"
excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*"
Expand Down
47 changes: 22 additions & 25 deletions qa/rolling-upgrade/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,7 @@ import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask

apply from : "$rootDir/qa/build.gradle"

String default_bwc_version = System.getProperty("bwc.version")
Boolean bwcBundleTest = (project.findProperty('customDistributionDownloadType') != null &&
project.properties['customDistributionDownloadType'] == "bundle")
String neural_search_bwc_version = bwcBundleTest ? System.getProperty("bwc.bundle.version"): System.getProperty("tests.bwc.version", default_bwc_version)
String currentBundleVersion = opensearch_version.replace("-SNAPSHOT","")
def ext=rootProject.ext
String baseName = "neuralSearchBwcCluster-rolling"

// Creates a test cluster of previous version and loads k-NN plugin of bwcVersion
Expand All @@ -20,14 +16,15 @@ testClusters {
testDistribution = "ARCHIVE"
jvmArgs("-Xms1g", "-Xmx4g")
numberOfNodes = 3
if(bwcBundleTest){
versions = [neural_search_bwc_version, currentBundleVersion]
if(ext.bwcBundleTest){
versions = [ext.neural_search_bwc_version, ext.currentBundleVersion]
def path=ext.opensearch_tmp_dir
nodes.each { node ->
node.extraConfigFile("kirk.pem", file("$rootDir/src/test/resources/security/kirk.pem"))
node.extraConfigFile("kirk-key.pem", file("$rootDir/src/test/resources/security/kirk-key.pem"))
node.extraConfigFile("esnode.pem", file("$rootDir/src/test/resources/security/esnode.pem"))
node.extraConfigFile("esnode-key.pem", file("$rootDir/src/test/resources/security/esnode-key.pem"))
node.extraConfigFile("root-ca.pem", file("$rootDir/src/test/resources/security/root-ca.pem"))
node.extraConfigFile("kirk.pem", file("$path/kirk.pem"))
node.extraConfigFile("kirk-key.pem", file("$path/kirk-key.pem"))
node.extraConfigFile("esnode.pem", file("$path/esnode.pem"))
node.extraConfigFile("esnode-key.pem", file("$path/esnode-key.pem"))
node.extraConfigFile("root-ca.pem", file("$path/root-ca.pem"))
node.setting("plugins.security.disabled", "true")
node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
Expand All @@ -47,7 +44,7 @@ testClusters {
node.setting("plugins.security.system_indices.enabled", "true")
}
}else{
versions = [neural_search_bwc_version, opensearch_version]
versions = [ext.neural_search_bwc_version, opensearch_version]
plugin(project.tasks.unZipBwcMlCommonsPlugin.archiveFile)
plugin(project.tasks.unZipBwcKnnPlugin.archiveFile)
plugin(project.tasks.unZipBwcPlugin.archiveFile)
Expand All @@ -60,16 +57,16 @@ testClusters {

// Task to run BWC tests against the old cluster
task testAgainstOldCluster(type: StandaloneRestIntegTestTask) {
if(!bwcBundleTest){
if(!ext.bwcBundleTest){
dependsOn "unZipBwcPlugin"
}
useCluster testClusters."${baseName}"
systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster'
systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version
systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version
systemProperty 'tests.skip_delete_model_index', 'true'

//Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version.
if (neural_search_bwc_version.startsWith("2.9") || neural_search_bwc_version.startsWith("2.10")){
if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){
filter {
excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*"
excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*"
Expand All @@ -88,7 +85,7 @@ task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) {
useCluster testClusters."${baseName}"
dependsOn rootProject.tasks.assemble
dependsOn "testAgainstOldCluster"
if (bwcBundleTest){
if (ext.bwcBundleTest){
doFirst {
testClusters."${baseName}".nextNodeToNextVersion()
}
Expand All @@ -100,10 +97,10 @@ task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) {
systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster'
systemProperty 'tests.rest.first_round', 'true'
systemProperty 'tests.skip_delete_model_index', 'true'
systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version
systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version

//Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version.
if (neural_search_bwc_version.startsWith("2.9") || neural_search_bwc_version.startsWith("2.10")){
if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){
filter {
excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*"
excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*"
Expand All @@ -121,7 +118,7 @@ task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) {
task testAgainstTwoThirdsUpgradedCluster(type: StandaloneRestIntegTestTask) {
dependsOn "testAgainstOneThirdUpgradedCluster"
useCluster testClusters."${baseName}"
if (bwcBundleTest){
if (ext.bwcBundleTest){
doFirst {
testClusters."${baseName}".nextNodeToNextVersion()
}
Expand All @@ -133,10 +130,10 @@ task testAgainstTwoThirdsUpgradedCluster(type: StandaloneRestIntegTestTask) {
systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster'
systemProperty 'tests.rest.first_round', 'false'
systemProperty 'tests.skip_delete_model_index', 'true'
systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version
systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version

//Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version.
if (neural_search_bwc_version.startsWith("2.9") || neural_search_bwc_version.startsWith("2.10")){
if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){
filter {
excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*"
excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*"
Expand All @@ -154,7 +151,7 @@ task testAgainstTwoThirdsUpgradedCluster(type: StandaloneRestIntegTestTask) {
task testRollingUpgrade(type: StandaloneRestIntegTestTask) {
dependsOn "testAgainstTwoThirdsUpgradedCluster"
useCluster testClusters."${baseName}"
if (bwcBundleTest){
if (ext.bwcBundleTest){
doFirst {
testClusters."${baseName}".nextNodeToNextVersion()
}
Expand All @@ -166,10 +163,10 @@ task testRollingUpgrade(type: StandaloneRestIntegTestTask) {
mustRunAfter "testAgainstOneThirdUpgradedCluster"
systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster'
systemProperty 'tests.skip_delete_model_index', 'true'
systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version
systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version

//Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version.
if (neural_search_bwc_version.startsWith("2.9") || neural_search_bwc_version.startsWith("2.10")){
if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){
filter {
excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*"
excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*"
Expand Down
28 changes: 0 additions & 28 deletions src/test/resources/security/esnode-key.pem
View file

This file was deleted.

28 changes: 0 additions & 28 deletions src/test/resources/security/esnode.pem
View file

This file was deleted.

Loading

0 comments on commit 57b5616

Please sign in to comment.