diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index e7cc123d4d62a..4d71ffd70a2ad 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -49,7 +49,7 @@ Fork [opensearch-project/OpenSearch](https://github.com/opensearch-project/OpenS OpenSearch builds using Java 14 at a minimum. This means you must have a JDK 14 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 14 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-14`. -One easy way to get Java 14 on *nix is to use [sdkman](https://sdkman.io/). +One easy way to get Java 14 on *nix is to use [sdkman](https://sdkman.io/). ```bash curl -s "https://get.sdkman.io" | bash @@ -189,6 +189,14 @@ The canonical example of this is the ICU analysis plugin. It is important for fo Another example is the `discovery-gce` plugin. It is *vital* to folks running in [GCP](https://cloud.google.com/) but useless otherwise and it depends on a dozen extra jars. +### `sandbox` + +This is where the community can add experimental features in to OpenSearch. There are three directories inside the sandbox - `libs`, `modules` and `plugins` - which mirror the subdirectories in the project root and have the same guidelines for deciding on where a new feature goes. The artifacts from `libs` and `modules` will be automatically included in the **snapshot** distributions. Once a certain feature is deemed worthy to be included in the OpenSearch release, it will be promoted to the corresponding subdirectory in the project root. **Note**: The sandbox code do not have any other guarantees such as backwards compatibility or long term support and can be removed at any time. + +To exclude the modules from snapshot distributions, use the `sandbox.enabled` system property. + + ./gradlew assemble -Dsandbox.enabled=false + ### `qa` Honestly this is kind of in flux and we're not 100% sure where we'll end up. We welcome your thoughts and help. diff --git a/distribution/build.gradle b/distribution/build.gradle index 3e242aa1a2fc0..2f1624ecf6235 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -238,6 +238,34 @@ project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each { restTestExpansions['expected.modules.count'] += 1 } +// copy all sandbox modules if the distribution is a snapshot +if (VersionProperties.isOpenSearchSnapshot()) { + Properties sysProps = System.getProperties(); + // setting this property to false will exclude the sandbox modules from the distribution + final String enableSandbox = sysProps.getProperty("sandbox.enabled", "true"); + if(sysProps != null && enableSandbox == "true") { + tasks.withType(NoticeTask).configureEach { + project(':sandbox:libs').subprojects.each { Project lib -> + licensesDir lib.file('licenses') + source lib.file('src/main/java') + } + } + project.rootProject.subprojects.findAll { it.parent.path == ':sandbox:modules' }.each { Project module -> + File licenses = new File(module.projectDir, 'licenses') + if (licenses.exists()) { + buildNotice.configure { + licensesDir licenses + source module.file('src/main/java') + } + } + + copyModule(processOutputsTaskProvider, module) + copyLog4jProperties(buildLog4jConfigTaskProvider, module) + restTestExpansions['expected.modules.count'] += 1 + } + } +} + copyModule(processSystemdOutputsTaskProvider, project(':modules:systemd')) project(':test:external-modules').subprojects.each { Project testModule -> @@ -268,7 +296,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { * Properties to expand when copying packaging files * *****************************************************************************/ configurations { - ['libs', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli'].each { + ['libs', 'libsPluginCli', 'libsKeystoreCli'].each { create(it) { canBeConsumed = false canBeResolved = true diff --git a/sandbox/build.gradle b/sandbox/build.gradle new file mode 100644 index 0000000000000..379595a4ba087 --- /dev/null +++ b/sandbox/build.gradle @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * This module provides a space in OpenSearch for the community to easily experiment with new ideas and innovate. + * Ideally, this is where an experimental features will reside before it can be promoted to the corresponding directory + * in the project root. The sandbox module contains three subdirectories, that mirror the root libs, modules and + * plugins directories, each with similar intention. + * + * All artifacts from the sandbox/libs and sandbox/modules will be included in the snapshot distributions automatically. + * During assembling distributions, however, we will check if the following two conditions are met, for including the + * sandbox modules, + + * 1. The distribution is a snapshot i.e. the build system property build.snapshot is set to true. We use this because, + * it will prevent accidental inclusion of these artifacts in a release distribution. + * + * 2. The sandbox.enabled system property is set to true. This new extra flag is added because we can exclude the + * modules from the snapshot distributions, if needed. For instance, we may want to run performance tests on snapshots + * without the sandbox modules. + * + * To build the distributions without the sandbox modules, + * ./gradlew assemble -Dsandbox.enabled=false + * + * Similarly we can run OpenSearch from source without the sandbox modules + * ./gradlew run -Dsandbox.enabled=false + */ +subprojects { + group = 'org.opensearch.sandbox' +} diff --git a/sandbox/libs/build.gradle b/sandbox/libs/build.gradle new file mode 100644 index 0000000000000..a2ebb4764e89e --- /dev/null +++ b/sandbox/libs/build.gradle @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +subprojects { + /* + * All subprojects are java projects using OpenSearch's standard build + * tools. + */ + apply plugin: 'opensearch.build' + + /* + * Subprojects may depend on the "core" lib but may not depend on any + * other libs. This keeps our dependencies simpler. + */ + project.afterEvaluate { + configurations.all { Configuration conf -> + dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep -> + Project depProject = dep.dependencyProject + if (depProject != null + && false == depProject.path.equals(':libs:opensearch-core') + && depProject.path.startsWith(':libs')) { + throw new InvalidUserDataException("projects in :libs " + + "may not depend on other projects libs except " + + ":libs:opensearch-core but " + + "${project.path} depends on ${depProject.path}") + } + } + } + } +} diff --git a/sandbox/modules/build.gradle b/sandbox/modules/build.gradle new file mode 100644 index 0000000000000..61afb2c568e1b --- /dev/null +++ b/sandbox/modules/build.gradle @@ -0,0 +1,19 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +configure(subprojects.findAll { it.parent.path == project.path }) { + group = 'org.opensearch.sandbox' + apply plugin: 'opensearch.testclusters' + apply plugin: 'opensearch.opensearchplugin' + + opensearchplugin { + name project.name + licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt') + noticeFile rootProject.file('NOTICE.txt') + } +} diff --git a/sandbox/plugins/build.gradle b/sandbox/plugins/build.gradle new file mode 100644 index 0000000000000..61afb2c568e1b --- /dev/null +++ b/sandbox/plugins/build.gradle @@ -0,0 +1,19 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +configure(subprojects.findAll { it.parent.path == project.path }) { + group = 'org.opensearch.sandbox' + apply plugin: 'opensearch.testclusters' + apply plugin: 'opensearch.opensearchplugin' + + opensearchplugin { + name project.name + licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt') + noticeFile rootProject.file('NOTICE.txt') + } +} diff --git a/settings.gradle b/settings.gradle index 6599a84891db4..429c2005250ea 100644 --- a/settings.gradle +++ b/settings.gradle @@ -106,6 +106,7 @@ project(':example-plugins').projectDir = new File(rootProject.projectDir, 'plugi addSubProjects('', new File(rootProject.projectDir, 'libs')) addSubProjects('', new File(rootProject.projectDir, 'modules')) addSubProjects('', new File(rootProject.projectDir, 'plugins')) +addSubProjects('', new File(rootProject.projectDir, 'sandbox')) addSubProjects('', new File(rootProject.projectDir, 'qa')) addSubProjects('test', new File(rootProject.projectDir, 'test/external-modules'))