diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 1090226b0..1d8bf6884 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -4,7 +4,8 @@ on: branches: - "*" - "feature/**" - pull_request: + pull_request_target: + types: [ opened, synchronize, reopened ] branches: - "*" - "feature/**" @@ -55,7 +56,7 @@ jobs: - name: Run build run: | - ./gradlew precommit + ./gradlew precommit --parallel - name: Upload Coverage Report uses: codecov/codecov-action@v1 diff --git a/.idea/runConfigurations/DebugNeuralSearch.xml b/.idea/runConfigurations/DebugNeuralSearch.xml new file mode 100644 index 000000000..6246c430e --- /dev/null +++ b/.idea/runConfigurations/DebugNeuralSearch.xml @@ -0,0 +1,16 @@ + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations/Run_Neural_Search.xml b/.idea/runConfigurations/Run_Neural_Search.xml new file mode 100644 index 000000000..d881bd512 --- /dev/null +++ b/.idea/runConfigurations/Run_Neural_Search.xml @@ -0,0 +1,23 @@ + + + + + + + true + true + false + + + \ No newline at end of file diff --git a/.idea/runConfigurations/Run_With_Debug_Port.xml b/.idea/runConfigurations/Run_With_Debug_Port.xml new file mode 100644 index 000000000..3a2b6fe4f --- /dev/null +++ b/.idea/runConfigurations/Run_With_Debug_Port.xml @@ -0,0 +1,24 @@ + + + + + + + true + true + false + + + \ No newline at end of file diff --git a/build.gradle b/build.gradle index 7bde310fa..614131f33 100644 --- a/build.gradle +++ b/build.gradle @@ -6,6 +6,7 @@ * This project uses @Incubating APIs which are subject to change. */ import org.opensearch.gradle.test.RestIntegTestTask +import java.util.concurrent.Callable apply plugin: 'java' apply plugin: 'idea' @@ -13,6 +14,7 @@ apply plugin: 'opensearch.opensearchplugin' apply plugin: 'opensearch.pluginzip' apply plugin: 'jacoco' apply plugin: "com.diffplug.spotless" +apply plugin: 'io.freefair.lombok' def pluginName = 'neural-search' def pluginDescription = 'A plugin that adds dense neural retrieval into the OpenSearch ecosytem' @@ -59,7 +61,6 @@ opensearchplugin { noticeFile rootProject.file('NOTICE') } -licenseHeaders.enabled = true dependencyLicenses.enabled = false thirdPartyAudit.enabled = false loggerUsageCheck.enabled = false @@ -68,7 +69,9 @@ validateNebulaPom.enabled = false buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") + // as we don't have 3.0.0, 2.4.0 version for K-NN on darwin we need to keep OpenSearch version as 2.3 for now. + // Github issue: https://github.com/opensearch-project/opensearch-build/issues/2662 + opensearch_version = System.getProperty("opensearch.version", "2.3.0-SNAPSHOT") buildVersionQualifier = System.getProperty("build.version_qualifier", "") isSnapshot = "true" == System.getProperty("build.snapshot", "true") version_tokens = opensearch_version.tokenize('-') @@ -82,6 +85,13 @@ buildscript { opensearch_build += "-SNAPSHOT" } opensearch_group = "org.opensearch" + opensearch_no_snapshot = opensearch_build.replace("-SNAPSHOT","") + k_NN_resource_folder = "build/resources/k-NN" + ml_common_resource_folder = "build/resources/ml-commons" + //TODO: we need a better way to construct this URL as, this URL is valid for released version of K-NN, ML-Plugin. + // Github issue: https://github.com/opensearch-project/opensearch-build/issues/2662 + k_NN_build_download_url = "https://aws.oss.sonatype.org/content/repositories/releases/org/opensearch/plugin/opensearch-knn/" + opensearch_no_snapshot + "/opensearch-knn-" + opensearch_no_snapshot +".zip" + ml_common_build_download_url = "https://aws.oss.sonatype.org/content/repositories/releases/org/opensearch/plugin/opensearch-ml-plugin/" + opensearch_no_snapshot + "/opensearch-ml-plugin-" + opensearch_no_snapshot +".zip" } repositories { @@ -93,6 +103,7 @@ buildscript { dependencies { classpath "${opensearch_group}.gradle:build-tools:${opensearch_version}" classpath "com.diffplug.spotless:spotless-plugin-gradle:5.6.1" + classpath "io.freefair.gradle:lombok-plugin:6.4.3" } } @@ -113,12 +124,25 @@ repositories { maven { url "https://plugins.gradle.org/m2/" } } +dependencies { + api "org.opensearch:opensearch:${opensearch_version}" + api group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}" +} + +compileJava { + options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) +} +compileTestJava { + options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) +} + def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile opensearch_tmp_dir.mkdirs() def _numNodes = findProperty('numNodes') as Integer ?: 1 test { include '**/*Tests.class' + systemProperty 'tests.security.manager', 'false' } // Setting up Integration Tests @@ -161,8 +185,47 @@ integTest { testClusters.integTest { testDistribution = "ARCHIVE" + // Install ML-Plugin on the integTest cluster nodes + plugin(provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + if (new File("$project.rootDir/$ml_common_resource_folder").exists()) { + project.delete(files("$project.rootDir/$ml_common_resource_folder")) + } + project.mkdir ml_common_resource_folder + ant.get(src: ml_common_build_download_url, + dest: ml_common_resource_folder, + httpusecaches: false) + return fileTree(ml_common_resource_folder).getSingleFile() + } + } + } + })) + + // Install K-NN plugin on the integTest cluster nodes + plugin(provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + if (new File("$project.rootDir/$k_NN_resource_folder").exists()) { + project.delete(files("$project.rootDir/$k_NN_resource_folder")) + } + project.mkdir k_NN_resource_folder + ant.get(src: k_NN_build_download_url, + dest: k_NN_resource_folder, + httpusecaches: false) + return fileTree(k_NN_resource_folder).getSingleFile() + } + } + } + })) - // This installs our plugin into the testClusters + // This installs our neural-search plugin into the testClusters plugin(project.tasks.bundlePlugin.archiveFile) // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 if (_numNodes > 1) numberOfNodes = _numNodes @@ -178,6 +241,28 @@ testClusters.integTest { } } +// Remote Integration Tests +task integTestRemote(type: RestIntegTestTask) { + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath + + systemProperty "https", System.getProperty("https") + systemProperty "user", System.getProperty("user") + systemProperty "password", System.getProperty("password") + + systemProperty 'cluster.number_of_nodes', "${_numNodes}" + + systemProperty 'tests.security.manager', 'false' + + // Run tests with remote cluster only if rest case is defined + if (System.getProperty("tests.rest.cluster") != null) { + filter { + includeTestsMatching "org.opensearch.neuralsearch.*IT" + } + } +} + + run { useCluster testClusters.integTest } diff --git a/src/test/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java b/src/test/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java new file mode 100644 index 000000000..19fd74659 --- /dev/null +++ b/src/test/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java @@ -0,0 +1,162 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.neuralsearch; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.apache.http.Header; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.message.BasicHeader; +import org.apache.http.ssl.SSLContextBuilder; +import org.junit.After; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestClientBuilder; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.common.xcontent.DeprecationHandler; +import org.opensearch.common.xcontent.NamedXContentRegistry; +import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.test.rest.OpenSearchRestTestCase; + +/** + * Base class for running the integration tests on a secure cluster. The plugin IT test should either extend this + * class or create another base class by extending this class to make sure that their IT can be run on secure clusters. + */ +public abstract class OpenSearchSecureRestTestCase extends OpenSearchRestTestCase { + + private static final String PROTOCOL_HTTP = "http"; + private static final String PROTOCOL_HTTPS = "https"; + private static final String SYS_PROPERTY_KEY_HTTPS = "https"; + private static final String SYS_PROPERTY_KEY_CLUSTER_ENDPOINT = "tests.rest.cluster"; + private static final String SYS_PROPERTY_KEY_USER = "user"; + private static final String SYS_PROPERTY_KEY_PASSWORD = "password"; + private static final String DEFAULT_SOCKET_TIMEOUT = "60s"; + private static final String INTERNAL_INDICES_PREFIX = "."; + private static String protocol; + + @Override + protected String getProtocol() { + if (protocol == null) { + protocol = readProtocolFromSystemProperty(); + } + return protocol; + } + + private String readProtocolFromSystemProperty() { + final boolean isHttps = Optional.ofNullable(System.getProperty(SYS_PROPERTY_KEY_HTTPS)).map("true"::equalsIgnoreCase).orElse(false); + if (!isHttps) { + return PROTOCOL_HTTP; + } + + // currently only external cluster is supported for security enabled testing + if (Optional.ofNullable(System.getProperty(SYS_PROPERTY_KEY_CLUSTER_ENDPOINT)).isEmpty()) { + throw new RuntimeException("cluster url should be provided for security enabled testing"); + } + return PROTOCOL_HTTPS; + } + + @Override + protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { + final RestClientBuilder builder = RestClient.builder(hosts); + if (PROTOCOL_HTTPS.equals(getProtocol())) { + configureHttpsClient(builder, settings); + } else { + configureClient(builder, settings); + } + + return builder.build(); + } + + private void configureHttpsClient(final RestClientBuilder builder, final Settings settings) { + final Map headers = ThreadContext.buildDefaultHeaders(settings); + final Header[] defaultHeaders = new Header[headers.size()]; + int i = 0; + for (Map.Entry entry : headers.entrySet()) { + defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue()); + } + builder.setDefaultHeaders(defaultHeaders); + builder.setHttpClientConfigCallback(httpClientBuilder -> { + final String userName = Optional.ofNullable(System.getProperty(SYS_PROPERTY_KEY_USER)) + .orElseThrow(() -> new RuntimeException("user name is missing")); + final String password = Optional.ofNullable(System.getProperty(SYS_PROPERTY_KEY_PASSWORD)) + .orElseThrow(() -> new RuntimeException("password is missing")); + final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password)); + try { + return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider) + // disable the certificate since our testing cluster just uses the default security configuration + .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .setSSLContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build()); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT); + final TimeValue socketTimeout = TimeValue.parseTimeValue( + socketTimeoutString == null ? DEFAULT_SOCKET_TIMEOUT : socketTimeoutString, + CLIENT_SOCKET_TIMEOUT + ); + builder.setRequestConfigCallback(conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis()))); + if (settings.hasValue(CLIENT_PATH_PREFIX)) { + builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX)); + } + } + + /** + * wipeAllIndices won't work since it cannot delete security index. Use deleteExternalIndices instead. + */ + @Override + protected boolean preserveIndicesUponCompletion() { + return true; + } + + @After + public void deleteExternalIndices() throws IOException { + final Response response = client().performRequest(new Request("GET", "/_cat/indices?format=json" + "&expand_wildcards=all")); + final XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType().getValue()); + try ( + final XContentParser parser = xContentType.xContent() + .createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + response.getEntity().getContent() + ) + ) { + final XContentParser.Token token = parser.nextToken(); + final List> parserList; + if (token == XContentParser.Token.START_ARRAY) { + parserList = parser.listOrderedMap().stream().map(obj -> (Map) obj).collect(Collectors.toList()); + } else { + parserList = Collections.singletonList(parser.mapOrdered()); + } + + final List externalIndices = parserList.stream() + .map(index -> (String) index.get("index")) + .filter(indexName -> indexName != null) + .filter(indexName -> !indexName.startsWith(INTERNAL_INDICES_PREFIX)) + .collect(Collectors.toList()); + + for (final String indexName : externalIndices) { + adminClient().performRequest(new Request("DELETE", "/" + indexName)); + } + } + } +} diff --git a/src/test/java/org/opensearch/neuralsearch/plugin/NeuralSearchIT.java b/src/test/java/org/opensearch/neuralsearch/plugin/NeuralSearchIT.java index d49c6a9a6..7dbbbee25 100644 --- a/src/test/java/org/opensearch/neuralsearch/plugin/NeuralSearchIT.java +++ b/src/test/java/org/opensearch/neuralsearch/plugin/NeuralSearchIT.java @@ -6,9 +6,9 @@ package org.opensearch.neuralsearch.plugin; import org.junit.Assert; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.neuralsearch.OpenSearchSecureRestTestCase; -public class NeuralSearchIT extends OpenSearchIntegTestCase { +public class NeuralSearchIT extends OpenSearchSecureRestTestCase { /** * Dummy test case for passing the build. diff --git a/src/test/java/org/opensearch/neuralsearch/plugin/ValidateDependentPluginInstallationIT.java b/src/test/java/org/opensearch/neuralsearch/plugin/ValidateDependentPluginInstallationIT.java new file mode 100644 index 000000000..b0c0fd1b8 --- /dev/null +++ b/src/test/java/org/opensearch/neuralsearch/plugin/ValidateDependentPluginInstallationIT.java @@ -0,0 +1,122 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.neuralsearch.plugin; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.Set; + +import org.junit.Assert; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.common.Strings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.neuralsearch.OpenSearchSecureRestTestCase; +import org.opensearch.rest.RestRequest; + +/** + * This test validates that dependent plugin defined in build.gradle -> opensearchplugin.extendedPlugins are + * installed or not. We don't want to run these tests before every IT as it can slow down the execution of IT. Hence + * doing it separately. + */ +public class ValidateDependentPluginInstallationIT extends OpenSearchSecureRestTestCase { + + private static final String KNN_INDEX_NAME = "neuralsearchknnindexforvalidation"; + private static final String KNN_VECTOR_FIELD_NAME = "vectorField"; + private static final Set DEPENDENT_PLUGINS = Set.of("opensearch-ml", "opensearch-knn"); + private static final String GET_PLUGINS_URL = "_cat/plugins"; + private static final String ML_PLUGIN_STATS_URL = "_plugins/_ml/stats"; + private static final String KNN_DOCUMENT_URL = KNN_INDEX_NAME + "/_doc/1?refresh"; + + public void testDependentPluginsInstalled() throws IOException { + final Set installedPlugins = getAllInstalledPlugins(); + Assert.assertTrue(installedPlugins.containsAll(DEPENDENT_PLUGINS)); + } + + /** + * Validate K-NN Plugin Setup by creating a k-NN index and then deleting the index. Not adding the index deletion + * in the cleanup setup as index creation was not part of whole setup for this test cases class. + */ + public void testValidateKNNPluginSetup() throws IOException { + createBasicKnnIndex(); + Assert.assertTrue(indexExists(KNN_INDEX_NAME)); + indexDocument(); + getDocument(); + deleteIndex(KNN_INDEX_NAME); + Assert.assertFalse(indexExists(KNN_INDEX_NAME)); + } + + /** + * Validate ML-Plugin setup. We are using the stats API of ML Plugin to validate plugin is present or not. This + * was the best API that we can use without creating any side effects. + */ + public void testValidateMLPluginSetup() throws IOException { + final Request request = new Request(RestRequest.Method.GET.name(), ML_PLUGIN_STATS_URL); + assertOK(client().performRequest(request)); + } + + private void createBasicKnnIndex() throws IOException { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(KNN_VECTOR_FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", Integer.toString(3)) + .startObject("method") + .field("engine", "lucene") + .field("name", "hnsw") + .endObject() + .endObject() + .endObject() + .endObject() + ); + mapping = mapping.substring(1, mapping.length() - 1); + createIndex(KNN_INDEX_NAME, Settings.EMPTY, mapping); + } + + private Set getAllInstalledPlugins() throws IOException { + final Request request = new Request(RestRequest.Method.GET.name(), GET_PLUGINS_URL); + final Response response = client().performRequest(request); + assertOK(response); + final BufferedReader responseReader = new BufferedReader( + new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8) + ); + final Set installedPluginsSet = new HashSet<>(); + String line; + while ((line = responseReader.readLine()) != null) { + // Output looks like this: integTest-0 opensearch-knn 2.3.0.0 + final String pluginName = line.split("\\s+")[1]; + installedPluginsSet.add(pluginName); + } + return installedPluginsSet; + } + + private void indexDocument() throws IOException { + final String indexRequestBody = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startArray(KNN_VECTOR_FIELD_NAME) + .value(1.0) + .value(2.0) + .value(4.0) + .endArray() + .endObject() + ); + final Request indexRequest = new Request(RestRequest.Method.POST.name(), KNN_DOCUMENT_URL); + indexRequest.setJsonEntity(indexRequestBody); + assertOK(client().performRequest(indexRequest)); + } + + private void getDocument() throws IOException { + final Request getDocumentRequest = new Request(RestRequest.Method.GET.name(), KNN_DOCUMENT_URL); + assertOK(client().performRequest(getDocumentRequest)); + } +}