From 6a570bd86eb7627db007d3783c0f2ccbe3270e62 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Thu, 7 Dec 2023 13:03:51 -0800 Subject: [PATCH 01/45] Reformatting test package Signed-off-by: Varun Jain --- build.gradle | 9 +++++++++ .../processor/NeuralQueryEnricherProcessorIT.java | 7 +++---- .../processor/NormalizationProcessorIT.java | 9 ++++----- .../neuralsearch/processor/ScoreCombinationIT.java | 7 +++---- .../processor/ScoreNormalizationIT.java | 7 +++---- .../processor/SparseEncodingProcessIT.java | 2 +- .../processor/TextEmbeddingProcessorIT.java | 2 +- .../processor/TextImageEmbeddingProcessorIT.java | 2 +- .../neuralsearch/query/HybridQueryIT.java | 10 ++++------ .../neuralsearch/query/NeuralQueryIT.java | 8 +++----- .../neuralsearch/query/NeuralSparseQueryIT.java | 2 +- .../neuralsearch}/BaseNeuralSearchIT.java | 3 +-- .../neuralsearch}/BaseSparseEncodingIT.java | 2 +- .../neuralsearch/OpenSearchSecureRestTestCase.java | 0 .../org/opensearch/neuralsearch/TestUtils.java | 14 +++++++++++++- 15 files changed, 48 insertions(+), 36 deletions(-) rename src/{test/java/org/opensearch/neuralsearch/common => testFixtures/java/org/opensearch/neuralsearch}/BaseNeuralSearchIT.java (99%) rename src/{test/java/org/opensearch/neuralsearch/common => testFixtures/java/org/opensearch/neuralsearch}/BaseSparseEncodingIT.java (99%) rename src/{test => testFixtures}/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java (100%) rename src/{test => testFixtures}/java/org/opensearch/neuralsearch/TestUtils.java (93%) diff --git a/build.gradle b/build.gradle index 7220fda23..2afa4cf76 100644 --- a/build.gradle +++ b/build.gradle @@ -11,6 +11,7 @@ import org.opensearch.gradle.test.RestIntegTestTask import java.util.concurrent.Callable apply plugin: 'java' +apply plugin: 'java-test-fixtures' apply plugin: 'idea' apply plugin: 'opensearch.opensearchplugin' apply plugin: 'opensearch.pluginzip' @@ -145,6 +146,7 @@ dependencies { compileOnly fileTree(dir: knnJarDirectory, include: '*.jar') api group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}" implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.10' + testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}" // ml-common excluded reflection for runtime so we need to add it by ourselves. // https://github.com/opensearch-project/ml-commons/commit/464bfe34c66d7a729a00dd457f03587ea4e504d9 // TODO: Remove following three lines of dependencies if ml-common include them in their jar @@ -154,6 +156,10 @@ dependencies { runtimeOnly group: 'org.apache.commons', name: 'commons-text', version: '1.10.0' runtimeOnly group: 'com.google.code.gson', name: 'gson', version: '2.10.1' runtimeOnly group: 'org.json', name: 'json', version: '20231013' + testFixturesImplementation "org.opensearch:common-utils:${version}" + testFixturesImplementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.10' + testFixturesCompileOnly group: 'com.google.guava', name: 'guava', version:'32.0.1-jre' + testFixturesCompileOnly fileTree(dir: knnJarDirectory, include: '*.jar') } // In order to add the jar to the classpath, we need to unzip the @@ -173,6 +179,9 @@ compileJava { compileTestJava { options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) } +compileTestFixturesJava { + options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) +} def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile opensearch_tmp_dir.mkdirs() diff --git a/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java index 7e7660457..b608dbcd8 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java @@ -5,6 +5,8 @@ package org.opensearch.neuralsearch.processor; +import static org.opensearch.neuralsearch.TestUtils.TEST_DIMENSION; +import static org.opensearch.neuralsearch.TestUtils.TEST_SPACE_TYPE; import static org.opensearch.neuralsearch.TestUtils.createRandomVector; import java.util.Collections; @@ -15,8 +17,7 @@ import org.junit.After; import org.junit.Before; import org.opensearch.common.settings.Settings; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.neuralsearch.common.BaseNeuralSearchIT; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; import com.google.common.primitives.Floats; @@ -27,8 +28,6 @@ public class NeuralQueryEnricherProcessorIT extends BaseNeuralSearchIT { private static final String search_pipeline = "search-pipeline"; private static final String ingest_pipeline = "nlp-pipeline"; private static final String TEST_KNN_VECTOR_FIELD_NAME_1 = "test-knn-vector-1"; - private static final int TEST_DIMENSION = 768; - private static final SpaceType TEST_SPACE_TYPE = SpaceType.L2; private final float[] testVector = createRandomVector(TEST_DIMENSION); @Before diff --git a/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorIT.java index 86e75f736..441c5221b 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorIT.java @@ -5,6 +5,9 @@ package org.opensearch.neuralsearch.processor; +import static org.opensearch.neuralsearch.TestUtils.RELATION_EQUAL_TO; +import static org.opensearch.neuralsearch.TestUtils.TEST_DIMENSION; +import static org.opensearch.neuralsearch.TestUtils.TEST_SPACE_TYPE; import static org.opensearch.neuralsearch.TestUtils.createRandomVector; import java.io.IOException; @@ -23,8 +26,7 @@ import org.junit.Before; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.TermQueryBuilder; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.neuralsearch.common.BaseNeuralSearchIT; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; import org.opensearch.neuralsearch.query.HybridQueryBuilder; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; @@ -47,14 +49,11 @@ public class NormalizationProcessorIT extends BaseNeuralSearchIT { private static final String TEST_KNN_VECTOR_FIELD_NAME_1 = "test-knn-vector-1"; private static final String TEST_TEXT_FIELD_NAME_1 = "test-text-field-1"; private static final String TEST_TEXT_FIELD_NAME_2 = "test-text-field-2"; - private static final int TEST_DIMENSION = 768; - private static final SpaceType TEST_SPACE_TYPE = SpaceType.L2; private static final String SEARCH_PIPELINE = "phase-results-pipeline"; private final float[] testVector1 = createRandomVector(TEST_DIMENSION); private final float[] testVector2 = createRandomVector(TEST_DIMENSION); private final float[] testVector3 = createRandomVector(TEST_DIMENSION); private final float[] testVector4 = createRandomVector(TEST_DIMENSION); - private final static String RELATION_EQUAL_TO = "eq"; @Before public void setUp() throws Exception { diff --git a/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationIT.java b/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationIT.java index 4993df7fb..50411c7c6 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationIT.java @@ -7,6 +7,8 @@ import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; +import static org.opensearch.neuralsearch.TestUtils.TEST_DIMENSION; +import static org.opensearch.neuralsearch.TestUtils.TEST_SPACE_TYPE; import static org.opensearch.neuralsearch.TestUtils.assertHybridSearchResults; import static org.opensearch.neuralsearch.TestUtils.assertWeightedScores; import static org.opensearch.neuralsearch.TestUtils.createRandomVector; @@ -22,8 +24,7 @@ import org.junit.Before; import org.opensearch.client.ResponseException; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.neuralsearch.common.BaseNeuralSearchIT; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; import org.opensearch.neuralsearch.query.HybridQueryBuilder; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; @@ -42,8 +43,6 @@ public class ScoreCombinationIT extends BaseNeuralSearchIT { private static final String TEST_DOC_TEXT5 = "Say hello and enter my friend"; private static final String TEST_KNN_VECTOR_FIELD_NAME_1 = "test-knn-vector-1"; private static final String TEST_TEXT_FIELD_NAME_1 = "test-text-field-1"; - private static final int TEST_DIMENSION = 768; - private static final SpaceType TEST_SPACE_TYPE = SpaceType.L2; private static final String SEARCH_PIPELINE = "phase-results-pipeline"; private final float[] testVector1 = createRandomVector(TEST_DIMENSION); private final float[] testVector2 = createRandomVector(TEST_DIMENSION); diff --git a/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationIT.java b/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationIT.java index aa133c44d..a81db429e 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationIT.java @@ -5,6 +5,8 @@ package org.opensearch.neuralsearch.processor; +import static org.opensearch.neuralsearch.TestUtils.TEST_DIMENSION; +import static org.opensearch.neuralsearch.TestUtils.TEST_SPACE_TYPE; import static org.opensearch.neuralsearch.TestUtils.assertHybridSearchResults; import static org.opensearch.neuralsearch.TestUtils.createRandomVector; @@ -18,8 +20,7 @@ import org.junit.After; import org.junit.Before; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.neuralsearch.common.BaseNeuralSearchIT; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; import org.opensearch.neuralsearch.query.HybridQueryBuilder; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; @@ -34,8 +35,6 @@ public class ScoreNormalizationIT extends BaseNeuralSearchIT { private static final String TEST_DOC_TEXT4 = "Hello, I'm glad to you see you pal"; private static final String TEST_KNN_VECTOR_FIELD_NAME_1 = "test-knn-vector-1"; private static final String TEST_TEXT_FIELD_NAME_1 = "test-text-field-1"; - private static final int TEST_DIMENSION = 768; - private static final SpaceType TEST_SPACE_TYPE = SpaceType.L2; private static final String SEARCH_PIPELINE = "phase-results-pipeline"; private final float[] testVector1 = createRandomVector(TEST_DIMENSION); private final float[] testVector2 = createRandomVector(TEST_DIMENSION); diff --git a/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessIT.java b/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessIT.java index 893b47ede..cfa926311 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessIT.java @@ -18,7 +18,7 @@ import org.opensearch.client.Response; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.neuralsearch.common.BaseSparseEncodingIT; +import org.opensearch.neuralsearch.BaseSparseEncodingIT; import com.google.common.collect.ImmutableList; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java index de7a70add..58343e3f2 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java @@ -18,7 +18,7 @@ import org.opensearch.client.Response; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.neuralsearch.common.BaseNeuralSearchIT; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; import com.google.common.collect.ImmutableList; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java index 055344352..873204157 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java @@ -18,7 +18,7 @@ import org.opensearch.client.Response; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.neuralsearch.common.BaseNeuralSearchIT; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; import com.google.common.collect.ImmutableList; diff --git a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryIT.java b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryIT.java index 4a8f0d065..bfe55f1f7 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryIT.java +++ b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryIT.java @@ -9,6 +9,9 @@ import static org.hamcrest.Matchers.containsString; import static org.opensearch.index.query.QueryBuilders.matchQuery; import static org.opensearch.neuralsearch.TestUtils.DELTA_FOR_SCORE_ASSERTION; +import static org.opensearch.neuralsearch.TestUtils.RELATION_EQUAL_TO; +import static org.opensearch.neuralsearch.TestUtils.TEST_DIMENSION; +import static org.opensearch.neuralsearch.TestUtils.TEST_SPACE_TYPE; import static org.opensearch.neuralsearch.TestUtils.createRandomVector; import java.io.IOException; @@ -31,8 +34,7 @@ import org.opensearch.index.query.NestedQueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.TermQueryBuilder; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.neuralsearch.common.BaseNeuralSearchIT; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; import com.google.common.primitives.Floats; @@ -55,9 +57,6 @@ public class HybridQueryIT extends BaseNeuralSearchIT { private static final String TEST_KNN_VECTOR_FIELD_NAME_2 = "test-knn-vector-2"; private static final String TEST_TEXT_FIELD_NAME_1 = "test-text-field-1"; private static final String TEST_NESTED_TYPE_FIELD_NAME_1 = "user"; - - private static final int TEST_DIMENSION = 768; - private static final SpaceType TEST_SPACE_TYPE = SpaceType.L2; private static final String NESTED_FIELD_1 = "firstname"; private static final String NESTED_FIELD_2 = "lastname"; private static final String NESTED_FIELD_1_VALUE = "john"; @@ -65,7 +64,6 @@ public class HybridQueryIT extends BaseNeuralSearchIT { private final float[] testVector1 = createRandomVector(TEST_DIMENSION); private final float[] testVector2 = createRandomVector(TEST_DIMENSION); private final float[] testVector3 = createRandomVector(TEST_DIMENSION); - private final static String RELATION_EQUAL_TO = "eq"; private static final String SEARCH_PIPELINE = "phase-results-pipeline"; @Before diff --git a/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryIT.java b/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryIT.java index 6e089500b..25a2b5d85 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryIT.java +++ b/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryIT.java @@ -5,6 +5,8 @@ package org.opensearch.neuralsearch.query; +import static org.opensearch.neuralsearch.TestUtils.TEST_DIMENSION; +import static org.opensearch.neuralsearch.TestUtils.TEST_SPACE_TYPE; import static org.opensearch.neuralsearch.TestUtils.createRandomVector; import static org.opensearch.neuralsearch.TestUtils.objectToFloat; @@ -19,8 +21,7 @@ import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.MatchQueryBuilder; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.neuralsearch.common.BaseNeuralSearchIT; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; import com.google.common.primitives.Floats; @@ -36,9 +37,6 @@ public class NeuralQueryIT extends BaseNeuralSearchIT { private static final String TEST_KNN_VECTOR_FIELD_NAME_2 = "test-knn-vector-2"; private static final String TEST_TEXT_FIELD_NAME_1 = "test-text-field"; private static final String TEST_KNN_VECTOR_FIELD_NAME_NESTED = "nested.knn.field"; - - private static final int TEST_DIMENSION = 768; - private static final SpaceType TEST_SPACE_TYPE = SpaceType.L2; private final float[] testVector = createRandomVector(TEST_DIMENSION); @Before diff --git a/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryIT.java b/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryIT.java index 0a2463ecd..9177b74b7 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryIT.java +++ b/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryIT.java @@ -18,8 +18,8 @@ import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.neuralsearch.BaseSparseEncodingIT; import org.opensearch.neuralsearch.TestUtils; -import org.opensearch.neuralsearch.common.BaseSparseEncodingIT; public class NeuralSparseQueryIT extends BaseSparseEncodingIT { private static final String TEST_BASIC_INDEX_NAME = "test-sparse-basic-index"; diff --git a/src/test/java/org/opensearch/neuralsearch/common/BaseNeuralSearchIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java similarity index 99% rename from src/test/java/org/opensearch/neuralsearch/common/BaseNeuralSearchIT.java rename to src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java index e3e57a141..7d7caed8f 100644 --- a/src/test/java/org/opensearch/neuralsearch/common/BaseNeuralSearchIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.neuralsearch.common; +package org.opensearch.neuralsearch; import static org.opensearch.neuralsearch.common.VectorUtil.vectorAsListToArray; @@ -49,7 +49,6 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.knn.index.SpaceType; -import org.opensearch.neuralsearch.OpenSearchSecureRestTestCase; import org.opensearch.neuralsearch.util.NeuralSearchClusterUtil; import org.opensearch.test.ClusterServiceUtils; import org.opensearch.threadpool.TestThreadPool; diff --git a/src/test/java/org/opensearch/neuralsearch/common/BaseSparseEncodingIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java similarity index 99% rename from src/test/java/org/opensearch/neuralsearch/common/BaseSparseEncodingIT.java rename to src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java index d0231cfe6..665934805 100644 --- a/src/test/java/org/opensearch/neuralsearch/common/BaseSparseEncodingIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.neuralsearch.common; +package org.opensearch.neuralsearch; import java.nio.file.Files; import java.nio.file.Path; diff --git a/src/test/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java b/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java similarity index 100% rename from src/test/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java rename to src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java diff --git a/src/test/java/org/opensearch/neuralsearch/TestUtils.java b/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java similarity index 93% rename from src/test/java/org/opensearch/neuralsearch/TestUtils.java rename to src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java index 385855a2e..1d6c12eb4 100644 --- a/src/test/java/org/opensearch/neuralsearch/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java @@ -27,6 +27,7 @@ import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.knn.index.SpaceType; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.fetch.FetchSearchResult; @@ -34,8 +35,19 @@ public class TestUtils { - private static final String RELATION_EQUAL_TO = "eq"; + public static final String RELATION_EQUAL_TO = "eq"; public static final float DELTA_FOR_SCORE_ASSERTION = 0.001f; + public static final String RESTART_UPGRADE_OLD_CLUSTER = "tests.is_old_cluster"; + public static final String BWC_VERSION = "tests.plugin_bwc_version"; + public static final String NEURAL_SEARCH_BWC_PREFIX = "neuralsearch-bwc-"; + public static final String OLD_CLUSTER = "old_cluster"; + public static final String MIXED_CLUSTER = "mixed_cluster"; + public static final String UPGRADED_CLUSTER = "upgraded_cluster"; + public static final String ROLLING_UPGRADE_FIRST_ROUND = "tests.rest.first_round"; + public static final String BWCSUITE_CLUSTER = "tests.rest.bwcsuite_cluster"; + public static final String NODES_BWC_CLUSTER = "3"; + public static final int TEST_DIMENSION = 768; + public static final SpaceType TEST_SPACE_TYPE = SpaceType.L2; /** * Convert an xContentBuilder to a map From 724bd3604ac5715126e80943ece64840cefac47d Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Thu, 7 Dec 2023 15:04:12 -0800 Subject: [PATCH 02/45] Initial commit of BWC Test Signed-off-by: Varun Jain --- ...backwards_compatibility_tests_workflow.yml | 147 ++++++++++++++++++ build.gradle | 20 +++ gradle.properties | 3 + qa/build.gradle | 114 ++++++++++++++ qa/restart-upgrade/build.gradle | 115 ++++++++++++++ .../AbstractRestartUpgradeRestTestCase.java | 62 ++++++++ .../neuralsearch/bwc/TextSearch.java | 9 ++ qa/rolling-upgrade/build.gradle | 108 +++++++++++++ .../bwc/AbstractRollingUpgradeTestCase.java | 90 +++++++++++ repositories.gradle | 11 ++ settings.gradle | 4 + .../opensearch/neuralsearch/TestUtils.java | 1 + 12 files changed, 684 insertions(+) create mode 100644 .github/workflows/backwards_compatibility_tests_workflow.yml create mode 100644 qa/build.gradle create mode 100644 qa/restart-upgrade/build.gradle create mode 100644 qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java create mode 100644 qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java create mode 100644 qa/rolling-upgrade/build.gradle create mode 100644 qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java create mode 100644 repositories.gradle diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml new file mode 100644 index 000000000..2c69dc441 --- /dev/null +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -0,0 +1,147 @@ +name: Backwards Compatibility Tests NeuralSearch +on: + push: + branches: + - "*" + - "feature/**" + pull_request: + branches: + - "*" + - "feature/**" + +jobs: + Restart-Upgrade-BWCTests-NeuralSearch: + strategy: + matrix: + java: [ 11, 17 ] + os: [ubuntu-latest] + bwc_version : [ "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0-SNAPSHOT"] + opensearch_version : [ "3.0.0-SNAPSHOT" ] + exclude: + - os: windows-latest + bwc_version: "2.0.1" + - os: windows-latest + bwc_version: "2.1.0" + - os: windows-latest + bwc_version: "2.2.1" + - os: windows-latest + bwc_version: "2.3.0" + + name: NeuralSearch Restart-Upgrade BWC Tests + runs-on: ${{ matrix.os }} + env: + BWC_VERSION_RESTART_UPGRADE: ${{ matrix.bwc_version }} + + steps: + - name: Checkout NeuralSearch + uses: actions/checkout@v1 + + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - if: startsWith(matrix.os,'ubuntu') + name: Install dependencies on ubuntu + run: | + sudo apt-get install libopenblas-dev gfortran -y + + - if: startsWith(matrix.os,'windows') + name: Install MinGW Using Scoop on Windows + run: | + iex "& {$(irm get.scoop.sh)} -RunAsAdmin" + scoop bucket add main + scoop install mingw + + - if: startsWith(matrix.os,'windows') + name: Add MinGW to PATH on Windows + run: | + echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH + refreshenv + + - if: startsWith(matrix.os,'windows') + name: Download OpenBLAS on Windows + run: | + curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip + mkdir OpenBLAS + Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\ + mkdir ./src/main/resources/windowsDependencies + cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/ + rm .\OpenBLAS-0.3.21-x64.zip + rm -r .\OpenBLAS\ + + - if: startsWith(matrix.os,'windows') + name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Windows + run: | + echo "Running restart-upgrade backwards compatibility tests ..." + ./gradlew :qa:restart-upgrade:testRestartUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}' + + - if: startsWith(matrix.os,'ubuntu') + name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Ubuntu + run: | + echo "Running restart-upgrade backwards compatibility tests ..." + ./gradlew :qa:restart-upgrade:testRestartUpgrade -Dtests.bwc.version=$BWC_VERSION_RESTART_UPGRADE + + + Rolling-Upgrade-BWCTests-NeuralSearch: + strategy: + matrix: + java: [ 11, 17 ] + os: [ubuntu-latest] + bwc_version: [ "2.11.0-SNAPSHOT" ] + opensearch_version: [ "3.0.0-SNAPSHOT" ] + + name: NeuralSearch Rolling-Upgrade BWC Tests + runs-on: ${{ matrix.os }} + env: + BWC_VERSION_ROLLING_UPGRADE: ${{ matrix.bwc_version }} + + steps: + - name: Checkout NeuralSearch + uses: actions/checkout@v1 + + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - if: startsWith(matrix.os,'ubuntu') + name: Install dependencies on ubuntu + run: | + sudo apt-get install libopenblas-dev gfortran -y + + - if: startsWith(matrix.os,'windows') + name: Install MinGW Using Scoop on Windows + run: | + iex "& {$(irm get.scoop.sh)} -RunAsAdmin" + scoop bucket add main + scoop install mingw + + - if: startsWith(matrix.os,'windows') + name: Add MinGW to PATH on Windows + run: | + echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH + refreshenv + + - if: startsWith(matrix.os,'windows') + name: Download OpenBLAS on Windows + run: | + curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip + mkdir OpenBLAS + Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\ + mkdir ./src/main/resources/windowsDependencies + cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/ + rm .\OpenBLAS-0.3.21-x64.zip + rm -r .\OpenBLAS\ + + - if: startsWith(matrix.os,'windows') + name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Windows + run: | + echo "Running rolling-upgrade backwards compatibility tests ..." + ./gradlew :qa:rolling-upgrade:testRollingUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}' + + - if: startsWith(matrix.os,'ubuntu') + name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Ubuntu + run: | + echo "Running rolling-upgrade backwards compatibility tests ..." + ./gradlew :qa:rolling-upgrade:testRollingUpgrade -Dtests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE \ No newline at end of file diff --git a/build.gradle b/build.gradle index f2b545eca..9d9115327 100644 --- a/build.gradle +++ b/build.gradle @@ -124,6 +124,26 @@ allprojects { version = "${opensearch_build}" targetCompatibility = JavaVersion.VERSION_11 sourceCompatibility = JavaVersion.VERSION_11 + + apply from: rootProject.file('repositories.gradle').absoluteFile + plugins.withId('java') { + sourceCompatibility = targetCompatibility = "11" + } + + afterEvaluate { + project.dependencyLicenses.enabled = false + project.thirdPartyAudit.enabled = false + project.loggerUsageCheck.enabled = false + project.forbiddenApis.ignoreFailures = true + project.forbiddenPatterns { + setEnabled(false) + } + project.testingConventions.enabled = false + project.validateNebulaPom.enabled = false + project.licenseFile = rootProject.file('LICENSE.txt') + project.noticeFile = rootProject.file('NOTICE.txt') + project.forbiddenApis.ignoreFailures = true + } } repositories { diff --git a/gradle.properties b/gradle.properties index 5e5cd9ced..5d73e7e51 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,6 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 # +version=2.0.0 +systemProp.bwc.version=2.10.0 + # For fixing Spotless check with Java 17 org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \ diff --git a/qa/build.gradle b/qa/build.gradle new file mode 100644 index 000000000..c8342970d --- /dev/null +++ b/qa/build.gradle @@ -0,0 +1,114 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import org.apache.tools.ant.taskdefs.condition.Os +import java.nio.file.Files + +apply plugin: 'opensearch.testclusters' +apply plugin: 'opensearch.build' +apply plugin: 'opensearch.rest-test' +apply plugin: 'io.freefair.lombok' + +// Disable a few tasks that come with build +build.enabled = false +integTest.enabled = false +test.enabled = false +assemble.enabled = false +dependenciesInfo.enabled = false + +repositories { + mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } +} + +dependencies { + api "org.opensearch:opensearch:${opensearch_version}" + api group: 'commons-lang', name: 'commons-lang', version: '2.6' + api "org.apache.logging.log4j:log4j-api:${versions.log4j}" + api "org.apache.logging.log4j:log4j-core:${versions.log4j}" + api "junit:junit:${versions.junit}" + testImplementation "org.opensearch.test:framework:${opensearch_version}" + testImplementation(testFixtures(rootProject)) +} + +ext { + licenseFile = rootProject.file('LICENSE.txt') + noticeFile = rootProject.file('NOTICE.txt') +} + +def tmp_dir = project.file('build/private/artifact_tmp').absoluteFile +tmp_dir.mkdirs() +String default_bwc_version = System.getProperty("bwc.version") +String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) +boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") +String neural_search_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version + +String os_platform = "linux" +String artifact_type = "tar" +String file_ext = "tar.gz" + +if (Os.isFamily(Os.FAMILY_WINDOWS)) { + os_platform = "windows" + artifact_type = "zip" + file_ext = "zip" +} + +// Task to pull neural search plugin from archive +task pullBwcPlugin { + doFirst { + File[] tempFiles = tmp_dir.listFiles() + for (File child : tempFiles) { + if (child.exists() && child.toString().contains("opensearch-")) { + Files.delete(child.toPath()); + } + } + } + + doLast { + ext{ + if (isSnapshot) { + srcUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${neural_search_bwc_version_no_qualifier}/latest/${os_platform}/x64/${artifact_type}/dist/opensearch/opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}" + } else { + srcUrl = "https://artifacts.opensearch.org/releases/bundle/opensearch/${neural_search_bwc_version}/opensearch-${neural_search_bwc_version}-${os_platform}-x64.${file_ext}" + } + } + ant.get( + src: srcUrl, + dest: tmp_dir.absolutePath, + httpusecaches: false + ) + copy { + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + from zipTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) + } else { + from tarTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) + } + into tmp_dir.absolutePath + } + copy { + from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-neural-search")) + into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search") + } + delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}"), java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}") + } +} + +// Task to zip plugin from archive +task zipBwcPlugin(type: Zip) { + dependsOn "pullBwcPlugin" + from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search")) + destinationDirectory = tmp_dir + archiveFileName = "opensearch-neural-search-${neural_search_bwc_version_no_qualifier}.zip" + doLast { + delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search") + } +} + +task bwcTestSuite { + dependsOn ":qa:restart-upgrade:testRestartUpgrade" + dependsOn ":qa:rolling-upgrade:testRollingUpgrade" +} \ No newline at end of file diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle new file mode 100644 index 000000000..e3f8e4e93 --- /dev/null +++ b/qa/restart-upgrade/build.gradle @@ -0,0 +1,115 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask +import org.apache.tools.ant.taskdefs.condition.Os + +apply from : "$rootDir/qa/build.gradle" + +String default_bwc_version = System.getProperty("bwc.version") +String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) +boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") +String neural_search_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version +String baseName = "neuralSearchBwcCluster-restart" + +// Creates a test cluster of previous version and loads k-NN plugin of bwcVersion +testClusters { + "${baseName}" { + testDistribution = "ARCHIVE" + versions = [neural_search_bwc_version, opensearch_version] + numberOfNodes = 3 + plugin(project.tasks.zipBwcPlugin.archiveFile) + setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" + setting 'http.content_type.required', 'true' +// environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" +// if (Os.isFamily(Os.FAMILY_WINDOWS)) { +// // While running on Windows OS, setting the PATH environment variable to include the paths to dlls of JNI libraries and windows dependencies +// environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies") +// systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" +// } else { +// systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" +// } + + } +} + +// Task to run BWC tests against the old cluster +task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { + dependsOn "zipBwcPlugin" + useCluster testClusters."${baseName}" + systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster' + systemProperty 'tests.is_old_cluster', 'true' + systemProperty 'tests.skip_delete_model_index', 'true' + systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version + + // Skip test if version is 1.0 or 1.1 as they are not supported in those versions + if (neural_search_bwc_version.startsWith("1.0") || neural_search_bwc_version.startsWith("1.1")) { + filter { +// excludeTestsMatching "org.opensearch.knn.bwc.ModelIT" +// excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexCustomMethodFieldMapping" +// excludeTestsMatching "org.opensearch.knn.bwc.WarmupIT.testKNNWarmupCustomMethodFieldMapping" + } + } + + // Skip test if version is 1.2 or 1.3 as they are not supported in those versions + if (neural_search_bwc_version.startsWith("1.2") || neural_search_bwc_version.startsWith("1.3")) { + filter { + // excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testNullParametersOnUpgrade" + } + } + + // Skip test if version is 1.0, 1.1, 1.2 or 1.3 as they are not supported in those versions + if (neural_search_bwc_version.startsWith("1.") || neural_search_bwc_version.startsWith("2.")) { + filter { + // excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testEmptyParametersOnUpgrade" + } + } + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + systemProperty 'tests.security.manager', 'false' +} + +// All nodes are upgraded to latest version and run the tests +task testRestartUpgrade(type: StandaloneRestIntegTestTask) { + dependsOn "testAgainstOldCluster" + //dependsOn rootProject.tasks.buildJniLib + dependsOn rootProject.tasks.assemble + useCluster testClusters."${baseName}" + doFirst { + testClusters."${baseName}".environment("LD_LIBRARY_PATH", "$rootDir/jni/release") + testClusters."${baseName}".systemProperty("java.library.path", "$rootDir/jni/release") + testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion([rootProject.tasks.bundlePlugin.archiveFile]) + } + systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster' + systemProperty 'tests.skip_delete_model_index', 'true' + systemProperty 'tests.is_old_cluster', 'false' + systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version + + // Skip test if version is 1.0 or 1.1 as they are not supported in those versions + if (neural_search_bwc_version.startsWith("1.0") || neural_search_bwc_version.startsWith("1.1")) { + filter { +// excludeTestsMatching "org.opensearch.knn.bwc.ModelIT" +// excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexCustomMethodFieldMapping" +// excludeTestsMatching "org.opensearch.knn.bwc.WarmupIT.testKNNWarmupCustomMethodFieldMapping" + } + } + + // Skip test if version is 1.2 or 1.3 as they are not supported in those versions + if (neural_search_bwc_version.startsWith("1.2") || neural_search_bwc_version.startsWith("1.3")) { + filter { + //excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testNullParametersOnUpgrade" + } + } + + // Skip test if version is 1.0, 1.1, 1.2 or 1.3 as they are not supported in those versions + if (neural_search_bwc_version.startsWith("1.") || neural_search_bwc_version.startsWith("2.")) { + filter { + //excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testEmptyParametersOnUpgrade" + } + } + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + systemProperty 'tests.security.manager', 'false' +} \ No newline at end of file diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java new file mode 100644 index 000000000..aa2fbbe0b --- /dev/null +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.neuralsearch.bwc; + +import java.util.Locale; +import java.util.Optional; +import org.junit.Before; +import org.opensearch.common.settings.Settings; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; +import static org.opensearch.neuralsearch.TestUtils.CLIENT_TIMEOUT_VALUE; +import static org.opensearch.neuralsearch.TestUtils.RESTART_UPGRADE_OLD_CLUSTER; +import static org.opensearch.neuralsearch.TestUtils.BWC_VERSION; +import static org.opensearch.neuralsearch.TestUtils.NEURAL_SEARCH_BWC_PREFIX; +import org.opensearch.test.rest.OpenSearchRestTestCase; + +public abstract class AbstractRestartUpgradeRestTestCase extends BaseNeuralSearchIT { + protected static String testIndex; + + @Before + protected void setIndex() { + // Creating index name by concatenating "knn-bwc-" prefix with test method name + // for all the tests in this sub-project + testIndex = NEURAL_SEARCH_BWC_PREFIX + getTestName().toLowerCase(Locale.ROOT); + } + + @Override + protected final boolean preserveIndicesUponCompletion() { + return true; + } + + @Override + protected final boolean preserveReposUponCompletion() { + return true; + } + + @Override + protected boolean preserveTemplatesUponCompletion() { + return true; + } + + @Override + protected final Settings restClientSettings() { + return Settings.builder() + .put(super.restClientSettings()) + // increase the timeout here to 90 seconds to handle long waits for a green + // cluster health. the waits for green need to be longer than a minute to + // account for delayed shards + .put(OpenSearchRestTestCase.CLIENT_SOCKET_TIMEOUT, CLIENT_TIMEOUT_VALUE) + .build(); + } + + protected static final boolean isRunningAgainstOldCluster() { + return Boolean.parseBoolean(System.getProperty(RESTART_UPGRADE_OLD_CLUSTER)); + } + + protected final Optional getBWCVersion() { + return Optional.ofNullable(System.getProperty(BWC_VERSION, null)); + } +} \ No newline at end of file diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java new file mode 100644 index 000000000..c3232815d --- /dev/null +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java @@ -0,0 +1,9 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.neuralsearch.bwc; + +public class TextSearch { +} diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle new file mode 100644 index 000000000..863bfca9a --- /dev/null +++ b/qa/rolling-upgrade/build.gradle @@ -0,0 +1,108 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask +import org.apache.tools.ant.taskdefs.condition.Os + +apply from : "$rootDir/qa/build.gradle" + +String default_bwc_version = System.getProperty("bwc.version") +String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) +boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") +String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version +String baseName = "neuralSearchBwcCluster-rolling" + +// Creates a test cluster of previous version and loads k-NN plugin of bwcVersion +testClusters { + "${baseName}" { + testDistribution = "ARCHIVE" + versions = [neural_search_bwc_version, opensearch_version] + numberOfNodes = 3 + plugin(project.tasks.zipBwcPlugin.archiveFile) + setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" + setting 'http.content_type.required', 'true' +// environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" +// if (Os.isFamily(Os.FAMILY_WINDOWS)) { +// // While running on Windows OS, setting the PATH environment variable to include the paths to dlls of JNI libraries and windows dependencies +// environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies") +// systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" +// } else { +// systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" +// } + } +} + + +// Task to run BWC tests against the old cluster +task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { + dependsOn "zipBwcPlugin" + useCluster testClusters."${baseName}" + systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster' + systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version + systemProperty 'tests.skip_delete_model_index', 'true' + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + systemProperty 'tests.security.manager', 'false' +} + + +// Part of rolling upgrade. Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version +// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node. +task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) { + useCluster testClusters."${baseName}" + //dependsOn rootProject.tasks.buildJniLib + dependsOn rootProject.tasks.assemble + dependsOn "testAgainstOldCluster" + doFirst { + testClusters."${baseName}".getNodes().getAt("${baseName}" + "-0").environment("LD_LIBRARY_PATH", "$rootDir/jni/release") + testClusters."${baseName}".getNodes().getAt("${baseName}" + "-0").systemProperty("java.library.path", "$rootDir/jni/release") + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion([rootProject.tasks.bundlePlugin.archiveFile]) + } + systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster' + systemProperty 'tests.rest.first_round', 'true' + systemProperty 'tests.skip_delete_model_index', 'true' + systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + systemProperty 'tests.security.manager', 'false' +} + +// Part of rolling upgrade. Upgrades the second node to new OpenSearch version with upgraded plugin version after the +// first node is upgraded. This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes. +task testAgainstTwoThirdsUpgradedCluster(type: StandaloneRestIntegTestTask) { + dependsOn "testAgainstOneThirdUpgradedCluster" + useCluster testClusters."${baseName}" + doFirst { + testClusters."${baseName}".getNodes().getAt("${baseName}" + "-1").environment("LD_LIBRARY_PATH", "$rootDir/jni/release") + testClusters."${baseName}".getNodes().getAt("${baseName}" + "-1").systemProperty("java.library.path", "$rootDir/jni/release") + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion([rootProject.tasks.bundlePlugin.archiveFile]) + } + systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster' + systemProperty 'tests.rest.first_round', 'false' + systemProperty 'tests.skip_delete_model_index', 'true' + systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + systemProperty 'tests.security.manager', 'false' +} + +// Part of rolling upgrade. Upgrades the third node to new OpenSearch version with upgraded plugin version after the +// second node is upgraded. This results in a fully upgraded cluster. +task testRollingUpgrade(type: StandaloneRestIntegTestTask) { + dependsOn "testAgainstTwoThirdsUpgradedCluster" + useCluster testClusters."${baseName}" + doFirst { + testClusters."${baseName}".getNodes().getAt("${baseName}" + "-2").environment("LD_LIBRARY_PATH", "$rootDir/jni/release") + testClusters."${baseName}".getNodes().getAt("${baseName}" + "-2").systemProperty("java.library.path", "$rootDir/jni/release") + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion([rootProject.tasks.bundlePlugin.archiveFile]) + } + mustRunAfter "testAgainstOneThirdUpgradedCluster" + systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster' + systemProperty 'tests.skip_delete_model_index', 'true' + systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + systemProperty 'tests.security.manager', 'false' +} \ No newline at end of file diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java new file mode 100644 index 000000000..089ade9f3 --- /dev/null +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java @@ -0,0 +1,90 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.neuralsearch.bwc; + +import java.util.Locale; +import java.util.Optional; +import org.junit.Before; +import org.opensearch.common.settings.Settings; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; +import org.opensearch.test.rest.OpenSearchRestTestCase; +import static org.opensearch.neuralsearch.TestUtils.CLIENT_TIMEOUT_VALUE; +import static org.opensearch.neuralsearch.TestUtils.OLD_CLUSTER; +import static org.opensearch.neuralsearch.TestUtils.MIXED_CLUSTER; +import static org.opensearch.neuralsearch.TestUtils.UPGRADED_CLUSTER; +import static org.opensearch.neuralsearch.TestUtils.BWC_VERSION; +import static org.opensearch.neuralsearch.TestUtils.ROLLING_UPGRADE_FIRST_ROUND; +import static org.opensearch.neuralsearch.TestUtils.BWCSUITE_CLUSTER; +import static org.opensearch.neuralsearch.TestUtils.NEURAL_SEARCH_BWC_PREFIX; + +public abstract class AbstractRollingUpgradeTestCase extends BaseNeuralSearchIT { + protected String testIndex; + + @Before + protected void setIndex() { + // Creating index name by concatenating "knn-bwc-" prefix with test method name + // for all the tests in this sub-project + testIndex = NEURAL_SEARCH_BWC_PREFIX + getTestName().toLowerCase(Locale.ROOT); + } + + @Override + protected final boolean preserveIndicesUponCompletion() { + return true; + } + + @Override + protected final boolean preserveReposUponCompletion() { + return true; + } + + @Override + protected boolean preserveTemplatesUponCompletion() { + return true; + } + + @Override + protected final Settings restClientSettings() { + return Settings.builder() + .put(super.restClientSettings()) + // increase the timeout here to 90 seconds to handle long waits for a green + // cluster health. the waits for green need to be longer than a minute to + // account for delayed shards + .put(OpenSearchRestTestCase.CLIENT_SOCKET_TIMEOUT, CLIENT_TIMEOUT_VALUE) + .build(); + } + + protected enum ClusterType { + OLD, + MIXED, + UPGRADED; + + public static ClusterType instance(String value) { + switch (value) { + case OLD_CLUSTER: + return OLD; + case MIXED_CLUSTER: + return MIXED; + case UPGRADED_CLUSTER: + return UPGRADED; + default: + throw new IllegalArgumentException("unknown cluster type: " + value); + } + } + } + + protected final ClusterType getClusterType() { + return ClusterType.instance(System.getProperty(BWCSUITE_CLUSTER)); + } + + protected final boolean isFirstMixedRound() { + return Boolean.parseBoolean(System.getProperty(ROLLING_UPGRADE_FIRST_ROUND, "false")); + } + + protected final Optional getBWCVersion() { + return Optional.ofNullable(System.getProperty(BWC_VERSION, null)); + } + +} \ No newline at end of file diff --git a/repositories.gradle b/repositories.gradle new file mode 100644 index 000000000..0ce55eed3 --- /dev/null +++ b/repositories.gradle @@ -0,0 +1,11 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +repositories { + mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } +} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 57fb25f3c..5bfca1add 100644 --- a/settings.gradle +++ b/settings.gradle @@ -9,3 +9,7 @@ */ rootProject.name = 'neural-search' + +include ":qa" +include ":qa:rolling-upgrade" +include ":qa:restart-upgrade" \ No newline at end of file diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java b/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java index 1d6c12eb4..b07744730 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java @@ -48,6 +48,7 @@ public class TestUtils { public static final String NODES_BWC_CLUSTER = "3"; public static final int TEST_DIMENSION = 768; public static final SpaceType TEST_SPACE_TYPE = SpaceType.L2; + public static final String CLIENT_TIMEOUT_VALUE = "90s"; /** * Convert an xContentBuilder to a map From a9357f2f3d3e216970a31eb116c7dd6e37a2b850 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 19 Dec 2023 10:07:39 -0800 Subject: [PATCH 03/45] Text Search Signed-off-by: Varun Jain --- ...backwards_compatibility_tests_workflow.yml | 24 ++-- gradle.properties | 2 +- qa/build.gradle | 109 +++++++++++++++++- qa/restart-upgrade/build.gradle | 65 +++-------- .../neuralsearch/bwc/TextSearch.java | 60 +++++++++- .../CreateModelGroupRequestBody.json | 4 + .../src/test/resources/IndexMappings.json | 33 ++++++ .../test/resources/PipelineConfiguration.json | 18 +++ .../resources/UploadModelRequestBody.json | 15 +++ .../neuralsearch/BaseNeuralSearchIT.java | 48 ++++++-- 10 files changed, 297 insertions(+), 81 deletions(-) create mode 100644 qa/restart-upgrade/src/test/resources/CreateModelGroupRequestBody.json create mode 100644 qa/restart-upgrade/src/test/resources/IndexMappings.json create mode 100644 qa/restart-upgrade/src/test/resources/PipelineConfiguration.json create mode 100644 qa/restart-upgrade/src/test/resources/UploadModelRequestBody.json diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 2c69dc441..cde0be93b 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -13,19 +13,19 @@ jobs: Restart-Upgrade-BWCTests-NeuralSearch: strategy: matrix: - java: [ 11, 17 ] + java: [ 11, 17, 21 ] os: [ubuntu-latest] - bwc_version : [ "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0-SNAPSHOT"] + bwc_version : ["2.9.0", "2.10.0", "2.11.0-SNAPSHOT"] opensearch_version : [ "3.0.0-SNAPSHOT" ] - exclude: - - os: windows-latest - bwc_version: "2.0.1" - - os: windows-latest - bwc_version: "2.1.0" - - os: windows-latest - bwc_version: "2.2.1" - - os: windows-latest - bwc_version: "2.3.0" +# exclude: +# - os: windows-latest +# bwc_version: "2.0.1" +# - os: windows-latest +# bwc_version: "2.1.0" +# - os: windows-latest +# bwc_version: "2.2.1" +# - os: windows-latest +# bwc_version: "2.3.0" name: NeuralSearch Restart-Upgrade BWC Tests runs-on: ${{ matrix.os }} @@ -86,7 +86,7 @@ jobs: Rolling-Upgrade-BWCTests-NeuralSearch: strategy: matrix: - java: [ 11, 17 ] + java: [ 11, 17, 21 ] os: [ubuntu-latest] bwc_version: [ "2.11.0-SNAPSHOT" ] opensearch_version: [ "3.0.0-SNAPSHOT" ] diff --git a/gradle.properties b/gradle.properties index 5d73e7e51..242771ae4 100644 --- a/gradle.properties +++ b/gradle.properties @@ -4,7 +4,7 @@ # version=2.0.0 -systemProp.bwc.version=2.10.0 +systemProp.bwc.version=2.11.0 # For fixing Spotless check with Java 17 org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ diff --git a/qa/build.gradle b/qa/build.gradle index c8342970d..da875e143 100644 --- a/qa/build.gradle +++ b/qa/build.gradle @@ -4,6 +4,8 @@ */ import org.apache.tools.ant.taskdefs.condition.Os +import org.w3c.dom.Text + import java.nio.file.Files apply plugin: 'opensearch.testclusters' @@ -25,9 +27,18 @@ repositories { maven { url "https://plugins.gradle.org/m2/" } } +configurations { + zipArchive +} + +def knnJarDirectory = "$rootDir/build/dependencies/opensearch-knn" + dependencies { api "org.opensearch:opensearch:${opensearch_version}" - api group: 'commons-lang', name: 'commons-lang', version: '2.6' + zipArchive group: 'org.opensearch.plugin', name:'opensearch-knn', version: "${opensearch_build}" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}" + compileOnly fileTree(dir: knnJarDirectory, include: '*.jar') + //api group: 'commons-lang', name: 'commons-lang', version: '2.6' api "org.apache.logging.log4j:log4j-api:${versions.log4j}" api "org.apache.logging.log4j:log4j-core:${versions.log4j}" api "junit:junit:${versions.junit}" @@ -46,6 +57,7 @@ String default_bwc_version = System.getProperty("bwc.version") String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") String neural_search_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version +String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version String os_platform = "linux" String artifact_type = "tar" @@ -57,8 +69,7 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { file_ext = "zip" } -// Task to pull neural search plugin from archive -task pullBwcPlugin { +task deletetempDirectories { doFirst { File[] tempFiles = tmp_dir.listFiles() for (File child : tempFiles) { @@ -67,6 +78,41 @@ task pullBwcPlugin { } } } +} + +task pullMlCommonsBwcPlugin { + dependsOn "deletetempDirectories" + + doLast { + ext{ + if (isSnapshot) { + srcUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${neural_search_bwc_version_no_qualifier}/latest/${os_platform}/x64/${artifact_type}/dist/opensearch/opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}" + } else { + srcUrl = "https://artifacts.opensearch.org/releases/bundle/opensearch/${neural_search_bwc_version}/opensearch-${neural_search_bwc_version}-${os_platform}-x64.${file_ext}" + } + } + ant.get( + src: srcUrl, + dest: tmp_dir.absolutePath, + httpusecaches: false + ) + copy { + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + from zipTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) + } else { + from tarTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) + } + into tmp_dir.absolutePath + } + copy { + from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-ml")) + into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml") + } + } +} + +task pullKnnBwcPlugin { + dependsOn "deletetempDirectories" doLast { ext{ @@ -90,15 +136,69 @@ task pullBwcPlugin { into tmp_dir.absolutePath } copy { - from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-neural-search")) + from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-knn")) + into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn") + } + } +} + +// Task to pull neural search plugin from archive +task pullBwcPlugin { + dependsOn "deletetempDirectories" + + doLast { + ext{ + if (isSnapshot) { + srcUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${neural_search_bwc_version_no_qualifier}/latest/${os_platform}/x64/${artifact_type}/dist/opensearch/opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}" + } else { + srcUrl = "https://artifacts.opensearch.org/releases/bundle/opensearch/${neural_search_bwc_version}/opensearch-${neural_search_bwc_version}-${os_platform}-x64.${file_ext}" + } + } + ant.get( + src: srcUrl, + dest: tmp_dir.absolutePath, + httpusecaches: false + ) + copy { + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + from zipTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) + } else { + from tarTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) + } + into tmp_dir.absolutePath + } + copy { + from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-neural-search")) into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search") } delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}"), java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}") } } +task zipBwcMlCommonsPlugin(type: Zip) { + dependsOn "pullMlCommonsBwcPlugin" + from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml")) + destinationDirectory = tmp_dir + archiveFileName = "opensearch-ml-${neural_search_bwc_version_no_qualifier}.zip" + doLast { + delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml") + } +} + +task zipBwcKnnPlugin(type: Zip) { + dependsOn "zipBwcMlCommonsPlugin" + dependsOn "pullKnnBwcPlugin" + from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn")) + destinationDirectory = tmp_dir + archiveFileName = "opensearch-knn-${neural_search_bwc_version_no_qualifier}.zip" + doLast { + delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn") + } +} + // Task to zip plugin from archive task zipBwcPlugin(type: Zip) { + dependsOn "zipBwcKnnPlugin" dependsOn "pullBwcPlugin" from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search")) destinationDirectory = tmp_dir @@ -108,6 +208,7 @@ task zipBwcPlugin(type: Zip) { } } + task bwcTestSuite { dependsOn ":qa:restart-upgrade:testRestartUpgrade" dependsOn ":qa:rolling-upgrade:testRollingUpgrade" diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index e3f8e4e93..90852193b 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -13,6 +13,7 @@ String neural_search_bwc_version = System.getProperty("tests.bwc.version", defau boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") String neural_search_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version String baseName = "neuralSearchBwcCluster-restart" +String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version // Creates a test cluster of previous version and loads k-NN plugin of bwcVersion testClusters { @@ -20,23 +21,27 @@ testClusters { testDistribution = "ARCHIVE" versions = [neural_search_bwc_version, opensearch_version] numberOfNodes = 3 + plugin(project.tasks.zipBwcMlCommonsPlugin.archiveFile) + plugin(project.tasks.zipBwcKnnPlugin.archiveFile) plugin(project.tasks.zipBwcPlugin.archiveFile) setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' -// environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" -// if (Os.isFamily(Os.FAMILY_WINDOWS)) { -// // While running on Windows OS, setting the PATH environment variable to include the paths to dlls of JNI libraries and windows dependencies -// environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies") -// systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" -// } else { -// systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" -// } + environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + // While running on Windows OS, setting the PATH environment variable to include the paths to dlls of JNI libraries and windows dependencies + environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies") + systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" + } else { + systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" + } } } // Task to run BWC tests against the old cluster task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { + dependsOn "zipBwcMlCommonsPlugin" + dependsOn "zipBwcKnnPlugin" dependsOn "zipBwcPlugin" useCluster testClusters."${baseName}" systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster' @@ -44,28 +49,6 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { systemProperty 'tests.skip_delete_model_index', 'true' systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version - // Skip test if version is 1.0 or 1.1 as they are not supported in those versions - if (neural_search_bwc_version.startsWith("1.0") || neural_search_bwc_version.startsWith("1.1")) { - filter { -// excludeTestsMatching "org.opensearch.knn.bwc.ModelIT" -// excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexCustomMethodFieldMapping" -// excludeTestsMatching "org.opensearch.knn.bwc.WarmupIT.testKNNWarmupCustomMethodFieldMapping" - } - } - - // Skip test if version is 1.2 or 1.3 as they are not supported in those versions - if (neural_search_bwc_version.startsWith("1.2") || neural_search_bwc_version.startsWith("1.3")) { - filter { - // excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testNullParametersOnUpgrade" - } - } - - // Skip test if version is 1.0, 1.1, 1.2 or 1.3 as they are not supported in those versions - if (neural_search_bwc_version.startsWith("1.") || neural_search_bwc_version.startsWith("2.")) { - filter { - // excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testEmptyParametersOnUpgrade" - } - } nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") systemProperty 'tests.security.manager', 'false' @@ -87,28 +70,6 @@ task testRestartUpgrade(type: StandaloneRestIntegTestTask) { systemProperty 'tests.is_old_cluster', 'false' systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version - // Skip test if version is 1.0 or 1.1 as they are not supported in those versions - if (neural_search_bwc_version.startsWith("1.0") || neural_search_bwc_version.startsWith("1.1")) { - filter { -// excludeTestsMatching "org.opensearch.knn.bwc.ModelIT" -// excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexCustomMethodFieldMapping" -// excludeTestsMatching "org.opensearch.knn.bwc.WarmupIT.testKNNWarmupCustomMethodFieldMapping" - } - } - - // Skip test if version is 1.2 or 1.3 as they are not supported in those versions - if (neural_search_bwc_version.startsWith("1.2") || neural_search_bwc_version.startsWith("1.3")) { - filter { - //excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testNullParametersOnUpgrade" - } - } - - // Skip test if version is 1.0, 1.1, 1.2 or 1.3 as they are not supported in those versions - if (neural_search_bwc_version.startsWith("1.") || neural_search_bwc_version.startsWith("2.")) { - filter { - //excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testEmptyParametersOnUpgrade" - } - } nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") systemProperty 'tests.security.manager', 'false' diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java index c3232815d..b58dacc76 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java @@ -5,5 +5,63 @@ package org.opensearch.neuralsearch.bwc; -public class TextSearch { +import com.carrotsearch.randomizedtesting.RandomizedTest; +import java.nio.file.Files; +import java.nio.file.Path; +import static org.opensearch.neuralsearch.TestUtils.NODES_BWC_CLUSTER; + +public class TextSearch extends AbstractRestartUpgradeRestTestCase{ + + private static final String PIPELINE_NAME = "nlp-pipeline"; + private static String DOC_ID = "0"; + private static final String TEST_FIELD = "test-field"; + private static final String TEXT= "Hello world"; + + public void testIndex() throws Exception{ + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + + if (isRunningAgainstOldCluster()){ + String modelId= uploadTextEmbeddingModel(); + loadModel(modelId); + createPipelineProcessor(modelId,PIPELINE_NAME); + createIndexWithConfiguration( + testIndex, + Files.readString(Path.of(classLoader.getResource("processor/IndexMappings.json").toURI())), + PIPELINE_NAME + ); + addDocument(testIndex, DOC_ID,TEST_FIELD,TEXT); + }else { + System.out.println("===========================================================================================Cluster Upgraded"); + validateTestIndex(); + } + } + + + private void validateTestIndex() throws Exception { + int docCount=getDocCount(testIndex); + assertEquals(1,docCount); + deleteIndex(testIndex); + } + + private String uploadTextEmbeddingModel() throws Exception { + String requestBody = Files.readString(Path.of(classLoader.getResource("processor/UploadModelRequestBody.json").toURI())); + return registerModelGroupAndGetModelId(requestBody); + } + + private String registerModelGroupAndGetModelId(String requestBody) throws Exception { + String modelGroupRegisterRequestBody = Files.readString( + Path.of(classLoader.getResource("processor/CreateModelGroupRequestBody.json").toURI()) + ).replace("", "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)); + + String modelGroupId=registerModelGroup(modelGroupRegisterRequestBody); + + requestBody = requestBody.replace("", modelGroupId); + + return uploadModelId(requestBody); + } + + protected void createPipelineProcessor(String modelId, String pipelineName, ProcessorType processorType) throws Exception { + String requestBody=Files.readString(Path.of(classLoader.getResource("processor/PipelineConfiguration.json").toURI())); + createPipelineProcessor(requestBody,pipelineName,modelId); + } } diff --git a/qa/restart-upgrade/src/test/resources/CreateModelGroupRequestBody.json b/qa/restart-upgrade/src/test/resources/CreateModelGroupRequestBody.json new file mode 100644 index 000000000..91f68e222 --- /dev/null +++ b/qa/restart-upgrade/src/test/resources/CreateModelGroupRequestBody.json @@ -0,0 +1,4 @@ +{ + "name": "", + "description": "This is a public model group" +} \ No newline at end of file diff --git a/qa/restart-upgrade/src/test/resources/IndexMappings.json b/qa/restart-upgrade/src/test/resources/IndexMappings.json new file mode 100644 index 000000000..70ecf8335 --- /dev/null +++ b/qa/restart-upgrade/src/test/resources/IndexMappings.json @@ -0,0 +1,33 @@ +{ + "settings": { + "index": { + "knn": true, + "knn.algo_param.ef_search": 100, + "refresh_interval": "30s", + "default_pipeline": "%s" + }, + "number_of_shards": 1, + "number_of_replicas": 0 + }, + "mappings": { + "properties": { + "passage_embedding": { + "type": "knn_vector", + "dimension": 768, + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "lucene", + "parameters": { + "ef_construction": 128, + "m": 24 + } + } + }, + "passage_text": { + "type": "text" + } + } + } +} + diff --git a/qa/restart-upgrade/src/test/resources/PipelineConfiguration.json b/qa/restart-upgrade/src/test/resources/PipelineConfiguration.json new file mode 100644 index 000000000..5d0bfe90f --- /dev/null +++ b/qa/restart-upgrade/src/test/resources/PipelineConfiguration.json @@ -0,0 +1,18 @@ +{ + "description": "text embedding pipeline for hybrid", + "processors": [ + { + "text_embedding": { + "model_id": "%s", + "field_map": { + "title": "title_knn", + "favor_list": "favor_list_knn", + "favorites": { + "game": "game_knn", + "movie": "movie_knn" + } + } + } + } + ] +} \ No newline at end of file diff --git a/qa/restart-upgrade/src/test/resources/UploadModelRequestBody.json b/qa/restart-upgrade/src/test/resources/UploadModelRequestBody.json new file mode 100644 index 000000000..6769e78c7 --- /dev/null +++ b/qa/restart-upgrade/src/test/resources/UploadModelRequestBody.json @@ -0,0 +1,15 @@ +{ + "name": "traced_small_model", + "version": "1.0.0", + "model_format": "TORCH_SCRIPT", + "model_task_type": "text_embedding", + "model_content_hash_value": "e13b74006290a9d0f58c1376f9629d4ebc05a0f9385f40db837452b167ae9021", + "model_group_id": "", + "model_config": { + "model_type": "bert", + "embedding_dimension": 768, + "framework_type": "sentence_transformers", + "all_config": "{\"architectures\":[\"BertModel\"],\"max_position_embeddings\":512,\"model_type\":\"bert\",\"num_attention_heads\":12,\"num_hidden_layers\":6}" + }, + "url": "https://github.com/opensearch-project/ml-commons/blob/2.x/ml-algorithms/src/test/resources/org/opensearch/ml/engine/algorithms/text_embedding/traced_small_model.zip?raw=true" +} \ No newline at end of file diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java index 62a8af7a8..0ccdc58c3 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java @@ -22,10 +22,7 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; -import org.apache.hc.core5.http.ContentType; -import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpEntity; -import org.apache.hc.core5.http.HttpHeaders; +import org.apache.hc.core5.http.*; import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.http.io.entity.StringEntity; import org.apache.hc.core5.http.message.BasicHeader; @@ -138,7 +135,10 @@ protected String uploadModel(String requestBody) throws Exception { String modelGroupId = registerModelGroup(); // model group id is dynamically generated, we need to update model update request body after group is registered requestBody = requestBody.replace("", modelGroupId); + return uploadModelId(requestBody); + } + protected String uploadModelId(String requestBody) throws Exception { Response uploadResponse = makeRequest( client(), "POST", @@ -269,18 +269,17 @@ protected void createPipelineProcessor(String modelId, String pipelineName) thro } protected void createPipelineProcessor(String modelId, String pipelineName, ProcessorType processorType) throws Exception { + String requestBody = Files.readString(Path.of(classLoader.getResource(PIPELINE_CONFIGS_BY_TYPE.get(processorType)).toURI())); + createPipelineProcessor(requestBody, pipelineName, modelId); + } + + protected void createPipelineProcessor(String requestBody, String pipelineName, String modelId) throws Exception { Response pipelineCreateResponse = makeRequest( client(), "PUT", "/_ingest/pipeline/" + pipelineName, null, - toHttpEntity( - String.format( - LOCALE, - Files.readString(Path.of(classLoader.getResource(PIPELINE_CONFIGS_BY_TYPE.get(processorType)).toURI())), - modelId - ) - ), + toHttpEntity(String.format(LOCALE, requestBody, modelId)), ImmutableList.of(new BasicHeader(HttpHeaders.USER_AGENT, DEFAULT_USER_AGENT)) ); Map node = XContentHelper.convertToMap( @@ -793,6 +792,10 @@ private String registerModelGroup() { String modelGroupRegisterRequestBody = Files.readString( Path.of(classLoader.getResource("processor/CreateModelGroupRequestBody.json").toURI()) ).replace("", "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)); + return registerModelGroup(modelGroupRegisterRequestBody); + } + + protected String registerModelGroup(String modelGroupRegisterRequestBody) throws IOException, ParseException { Response modelGroupResponse = makeRequest( client(), "POST", @@ -811,6 +814,29 @@ private String registerModelGroup() { return modelGroupId; } + // Method that waits till the health of nodes in the cluster goes green + protected void waitForClusterHealthGreen(String numOfNodes) throws IOException { + Request waitForGreen = new Request("GET", "/_cluster/health"); + waitForGreen.addParameter("wait_for_nodes", numOfNodes); + waitForGreen.addParameter("wait_for_status", "green"); + client().performRequest(waitForGreen); + } + + /** + * Add a single Doc to an index + */ + protected void addDocument(String index, String docId, String fieldName, String text) throws IOException { + Request request = new Request("PUT", "/" + index + "/_doc/" + docId + "?refresh=true"); + + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field(fieldName, text).endObject(); + request.setJsonEntity(builder.toString()); + client().performRequest(request); + + request = new Request("POST", "/" + index + "/_refresh"); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + } + /** * Enumeration for types of pipeline processors, used to lookup resources like create * processor request as those are type specific From 8c0f22d5c39947521b90dbb533e8c632a54b538e Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 19 Dec 2023 10:28:51 -0800 Subject: [PATCH 04/45] Fixing bug Signed-off-by: Varun Jain --- .../test/java/org/opensearch/neuralsearch/bwc/TextSearch.java | 1 + .../resources/{ => processor}/CreateModelGroupRequestBody.json | 0 .../src/test/resources/{ => processor}/IndexMappings.json | 0 .../test/resources/{ => processor}/PipelineConfiguration.json | 0 .../test/resources/{ => processor}/UploadModelRequestBody.json | 0 5 files changed, 1 insertion(+) rename qa/restart-upgrade/src/test/resources/{ => processor}/CreateModelGroupRequestBody.json (100%) rename qa/restart-upgrade/src/test/resources/{ => processor}/IndexMappings.json (100%) rename qa/restart-upgrade/src/test/resources/{ => processor}/PipelineConfiguration.json (100%) rename qa/restart-upgrade/src/test/resources/{ => processor}/UploadModelRequestBody.json (100%) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java index b58dacc76..bc99c5f45 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java @@ -29,6 +29,7 @@ public void testIndex() throws Exception{ Files.readString(Path.of(classLoader.getResource("processor/IndexMappings.json").toURI())), PIPELINE_NAME ); + logger.info("=================================================================================Document Added"); addDocument(testIndex, DOC_ID,TEST_FIELD,TEXT); }else { System.out.println("===========================================================================================Cluster Upgraded"); diff --git a/qa/restart-upgrade/src/test/resources/CreateModelGroupRequestBody.json b/qa/restart-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json similarity index 100% rename from qa/restart-upgrade/src/test/resources/CreateModelGroupRequestBody.json rename to qa/restart-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json diff --git a/qa/restart-upgrade/src/test/resources/IndexMappings.json b/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json similarity index 100% rename from qa/restart-upgrade/src/test/resources/IndexMappings.json rename to qa/restart-upgrade/src/test/resources/processor/IndexMappings.json diff --git a/qa/restart-upgrade/src/test/resources/PipelineConfiguration.json b/qa/restart-upgrade/src/test/resources/processor/PipelineConfiguration.json similarity index 100% rename from qa/restart-upgrade/src/test/resources/PipelineConfiguration.json rename to qa/restart-upgrade/src/test/resources/processor/PipelineConfiguration.json diff --git a/qa/restart-upgrade/src/test/resources/UploadModelRequestBody.json b/qa/restart-upgrade/src/test/resources/processor/UploadModelRequestBody.json similarity index 100% rename from qa/restart-upgrade/src/test/resources/UploadModelRequestBody.json rename to qa/restart-upgrade/src/test/resources/processor/UploadModelRequestBody.json From 26eb40114aff2e7bf22823261e8f56ee06fd819a Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Wed, 20 Dec 2023 10:36:27 -0800 Subject: [PATCH 05/45] Text Search bug fix Signed-off-by: Varun Jain --- qa/build.gradle | 43 ------- qa/restart-upgrade/build.gradle | 44 ++++++- .../neuralsearch/bwc/TextSearch.java | 6 +- .../OpenSearchSecureRestTestCase.java | 110 ++++++++++++++++-- .../opensearch/neuralsearch/TestUtils.java | 4 + 5 files changed, 149 insertions(+), 58 deletions(-) diff --git a/qa/build.gradle b/qa/build.gradle index da875e143..8ea7dad0e 100644 --- a/qa/build.gradle +++ b/qa/build.gradle @@ -81,29 +81,7 @@ task deletetempDirectories { } task pullMlCommonsBwcPlugin { - dependsOn "deletetempDirectories" - doLast { - ext{ - if (isSnapshot) { - srcUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${neural_search_bwc_version_no_qualifier}/latest/${os_platform}/x64/${artifact_type}/dist/opensearch/opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}" - } else { - srcUrl = "https://artifacts.opensearch.org/releases/bundle/opensearch/${neural_search_bwc_version}/opensearch-${neural_search_bwc_version}-${os_platform}-x64.${file_ext}" - } - } - ant.get( - src: srcUrl, - dest: tmp_dir.absolutePath, - httpusecaches: false - ) - copy { - if (Os.isFamily(Os.FAMILY_WINDOWS)) { - from zipTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) - } else { - from tarTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) - } - into tmp_dir.absolutePath - } copy { from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-ml")) into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml") @@ -144,29 +122,8 @@ task pullKnnBwcPlugin { // Task to pull neural search plugin from archive task pullBwcPlugin { - dependsOn "deletetempDirectories" doLast { - ext{ - if (isSnapshot) { - srcUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${neural_search_bwc_version_no_qualifier}/latest/${os_platform}/x64/${artifact_type}/dist/opensearch/opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}" - } else { - srcUrl = "https://artifacts.opensearch.org/releases/bundle/opensearch/${neural_search_bwc_version}/opensearch-${neural_search_bwc_version}-${os_platform}-x64.${file_ext}" - } - } - ant.get( - src: srcUrl, - dest: tmp_dir.absolutePath, - httpusecaches: false - ) - copy { - if (Os.isFamily(Os.FAMILY_WINDOWS)) { - from zipTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) - } else { - from tarTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) - } - into tmp_dir.absolutePath - } copy { from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-neural-search")) into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search") diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index 90852193b..fcf3df0e2 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -5,6 +5,7 @@ import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask import org.apache.tools.ant.taskdefs.condition.Os +import java.util.concurrent.Callable apply from : "$rootDir/qa/build.gradle" @@ -15,6 +16,44 @@ String neural_search_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_v String baseName = "neuralSearchBwcCluster-restart" String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version +List> plugins = [provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.matching{include "**/opensearch-ml-plugin-${opensearch_build}.zip"}.getSingleFile() + } + } + } +}), provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.matching{include "**/opensearch-knn-${opensearch_build}.zip"}.getSingleFile() + } + } + } +}), rootProject.tasks.bundlePlugin.archiveFile] + +// task unzip { +// configurations.zipArchive.asFileTree.each { +// println(it.toPath()) +// } +// } + +// def tmp_dir = project.file('build/private/artifact_tmp').absoluteFile + + +// task unzip(type: Copy) { +// configurations.zipArchive.asFileTree.each { +// from(zipTree(it)) +// } +// into tmp_dir +// } + // Creates a test cluster of previous version and loads k-NN plugin of bwcVersion testClusters { "${baseName}" { @@ -57,13 +96,12 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { // All nodes are upgraded to latest version and run the tests task testRestartUpgrade(type: StandaloneRestIntegTestTask) { dependsOn "testAgainstOldCluster" - //dependsOn rootProject.tasks.buildJniLib dependsOn rootProject.tasks.assemble useCluster testClusters."${baseName}" doFirst { testClusters."${baseName}".environment("LD_LIBRARY_PATH", "$rootDir/jni/release") testClusters."${baseName}".systemProperty("java.library.path", "$rootDir/jni/release") - testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion([rootProject.tasks.bundlePlugin.archiveFile]) + testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion(plugins) } systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster' systemProperty 'tests.skip_delete_model_index', 'true' @@ -73,4 +111,4 @@ task testRestartUpgrade(type: StandaloneRestIntegTestTask) { nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") systemProperty 'tests.security.manager', 'false' -} \ No newline at end of file +} diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java index bc99c5f45..7fd707b00 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java @@ -8,6 +8,10 @@ import com.carrotsearch.randomizedtesting.RandomizedTest; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import org.opensearch.client.Request; +import org.opensearch.client.Response; + import static org.opensearch.neuralsearch.TestUtils.NODES_BWC_CLUSTER; public class TextSearch extends AbstractRestartUpgradeRestTestCase{ @@ -29,10 +33,8 @@ public void testIndex() throws Exception{ Files.readString(Path.of(classLoader.getResource("processor/IndexMappings.json").toURI())), PIPELINE_NAME ); - logger.info("=================================================================================Document Added"); addDocument(testIndex, DOC_ID,TEST_FIELD,TEXT); }else { - System.out.println("===========================================================================================Cluster Upgraded"); validateTestIndex(); } } diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java b/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java index 07f9e53d7..daeb35a33 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java @@ -7,12 +7,16 @@ import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_PER_ROUTE; import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_TOTAL; +import static org.opensearch.knn.common.KNNConstants.MODELS; +import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; +import static org.opensearch.neuralsearch.TestUtils.NEURAL_SEARCH_BWC_PREFIX; +import static org.opensearch.neuralsearch.TestUtils.OPENDISTRO_SECURITY; +import static org.opensearch.neuralsearch.TestUtils.OPENSEARCH_SYSTEM_INDEX_PREFIX; +import static org.opensearch.neuralsearch.TestUtils.SECURITY_AUDITLOG_PREFIX; +import static org.opensearch.neuralsearch.TestUtils.SKIP_DELETE_MODEL_INDEX; import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.*; import java.util.stream.Collectors; import org.apache.hc.client5.http.auth.AuthScope; @@ -24,11 +28,14 @@ import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.ParseException; +import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.http.message.BasicHeader; import org.apache.hc.core5.http.nio.ssl.TlsStrategy; import org.apache.hc.core5.ssl.SSLContextBuilder; import org.apache.hc.core5.util.Timeout; import org.junit.After; +import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.RestClient; @@ -36,10 +43,11 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.core.xcontent.DeprecationHandler; -import org.opensearch.core.xcontent.MediaType; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.*; +import org.opensearch.knn.plugin.KNNPlugin; +import org.opensearch.search.SearchHit; import org.opensearch.test.rest.OpenSearchRestTestCase; /** @@ -58,6 +66,12 @@ public abstract class OpenSearchSecureRestTestCase extends OpenSearchRestTestCas private static final String INTERNAL_INDICES_PREFIX = "."; private static String protocol; + private final Set IMMUTABLE_INDEX_PREFIXES = Set.of( + NEURAL_SEARCH_BWC_PREFIX, + SECURITY_AUDITLOG_PREFIX, + OPENSEARCH_SYSTEM_INDEX_PREFIX + ); + @Override protected String getProtocol() { if (protocol == null) { @@ -148,7 +162,7 @@ protected boolean preserveIndicesUponCompletion() { } @After - public void deleteExternalIndices() throws IOException { + public void deleteExternalIndices() throws IOException, ParseException { final Response response = client().performRequest(new Request("GET", "/_cat/indices?format=json" + "&expand_wildcards=all")); final MediaType xContentType = MediaType.fromMediaType(response.getEntity().getContentType()); try ( @@ -174,8 +188,84 @@ public void deleteExternalIndices() throws IOException { .collect(Collectors.toList()); for (final String indexName : externalIndices) { - adminClient().performRequest(new Request("DELETE", "/" + indexName)); + if (isIndexCleanupRequired(indexName)) { + wipeIndexContent(indexName); + continue; + } + if (!skipDeleteIndex(indexName)) { + adminClient().performRequest(new Request("DELETE", "/" + indexName)); + } } } } + + private boolean isIndexCleanupRequired(final String index) { + return MODEL_INDEX_NAME.equals(index) && !getSkipDeleteModelIndexFlag(); + } + + private void wipeIndexContent(String indexName) throws IOException, ParseException { + deleteModels(getModelIds()); + deleteAllDocs(indexName); + } + + private List getModelIds() throws IOException, ParseException { + final String restURIGetModels = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, "_search"); + final Response response = adminClient().performRequest(new Request("GET", restURIGetModels)); + + assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + + final String responseBody = EntityUtils.toString(response.getEntity()); + assertNotNull(responseBody); + + final XContentParser parser = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody); + final SearchResponse searchResponse = SearchResponse.fromXContent(parser); + + return Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toList()); + } + + private void deleteModels(final List modelIds) throws IOException { + for (final String testModelID : modelIds) { + final String restURIGetModel = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, testModelID); + final Response getModelResponse = adminClient().performRequest(new Request("GET", restURIGetModel)); + if (RestStatus.OK != RestStatus.fromCode(getModelResponse.getStatusLine().getStatusCode())) { + continue; + } + final String restURIDeleteModel = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, testModelID); + adminClient().performRequest(new Request("DELETE", restURIDeleteModel)); + } + } + + private void deleteAllDocs(final String indexName) throws IOException { + final String restURIDeleteByQuery = String.join("/", indexName, "_delete_by_query"); + final Request request = new Request("POST", restURIDeleteByQuery); + final XContentBuilder matchAllDocsQuery = XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("match_all") + .endObject() + .endObject() + .endObject(); + + request.setJsonEntity(matchAllDocsQuery.toString()); + adminClient().performRequest(request); + } + + private boolean getSkipDeleteModelIndexFlag() { + return Boolean.parseBoolean(System.getProperty(SKIP_DELETE_MODEL_INDEX, "false")); + } + + private boolean skipDeleteModelIndex(String indexName) { + return (MODEL_INDEX_NAME.equals(indexName) && getSkipDeleteModelIndexFlag()); + } + + private boolean skipDeleteIndex(String indexName) { + if (indexName != null + && !OPENDISTRO_SECURITY.equals(indexName) + && IMMUTABLE_INDEX_PREFIXES.stream().noneMatch(indexName::startsWith) + && !skipDeleteModelIndex(indexName)) { + return false; + } + + return true; + } } diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java b/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java index b07744730..9b805b227 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java @@ -49,6 +49,10 @@ public class TestUtils { public static final int TEST_DIMENSION = 768; public static final SpaceType TEST_SPACE_TYPE = SpaceType.L2; public static final String CLIENT_TIMEOUT_VALUE = "90s"; + public static final String OPENDISTRO_SECURITY = ".opendistro_security"; + public static final String SKIP_DELETE_MODEL_INDEX = "tests.skip_delete_model_index"; + public static final String SECURITY_AUDITLOG_PREFIX = "security-auditlog"; + public static final String OPENSEARCH_SYSTEM_INDEX_PREFIX = ".opensearch"; /** * Convert an xContentBuilder to a map From 79261895ce979c65d094779fb46dcc359f7c1006 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Wed, 20 Dec 2023 11:09:44 -0800 Subject: [PATCH 06/45] Adding windows platform in bwc Signed-off-by: Varun Jain --- .../workflows/backwards_compatibility_tests_workflow.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index cde0be93b..e197eeab0 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -14,8 +14,8 @@ jobs: strategy: matrix: java: [ 11, 17, 21 ] - os: [ubuntu-latest] - bwc_version : ["2.9.0", "2.10.0", "2.11.0-SNAPSHOT"] + os: [ubuntu-latest,windows-latest] + bwc_version : ["2.9.0", "2.10.0", "2.11.0","2.12.0-SNAPSHOT"] opensearch_version : [ "3.0.0-SNAPSHOT" ] # exclude: # - os: windows-latest @@ -87,8 +87,8 @@ jobs: strategy: matrix: java: [ 11, 17, 21 ] - os: [ubuntu-latest] - bwc_version: [ "2.11.0-SNAPSHOT" ] + os: [ubuntu-latest,windows-latest] + bwc_version: [ "2.9.0", "2.10.0", "2.11.0","2.12.0-SNAPSHOT" ] opensearch_version: [ "3.0.0-SNAPSHOT" ] name: NeuralSearch Rolling-Upgrade BWC Tests From 1f880ddb4ab7012b07e952483c88f06982b48c4b Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Wed, 20 Dec 2023 11:20:36 -0800 Subject: [PATCH 07/45] Adding windows platform in bwc Signed-off-by: Varun Jain --- .github/workflows/backwards_compatibility_tests_workflow.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index e197eeab0..5848a30b4 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -57,6 +57,7 @@ jobs: name: Add MinGW to PATH on Windows run: | echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH + Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" refreshenv - if: startsWith(matrix.os,'windows') @@ -121,6 +122,7 @@ jobs: name: Add MinGW to PATH on Windows run: | echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH + Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" refreshenv - if: startsWith(matrix.os,'windows') From a92e37c6d3d9bc8a1d87f063c4511e897765043d Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Wed, 20 Dec 2023 12:18:45 -0800 Subject: [PATCH 08/45] Rolling Upgrade tests Signed-off-by: Varun Jain --- qa/restart-upgrade/build.gradle | 1 - .../neuralsearch/bwc/TextSearch.java | 14 ++++ qa/rolling-upgrade/build.gradle | 16 ++-- .../neuralsearch/bwc/TextSearch.java | 82 +++++++++++++++++++ .../CreateModelGroupRequestBody.json | 4 + .../resources/processor/IndexMappings.json | 32 ++++++++ .../processor/PipelineConfiguration.json | 18 ++++ .../processor/UploadModelRequestBody.json | 15 ++++ 8 files changed, 173 insertions(+), 9 deletions(-) create mode 100644 qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java create mode 100644 qa/rolling-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json create mode 100644 qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json create mode 100644 qa/rolling-upgrade/src/test/resources/processor/PipelineConfiguration.json create mode 100644 qa/rolling-upgrade/src/test/resources/processor/UploadModelRequestBody.json diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index fcf3df0e2..6ab0849dd 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -12,7 +12,6 @@ apply from : "$rootDir/qa/build.gradle" String default_bwc_version = System.getProperty("bwc.version") String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") -String neural_search_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version String baseName = "neuralSearchBwcCluster-restart" String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java index 7fd707b00..3a7ea2388 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java @@ -9,10 +9,12 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.Map; import org.opensearch.client.Request; import org.opensearch.client.Response; import static org.opensearch.neuralsearch.TestUtils.NODES_BWC_CLUSTER; +import org.opensearch.neuralsearch.query.NeuralQueryBuilder; public class TextSearch extends AbstractRestartUpgradeRestTestCase{ @@ -43,6 +45,13 @@ public void testIndex() throws Exception{ private void validateTestIndex() throws Exception { int docCount=getDocCount(testIndex); assertEquals(1,docCount); + NeuralQueryBuilder neuralQueryBuilder = new NeuralQueryBuilder(); + neuralQueryBuilder.fieldName(TEST_FIELD); + neuralQueryBuilder.queryText(TEXT); + neuralQueryBuilder.k(1); + Map searchResponseAsMap = search(testIndex,neuralQueryBuilder,1); + Map total = getTotalHits(searchResponseAsMap); + assertNotNull(total.get("value")); deleteIndex(testIndex); } @@ -67,4 +76,9 @@ protected void createPipelineProcessor(String modelId, String pipelineName, Proc String requestBody=Files.readString(Path.of(classLoader.getResource("processor/PipelineConfiguration.json").toURI())); createPipelineProcessor(requestBody,pipelineName,modelId); } + + private Map getTotalHits(Map searchResponseAsMap) { + Map hitsMap = (Map) searchResponseAsMap.get("hits"); + return (Map) hitsMap.get("total"); + } } diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 863bfca9a..b03959645 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -23,14 +23,14 @@ testClusters { plugin(project.tasks.zipBwcPlugin.archiveFile) setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' -// environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" -// if (Os.isFamily(Os.FAMILY_WINDOWS)) { -// // While running on Windows OS, setting the PATH environment variable to include the paths to dlls of JNI libraries and windows dependencies -// environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies") -// systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" -// } else { -// systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" -// } + environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + // While running on Windows OS, setting the PATH environment variable to include the paths to dlls of JNI libraries and windows dependencies + environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies") + systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" + } else { + systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" + } } } diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java new file mode 100644 index 000000000..0e133ff40 --- /dev/null +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.neuralsearch.bwc; + +import com.carrotsearch.randomizedtesting.RandomizedTest; +import java.nio.file.Files; +import java.nio.file.Path; +import static org.opensearch.neuralsearch.TestUtils.NODES_BWC_CLUSTER; + +public class TextSearch extends AbstractRollingUpgradeTestCase{ + private static final String PIPELINE_NAME = "nlp-pipeline"; + private static final String TEST_FIELD = "test-field"; + private static final String TEXT= "Hello world"; + private static final String TEXT_MIXED= "Hello world mixed"; + private static final String TEXT_UPGRADED= "Hello world upgraded"; + private static final int NUM_DOCS = 1; + + public void testIndex() throws Exception{ + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + switch (getClusterType()){ + case OLD: + String modelId= uploadTextEmbeddingModel(); + loadModel(modelId); + createPipelineProcessor(modelId,PIPELINE_NAME); + createIndexWithConfiguration( + testIndex, + Files.readString(Path.of(classLoader.getResource("processor/IndexMappings.json").toURI())), + PIPELINE_NAME + ); + addDocument(testIndex, "0",TEST_FIELD,TEXT); + break; + case MIXED: + int totalDocsCountMixed; + if (isFirstMixedRound()){ + totalDocsCountMixed=NUM_DOCS; + validateTestIndexOnUpgrade(totalDocsCountMixed); + addDocument(testIndex, "1",TEST_FIELD,TEXT_MIXED); + }else{ + totalDocsCountMixed=2*NUM_DOCS; + validateTestIndexOnUpgrade(totalDocsCountMixed); + } + break; + case UPGRADED: + int totalDocsCountUpgraded=3*NUM_DOCS; + addDocument(testIndex, "2",TEST_FIELD,TEXT_UPGRADED); + validateTestIndexOnUpgrade(totalDocsCountUpgraded); + deleteIndex(testIndex); + break; + } + + } + + private void validateTestIndexOnUpgrade(int numberOfDocs) throws Exception { + int docCount=getDocCount(testIndex); + assertEquals(numberOfDocs,docCount); + } + + private String uploadTextEmbeddingModel() throws Exception { + String requestBody = Files.readString(Path.of(classLoader.getResource("processor/UploadModelRequestBody.json").toURI())); + return registerModelGroupAndGetModelId(requestBody); + } + + private String registerModelGroupAndGetModelId(String requestBody) throws Exception { + String modelGroupRegisterRequestBody = Files.readString( + Path.of(classLoader.getResource("processor/CreateModelGroupRequestBody.json").toURI()) + ).replace("", "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)); + + String modelGroupId=registerModelGroup(modelGroupRegisterRequestBody); + + requestBody = requestBody.replace("", modelGroupId); + + return uploadModelId(requestBody); + } + + protected void createPipelineProcessor(String modelId, String pipelineName, ProcessorType processorType) throws Exception { + String requestBody=Files.readString(Path.of(classLoader.getResource("processor/PipelineConfiguration.json").toURI())); + createPipelineProcessor(requestBody,pipelineName,modelId); + } +} diff --git a/qa/rolling-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json b/qa/rolling-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json new file mode 100644 index 000000000..91f68e222 --- /dev/null +++ b/qa/rolling-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json @@ -0,0 +1,4 @@ +{ + "name": "", + "description": "This is a public model group" +} \ No newline at end of file diff --git a/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json b/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json new file mode 100644 index 000000000..03a683079 --- /dev/null +++ b/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json @@ -0,0 +1,32 @@ +{ + "settings": { + "index": { + "knn": true, + "knn.algo_param.ef_search": 100, + "refresh_interval": "30s", + "default_pipeline": "%s" + }, + "number_of_shards": 1, + "number_of_replicas": 0 + }, + "mappings": { + "properties": { + "passage_embedding": { + "type": "knn_vector", + "dimension": 768, + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "lucene", + "parameters": { + "ef_construction": 128, + "m": 24 + } + } + }, + "passage_text": { + "type": "text" + } + } + } +} \ No newline at end of file diff --git a/qa/rolling-upgrade/src/test/resources/processor/PipelineConfiguration.json b/qa/rolling-upgrade/src/test/resources/processor/PipelineConfiguration.json new file mode 100644 index 000000000..5d0bfe90f --- /dev/null +++ b/qa/rolling-upgrade/src/test/resources/processor/PipelineConfiguration.json @@ -0,0 +1,18 @@ +{ + "description": "text embedding pipeline for hybrid", + "processors": [ + { + "text_embedding": { + "model_id": "%s", + "field_map": { + "title": "title_knn", + "favor_list": "favor_list_knn", + "favorites": { + "game": "game_knn", + "movie": "movie_knn" + } + } + } + } + ] +} \ No newline at end of file diff --git a/qa/rolling-upgrade/src/test/resources/processor/UploadModelRequestBody.json b/qa/rolling-upgrade/src/test/resources/processor/UploadModelRequestBody.json new file mode 100644 index 000000000..6769e78c7 --- /dev/null +++ b/qa/rolling-upgrade/src/test/resources/processor/UploadModelRequestBody.json @@ -0,0 +1,15 @@ +{ + "name": "traced_small_model", + "version": "1.0.0", + "model_format": "TORCH_SCRIPT", + "model_task_type": "text_embedding", + "model_content_hash_value": "e13b74006290a9d0f58c1376f9629d4ebc05a0f9385f40db837452b167ae9021", + "model_group_id": "", + "model_config": { + "model_type": "bert", + "embedding_dimension": 768, + "framework_type": "sentence_transformers", + "all_config": "{\"architectures\":[\"BertModel\"],\"max_position_embeddings\":512,\"model_type\":\"bert\",\"num_attention_heads\":12,\"num_hidden_layers\":6}" + }, + "url": "https://github.com/opensearch-project/ml-commons/blob/2.x/ml-algorithms/src/test/resources/org/opensearch/ml/engine/algorithms/text_embedding/traced_small_model.zip?raw=true" +} \ No newline at end of file From 723fc13d7162906df4c0960fdcb7616a76854e44 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Wed, 20 Dec 2023 14:00:48 -0800 Subject: [PATCH 09/45] Bux Fix in rolling upgrade Signed-off-by: Varun Jain --- qa/rolling-upgrade/build.gradle | 38 +++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index b03959645..621b8036c 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -6,6 +6,8 @@ import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask import org.apache.tools.ant.taskdefs.condition.Os +import java.util.concurrent.Callable + apply from : "$rootDir/qa/build.gradle" String default_bwc_version = System.getProperty("bwc.version") @@ -14,12 +16,36 @@ boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version String baseName = "neuralSearchBwcCluster-rolling" +List> plugins = [provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.matching{include "**/opensearch-ml-plugin-${opensearch_build}.zip"}.getSingleFile() + } + } + } +}), provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.matching{include "**/opensearch-knn-${opensearch_build}.zip"}.getSingleFile() + } + } + } +}), rootProject.tasks.bundlePlugin.archiveFile] + // Creates a test cluster of previous version and loads k-NN plugin of bwcVersion testClusters { "${baseName}" { testDistribution = "ARCHIVE" versions = [neural_search_bwc_version, opensearch_version] numberOfNodes = 3 + plugin(project.tasks.zipBwcMlCommonsPlugin.archiveFile) + plugin(project.tasks.zipBwcKnnPlugin.archiveFile) plugin(project.tasks.zipBwcPlugin.archiveFile) setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' @@ -37,6 +63,8 @@ testClusters { // Task to run BWC tests against the old cluster task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { + dependsOn "zipBwcMlCommonsPlugin" + dependsOn "zipBwcKnnPlugin" dependsOn "zipBwcPlugin" useCluster testClusters."${baseName}" systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster' @@ -47,18 +75,16 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { systemProperty 'tests.security.manager', 'false' } - // Part of rolling upgrade. Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version // This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node. task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) { + dependsOn "testAgainstOldCluster" useCluster testClusters."${baseName}" - //dependsOn rootProject.tasks.buildJniLib dependsOn rootProject.tasks.assemble - dependsOn "testAgainstOldCluster" doFirst { testClusters."${baseName}".getNodes().getAt("${baseName}" + "-0").environment("LD_LIBRARY_PATH", "$rootDir/jni/release") testClusters."${baseName}".getNodes().getAt("${baseName}" + "-0").systemProperty("java.library.path", "$rootDir/jni/release") - testClusters."${baseName}".upgradeNodeAndPluginToNextVersion([rootProject.tasks.bundlePlugin.archiveFile]) + testClusters."${baseName}".getNodes().getAt("${baseName}" + "-0").upgradeNodeAndPluginToNextVersion(plugins) } systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster' systemProperty 'tests.rest.first_round', 'true' @@ -77,7 +103,7 @@ task testAgainstTwoThirdsUpgradedCluster(type: StandaloneRestIntegTestTask) { doFirst { testClusters."${baseName}".getNodes().getAt("${baseName}" + "-1").environment("LD_LIBRARY_PATH", "$rootDir/jni/release") testClusters."${baseName}".getNodes().getAt("${baseName}" + "-1").systemProperty("java.library.path", "$rootDir/jni/release") - testClusters."${baseName}".upgradeNodeAndPluginToNextVersion([rootProject.tasks.bundlePlugin.archiveFile]) + testClusters."${baseName}".getNodes().getAt("${baseName}" + "-1").upgradeNodeAndPluginToNextVersion(plugins) } systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster' systemProperty 'tests.rest.first_round', 'false' @@ -96,7 +122,7 @@ task testRollingUpgrade(type: StandaloneRestIntegTestTask) { doFirst { testClusters."${baseName}".getNodes().getAt("${baseName}" + "-2").environment("LD_LIBRARY_PATH", "$rootDir/jni/release") testClusters."${baseName}".getNodes().getAt("${baseName}" + "-2").systemProperty("java.library.path", "$rootDir/jni/release") - testClusters."${baseName}".upgradeNodeAndPluginToNextVersion([rootProject.tasks.bundlePlugin.archiveFile]) + testClusters."${baseName}".getNodes().getAt("${baseName}" + "-2").upgradeNodeAndPluginToNextVersion(plugins) } mustRunAfter "testAgainstOneThirdUpgradedCluster" systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster' From 975eaeaf31309b94a5baf168fa883384f9f3ed2c Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Wed, 20 Dec 2023 16:11:17 -0800 Subject: [PATCH 10/45] Bug Fix Rolling Upgrade Signed-off-by: Varun Jain --- .../backwards_compatibility_tests_workflow.yml | 2 +- qa/build.gradle | 3 ++- .../org/opensearch/neuralsearch/bwc/TextSearch.java | 11 ----------- qa/rolling-upgrade/build.gradle | 8 ++++---- 4 files changed, 7 insertions(+), 17 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 5848a30b4..0c21ccaa9 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -89,7 +89,7 @@ jobs: matrix: java: [ 11, 17, 21 ] os: [ubuntu-latest,windows-latest] - bwc_version: [ "2.9.0", "2.10.0", "2.11.0","2.12.0-SNAPSHOT" ] + bwc_version: [ "2.12.0-SNAPSHOT" ] opensearch_version: [ "3.0.0-SNAPSHOT" ] name: NeuralSearch Rolling-Upgrade BWC Tests diff --git a/qa/build.gradle b/qa/build.gradle index 8ea7dad0e..a9f3a3eb4 100644 --- a/qa/build.gradle +++ b/qa/build.gradle @@ -81,6 +81,7 @@ task deletetempDirectories { } task pullMlCommonsBwcPlugin { + //dependsOn "deletetempDirectories" doLast { copy { from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-ml")) @@ -122,7 +123,7 @@ task pullKnnBwcPlugin { // Task to pull neural search plugin from archive task pullBwcPlugin { - + //dependsOn "deletetempDirectories" doLast { copy { from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-neural-search")) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java index 3a7ea2388..286f0b6c6 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java @@ -8,13 +8,9 @@ import com.carrotsearch.randomizedtesting.RandomizedTest; import java.nio.file.Files; import java.nio.file.Path; -import java.util.List; import java.util.Map; -import org.opensearch.client.Request; -import org.opensearch.client.Response; import static org.opensearch.neuralsearch.TestUtils.NODES_BWC_CLUSTER; -import org.opensearch.neuralsearch.query.NeuralQueryBuilder; public class TextSearch extends AbstractRestartUpgradeRestTestCase{ @@ -45,13 +41,6 @@ public void testIndex() throws Exception{ private void validateTestIndex() throws Exception { int docCount=getDocCount(testIndex); assertEquals(1,docCount); - NeuralQueryBuilder neuralQueryBuilder = new NeuralQueryBuilder(); - neuralQueryBuilder.fieldName(TEST_FIELD); - neuralQueryBuilder.queryText(TEXT); - neuralQueryBuilder.k(1); - Map searchResponseAsMap = search(testIndex,neuralQueryBuilder,1); - Map total = getTotalHits(searchResponseAsMap); - assertNotNull(total.get("value")); deleteIndex(testIndex); } diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 621b8036c..99e890231 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -78,13 +78,13 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { // Part of rolling upgrade. Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version // This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node. task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) { - dependsOn "testAgainstOldCluster" useCluster testClusters."${baseName}" dependsOn rootProject.tasks.assemble + dependsOn "testAgainstOldCluster" doFirst { testClusters."${baseName}".getNodes().getAt("${baseName}" + "-0").environment("LD_LIBRARY_PATH", "$rootDir/jni/release") testClusters."${baseName}".getNodes().getAt("${baseName}" + "-0").systemProperty("java.library.path", "$rootDir/jni/release") - testClusters."${baseName}".getNodes().getAt("${baseName}" + "-0").upgradeNodeAndPluginToNextVersion(plugins) + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins) } systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster' systemProperty 'tests.rest.first_round', 'true' @@ -103,7 +103,7 @@ task testAgainstTwoThirdsUpgradedCluster(type: StandaloneRestIntegTestTask) { doFirst { testClusters."${baseName}".getNodes().getAt("${baseName}" + "-1").environment("LD_LIBRARY_PATH", "$rootDir/jni/release") testClusters."${baseName}".getNodes().getAt("${baseName}" + "-1").systemProperty("java.library.path", "$rootDir/jni/release") - testClusters."${baseName}".getNodes().getAt("${baseName}" + "-1").upgradeNodeAndPluginToNextVersion(plugins) + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins) } systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster' systemProperty 'tests.rest.first_round', 'false' @@ -122,7 +122,7 @@ task testRollingUpgrade(type: StandaloneRestIntegTestTask) { doFirst { testClusters."${baseName}".getNodes().getAt("${baseName}" + "-2").environment("LD_LIBRARY_PATH", "$rootDir/jni/release") testClusters."${baseName}".getNodes().getAt("${baseName}" + "-2").systemProperty("java.library.path", "$rootDir/jni/release") - testClusters."${baseName}".getNodes().getAt("${baseName}" + "-2").upgradeNodeAndPluginToNextVersion(plugins) + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins) } mustRunAfter "testAgainstOneThirdUpgradedCluster" systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster' From 5482a4e227a17fc13123cc46dbbc9e4b96e0fa7c Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Thu, 21 Dec 2023 09:53:51 -0800 Subject: [PATCH 11/45] Fixing Flaky tests Signed-off-by: Varun Jain --- qa/restart-upgrade/build.gradle | 16 ---------------- .../bwc/AbstractRollingUpgradeTestCase.java | 2 +- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index 6ab0849dd..4f2563aab 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -37,22 +37,6 @@ List> plugins = [provider(new Callable(){ } }), rootProject.tasks.bundlePlugin.archiveFile] -// task unzip { -// configurations.zipArchive.asFileTree.each { -// println(it.toPath()) -// } -// } - -// def tmp_dir = project.file('build/private/artifact_tmp').absoluteFile - - -// task unzip(type: Copy) { -// configurations.zipArchive.asFileTree.each { -// from(zipTree(it)) -// } -// into tmp_dir -// } - // Creates a test cluster of previous version and loads k-NN plugin of bwcVersion testClusters { "${baseName}" { diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java index 089ade9f3..42418f901 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java @@ -52,7 +52,7 @@ protected final Settings restClientSettings() { // increase the timeout here to 90 seconds to handle long waits for a green // cluster health. the waits for green need to be longer than a minute to // account for delayed shards - .put(OpenSearchRestTestCase.CLIENT_SOCKET_TIMEOUT, CLIENT_TIMEOUT_VALUE) + .put(OpenSearchRestTestCase.CLIENT_SOCKET_TIMEOUT, "120s") .build(); } From 38bd10b5f5c3c7190d522b9ae1d7cf212a258f5f Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Thu, 21 Dec 2023 10:36:07 -0800 Subject: [PATCH 12/45] Updating BWC version to latest Signed-off-by: Varun Jain --- .github/workflows/backwards_compatibility_tests_workflow.yml | 2 +- gradle.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 0c21ccaa9..d3fce2698 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -15,7 +15,7 @@ jobs: matrix: java: [ 11, 17, 21 ] os: [ubuntu-latest,windows-latest] - bwc_version : ["2.9.0", "2.10.0", "2.11.0","2.12.0-SNAPSHOT"] + bwc_version : ["2.12.0-SNAPSHOT"] opensearch_version : [ "3.0.0-SNAPSHOT" ] # exclude: # - os: windows-latest diff --git a/gradle.properties b/gradle.properties index 242771ae4..464bb2806 100644 --- a/gradle.properties +++ b/gradle.properties @@ -4,7 +4,7 @@ # version=2.0.0 -systemProp.bwc.version=2.11.0 +systemProp.bwc.version=2.12.0-SNAPSHOT # For fixing Spotless check with Java 17 org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ From b794610c1ff673478503ffbcffed9eca9281872d Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Thu, 21 Dec 2023 10:41:08 -0800 Subject: [PATCH 13/45] Fixing bwc test Signed-off-by: Varun Jain --- .github/workflows/backwards_compatibility_tests_workflow.yml | 2 +- gradle.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index d3fce2698..d4989374d 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -15,7 +15,7 @@ jobs: matrix: java: [ 11, 17, 21 ] os: [ubuntu-latest,windows-latest] - bwc_version : ["2.12.0-SNAPSHOT"] + bwc_version : ["2.11.0"] opensearch_version : [ "3.0.0-SNAPSHOT" ] # exclude: # - os: windows-latest diff --git a/gradle.properties b/gradle.properties index 464bb2806..242771ae4 100644 --- a/gradle.properties +++ b/gradle.properties @@ -4,7 +4,7 @@ # version=2.0.0 -systemProp.bwc.version=2.12.0-SNAPSHOT +systemProp.bwc.version=2.11.0 # For fixing Spotless check with Java 17 org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ From 74d9aef2d3ffff1fd48bf261af06645d62b74582 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 26 Dec 2023 11:46:25 -0800 Subject: [PATCH 14/45] Semantic Search Signed-off-by: Varun Jain --- CHANGELOG.md | 1 + qa/build.gradle | 3 -- .../{TextSearch.java => SemanticSearch.java} | 32 +++++++++++++---- .../{TextSearch.java => SemanticSearch.java} | 34 +++++++++++++++---- .../neuralsearch/BaseNeuralSearchIT.java | 13 +++++++ .../opensearch/neuralsearch/TestUtils.java | 9 +++++ 6 files changed, 77 insertions(+), 15 deletions(-) rename qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/{TextSearch.java => SemanticSearch.java} (68%) rename qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/{TextSearch.java => SemanticSearch.java} (71%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f02272c4..d95b63751 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements ### Bug Fixes ### Infrastructure +- BWC tests for Neural Search ([#515](https://github.com/opensearch-project/neural-search/pull/515)) ### Documentation ### Maintenance ### Refactoring diff --git a/qa/build.gradle b/qa/build.gradle index a9f3a3eb4..f5ecfb615 100644 --- a/qa/build.gradle +++ b/qa/build.gradle @@ -57,7 +57,6 @@ String default_bwc_version = System.getProperty("bwc.version") String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") String neural_search_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version -String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version String os_platform = "linux" String artifact_type = "tar" @@ -81,7 +80,6 @@ task deletetempDirectories { } task pullMlCommonsBwcPlugin { - //dependsOn "deletetempDirectories" doLast { copy { from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-ml")) @@ -123,7 +121,6 @@ task pullKnnBwcPlugin { // Task to pull neural search plugin from archive task pullBwcPlugin { - //dependsOn "deletetempDirectories" doLast { copy { from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-neural-search")) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java similarity index 68% rename from qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java rename to qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java index 286f0b6c6..86b322f8b 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java @@ -9,17 +9,24 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; +import java.util.ArrayList; + import static org.opensearch.neuralsearch.TestUtils.NODES_BWC_CLUSTER; -public class TextSearch extends AbstractRestartUpgradeRestTestCase{ +import static org.opensearch.neuralsearch.TestUtils.getModelId; +import static org.opensearch.neuralsearch.TestUtils.TEXT_EMBEDDING_PROCESSOR; +import org.opensearch.neuralsearch.query.NeuralQueryBuilder; + + +public class SemanticSearch extends AbstractRestartUpgradeRestTestCase{ private static final String PIPELINE_NAME = "nlp-pipeline"; private static String DOC_ID = "0"; - private static final String TEST_FIELD = "test-field"; + private static final String TEST_FIELD = "passage_text"; private static final String TEXT= "Hello world"; - public void testIndex() throws Exception{ + public void testSemanticSearch() throws Exception{ waitForClusterHealthGreen(NODES_BWC_CLUSTER); if (isRunningAgainstOldCluster()){ @@ -33,17 +40,29 @@ public void testIndex() throws Exception{ ); addDocument(testIndex, DOC_ID,TEST_FIELD,TEXT); }else { - validateTestIndex(); + Map pipeline= getIngestionPipeline(PIPELINE_NAME); + assertNotNull(pipeline); + String modelId=getModelId(pipeline, TEXT_EMBEDDING_PROCESSOR); + validateTestIndex(modelId); + deleteIndex(testIndex); } } - private void validateTestIndex() throws Exception { + private void validateTestIndex(String modelId) throws Exception { int docCount=getDocCount(testIndex); assertEquals(1,docCount); - deleteIndex(testIndex); + loadModel(modelId); + NeuralQueryBuilder neuralQueryBuilder = new NeuralQueryBuilder(); + neuralQueryBuilder.fieldName("passage_embedding"); + neuralQueryBuilder.modelId(modelId); + neuralQueryBuilder.queryText(TEXT); + neuralQueryBuilder.k(1); + Map response = search(testIndex, neuralQueryBuilder, 1); + assertNotNull(response); } + private String uploadTextEmbeddingModel() throws Exception { String requestBody = Files.readString(Path.of(classLoader.getResource("processor/UploadModelRequestBody.json").toURI())); return registerModelGroupAndGetModelId(requestBody); @@ -70,4 +89,5 @@ private Map getTotalHits(Map searchResponseAsMap Map hitsMap = (Map) searchResponseAsMap.get("hits"); return (Map) hitsMap.get("total"); } + } diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java similarity index 71% rename from qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java rename to qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java index 0e133ff40..a3f37e72b 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java @@ -8,9 +8,14 @@ import com.carrotsearch.randomizedtesting.RandomizedTest; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Map; +import java.util.ArrayList; import static org.opensearch.neuralsearch.TestUtils.NODES_BWC_CLUSTER; +import static org.opensearch.neuralsearch.TestUtils.getModelId; +import static org.opensearch.neuralsearch.TestUtils.TEXT_EMBEDDING_PROCESSOR; +import org.opensearch.neuralsearch.query.NeuralQueryBuilder; -public class TextSearch extends AbstractRollingUpgradeTestCase{ +public class SemanticSearch extends AbstractRollingUpgradeTestCase{ private static final String PIPELINE_NAME = "nlp-pipeline"; private static final String TEST_FIELD = "test-field"; private static final String TEXT= "Hello world"; @@ -18,7 +23,7 @@ public class TextSearch extends AbstractRollingUpgradeTestCase{ private static final String TEXT_UPGRADED= "Hello world upgraded"; private static final int NUM_DOCS = 1; - public void testIndex() throws Exception{ + public void testSemanticSearch() throws Exception{ waitForClusterHealthGreen(NODES_BWC_CLUSTER); switch (getClusterType()){ case OLD: @@ -33,29 +38,40 @@ public void testIndex() throws Exception{ addDocument(testIndex, "0",TEST_FIELD,TEXT); break; case MIXED: + modelId=getModelId(PIPELINE_NAME); int totalDocsCountMixed; if (isFirstMixedRound()){ totalDocsCountMixed=NUM_DOCS; - validateTestIndexOnUpgrade(totalDocsCountMixed); + validateTestIndexOnUpgrade(totalDocsCountMixed, modelId, TEXT); addDocument(testIndex, "1",TEST_FIELD,TEXT_MIXED); + }else{ totalDocsCountMixed=2*NUM_DOCS; - validateTestIndexOnUpgrade(totalDocsCountMixed); + validateTestIndexOnUpgrade(totalDocsCountMixed, modelId, TEXT_MIXED); } break; case UPGRADED: + modelId=getModelId(PIPELINE_NAME); int totalDocsCountUpgraded=3*NUM_DOCS; addDocument(testIndex, "2",TEST_FIELD,TEXT_UPGRADED); - validateTestIndexOnUpgrade(totalDocsCountUpgraded); + validateTestIndexOnUpgrade(totalDocsCountUpgraded, modelId, TEXT_UPGRADED); deleteIndex(testIndex); break; } } - private void validateTestIndexOnUpgrade(int numberOfDocs) throws Exception { + private void validateTestIndexOnUpgrade(int numberOfDocs, String modelId, String text) throws Exception { int docCount=getDocCount(testIndex); assertEquals(numberOfDocs,docCount); + loadModel(modelId); + NeuralQueryBuilder neuralQueryBuilder = new NeuralQueryBuilder(); + neuralQueryBuilder.fieldName("passage_embedding"); + neuralQueryBuilder.modelId(modelId); + neuralQueryBuilder.queryText(text); + neuralQueryBuilder.k(1); + Map response = search(testIndex, neuralQueryBuilder, 1); + assertNotNull(response); } private String uploadTextEmbeddingModel() throws Exception { @@ -79,4 +95,10 @@ protected void createPipelineProcessor(String modelId, String pipelineName, Proc String requestBody=Files.readString(Path.of(classLoader.getResource("processor/PipelineConfiguration.json").toURI())); createPipelineProcessor(requestBody,pipelineName,modelId); } + + private String getModelId(String pipelineName){ + Map pipeline = getIngestionPipeline(PIPELINE_NAME); + assertNotNull(pipeline); + return getModelId(pipeline,TEXT_EMBEDDING_PROCESSOR); + } } diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java index 0ccdc58c3..9c9999aba 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java @@ -837,6 +837,19 @@ protected void addDocument(String index, String docId, String fieldName, String assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); } + /** + * Get ingest pipeline + */ + @SneakyThrows + protected Map getIngestionPipeline(String pipelineName) { + Request request = new Request("GET", "/_ingest/pipeline/" + pipelineName); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + String responseBody = EntityUtils.toString(response.getEntity()); + Map responseMap = createParser(XContentType.JSON.xContent(), responseBody).map(); + return (Map) responseMap.get(pipelineName); + } + /** * Enumeration for types of pipeline processors, used to lookup resources like create * processor request as those are type specific diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java b/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java index 9b805b227..52b208e5d 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java @@ -53,6 +53,7 @@ public class TestUtils { public static final String SKIP_DELETE_MODEL_INDEX = "tests.skip_delete_model_index"; public static final String SECURITY_AUDITLOG_PREFIX = "security-auditlog"; public static final String OPENSEARCH_SYSTEM_INDEX_PREFIX = ".opensearch"; + public static final String TEXT_EMBEDDING_PROCESSOR = "text_embedding"; /** * Convert an xContentBuilder to a map @@ -302,4 +303,12 @@ private static Optional getMaxScore(Map searchResponseAsM Map hitsMap = (Map) searchResponseAsMap.get("hits"); return hitsMap.get("max_score") == null ? Optional.empty() : Optional.of(((Double) hitsMap.get("max_score")).floatValue()); } + + public static String getModelId(Map pipeline, String processor) { + ArrayList> processors = (ArrayList>) pipeline.get("processors"); + + Map textEmbeddingProcessor = (Map) processors.get(0).get(processor); + + return (String) textEmbeddingProcessor.get("model_id"); + } } From ba17401545569e17bee19a9f3529997f19986df9 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 26 Dec 2023 11:55:09 -0800 Subject: [PATCH 15/45] Bug Fix Signed-off-by: Varun Jain --- .../org/opensearch/neuralsearch/bwc/SemanticSearch.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java index a3f37e72b..e91bd5834 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java @@ -9,9 +9,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; -import java.util.ArrayList; +import org.opensearch.neuralsearch.TestUtils; import static org.opensearch.neuralsearch.TestUtils.NODES_BWC_CLUSTER; -import static org.opensearch.neuralsearch.TestUtils.getModelId; import static org.opensearch.neuralsearch.TestUtils.TEXT_EMBEDDING_PROCESSOR; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; @@ -97,8 +96,8 @@ protected void createPipelineProcessor(String modelId, String pipelineName, Proc } private String getModelId(String pipelineName){ - Map pipeline = getIngestionPipeline(PIPELINE_NAME); + Map pipeline = getIngestionPipeline(pipelineName); assertNotNull(pipeline); - return getModelId(pipeline,TEXT_EMBEDDING_PROCESSOR); + return TestUtils.getModelId(pipeline,TEXT_EMBEDDING_PROCESSOR); } } From eccaf2a284d472d5de2a570e7568d02d98f14aac Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 26 Dec 2023 14:42:27 -0800 Subject: [PATCH 16/45] Debugging Signed-off-by: Varun Jain --- .../neuralsearch/bwc/SemanticSearch.java | 5 ++++ .../neuralsearch/BaseNeuralSearchIT.java | 23 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java index e91bd5834..782efc2f2 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java @@ -9,11 +9,13 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; +import lombok.extern.log4j.Log4j2; import org.opensearch.neuralsearch.TestUtils; import static org.opensearch.neuralsearch.TestUtils.NODES_BWC_CLUSTER; import static org.opensearch.neuralsearch.TestUtils.TEXT_EMBEDDING_PROCESSOR; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; +@Log4j2 public class SemanticSearch extends AbstractRollingUpgradeTestCase{ private static final String PIPELINE_NAME = "nlp-pipeline"; private static final String TEST_FIELD = "test-field"; @@ -23,6 +25,9 @@ public class SemanticSearch extends AbstractRollingUpgradeTestCase{ private static final int NUM_DOCS = 1; public void testSemanticSearch() throws Exception{ + log.info("Get Cluster Type=========================="+getClusterType()); + getIndices(); + getShards(); waitForClusterHealthGreen(NODES_BWC_CLUSTER); switch (getClusterType()){ case OLD: diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java index 9c9999aba..63e2888d7 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java @@ -54,8 +54,10 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.SneakyThrows; +import lombok.extern.log4j.Log4j2; @ThreadLeakScope(ThreadLeakScope.Scope.NONE) +@Log4j2 public abstract class BaseNeuralSearchIT extends OpenSearchSecureRestTestCase { protected static final Locale LOCALE = Locale.ROOT; @@ -850,6 +852,27 @@ protected Map getIngestionPipeline(String pipelineName) { return (Map) responseMap.get(pipelineName); } + /** + * Get indices + */ + @SneakyThrows + protected void getIndices() { + Request request = new Request("GET", "/_cat/indices"); + Response response = client().performRequest(request); + String responseBody = EntityUtils.toString(response.getEntity()); + log.info("Indices are ============================"); + log.info(responseBody); + } + + @SneakyThrows + protected void getShards() { + Request request = new Request("GET", "/_cat/shards"); + Response response = client().performRequest(request); + String responseBody = EntityUtils.toString(response.getEntity()); + log.info("Shards are ============================"); + log.info(responseBody); + } + /** * Enumeration for types of pipeline processors, used to lookup resources like create * processor request as those are type specific From e400e06e03e087cb82e04bf98f32f51e17827e6b Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 26 Dec 2023 15:23:17 -0800 Subject: [PATCH 17/45] Bug Fix Signed-off-by: Varun Jain --- .../opensearch/neuralsearch/bwc/SemanticSearch.java | 4 +--- .../opensearch/neuralsearch/BaseNeuralSearchIT.java | 10 ++++------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java index 782efc2f2..d803f62cb 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java @@ -9,13 +9,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; -import lombok.extern.log4j.Log4j2; import org.opensearch.neuralsearch.TestUtils; import static org.opensearch.neuralsearch.TestUtils.NODES_BWC_CLUSTER; import static org.opensearch.neuralsearch.TestUtils.TEXT_EMBEDDING_PROCESSOR; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; -@Log4j2 public class SemanticSearch extends AbstractRollingUpgradeTestCase{ private static final String PIPELINE_NAME = "nlp-pipeline"; private static final String TEST_FIELD = "test-field"; @@ -25,7 +23,7 @@ public class SemanticSearch extends AbstractRollingUpgradeTestCase{ private static final int NUM_DOCS = 1; public void testSemanticSearch() throws Exception{ - log.info("Get Cluster Type=========================="+getClusterType()); + logger.info("Get Cluster Type=========================="+getClusterType()); getIndices(); getShards(); waitForClusterHealthGreen(NODES_BWC_CLUSTER); diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java index 63e2888d7..08b584b33 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java @@ -54,10 +54,8 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.SneakyThrows; -import lombok.extern.log4j.Log4j2; @ThreadLeakScope(ThreadLeakScope.Scope.NONE) -@Log4j2 public abstract class BaseNeuralSearchIT extends OpenSearchSecureRestTestCase { protected static final Locale LOCALE = Locale.ROOT; @@ -860,8 +858,8 @@ protected void getIndices() { Request request = new Request("GET", "/_cat/indices"); Response response = client().performRequest(request); String responseBody = EntityUtils.toString(response.getEntity()); - log.info("Indices are ============================"); - log.info(responseBody); + logger.info("Indices are ============================"); + logger.info(responseBody); } @SneakyThrows @@ -869,8 +867,8 @@ protected void getShards() { Request request = new Request("GET", "/_cat/shards"); Response response = client().performRequest(request); String responseBody = EntityUtils.toString(response.getEntity()); - log.info("Shards are ============================"); - log.info(responseBody); + logger.info("Shards are ============================"); + logger.info(responseBody); } /** From 038cd1aa663d11d8c46f403cb4df965dfbc54234 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 26 Dec 2023 15:55:52 -0800 Subject: [PATCH 18/45] Increase memory in nodes Signed-off-by: Varun Jain --- qa/restart-upgrade/build.gradle | 1 + qa/rolling-upgrade/build.gradle | 1 + 2 files changed, 2 insertions(+) diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index 4f2563aab..ea8f3b830 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -43,6 +43,7 @@ testClusters { testDistribution = "ARCHIVE" versions = [neural_search_bwc_version, opensearch_version] numberOfNodes = 3 + jvmArgs("-Xms1g", "-Xmx4g") plugin(project.tasks.zipBwcMlCommonsPlugin.archiveFile) plugin(project.tasks.zipBwcKnnPlugin.archiveFile) plugin(project.tasks.zipBwcPlugin.archiveFile) diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 99e890231..76560a3cf 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -47,6 +47,7 @@ testClusters { plugin(project.tasks.zipBwcMlCommonsPlugin.archiveFile) plugin(project.tasks.zipBwcKnnPlugin.archiveFile) plugin(project.tasks.zipBwcPlugin.archiveFile) + jvmArgs("-Xms1g", "-Xmx4g") setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" From 6bb2dd3cebf32c5405542bb9bfe5a0881a69798d Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 26 Dec 2023 16:05:55 -0800 Subject: [PATCH 19/45] Removing extra logging Signed-off-by: Varun Jain --- .../java/org/opensearch/neuralsearch/bwc/SemanticSearch.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java index d803f62cb..e91bd5834 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java @@ -23,9 +23,6 @@ public class SemanticSearch extends AbstractRollingUpgradeTestCase{ private static final int NUM_DOCS = 1; public void testSemanticSearch() throws Exception{ - logger.info("Get Cluster Type=========================="+getClusterType()); - getIndices(); - getShards(); waitForClusterHealthGreen(NODES_BWC_CLUSTER); switch (getClusterType()){ case OLD: From a5d17e1922f2dc0a532dc88b69285de2c85a98be Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 26 Dec 2023 19:10:34 -0800 Subject: [PATCH 20/45] Cleaning up Signed-off-by: Varun Jain --- ...backwards_compatibility_tests_workflow.yml | 11 +---- .../neuralsearch/bwc/SemanticSearch.java | 8 ++-- .../neuralsearch/bwc/SemanticSearch.java | 3 ++ .../neuralsearch/BaseNeuralSearchIT.java | 46 ++++++++++--------- .../OpenSearchSecureRestTestCase.java | 7 ++- 5 files changed, 38 insertions(+), 37 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index d4989374d..6500011f4 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -15,17 +15,8 @@ jobs: matrix: java: [ 11, 17, 21 ] os: [ubuntu-latest,windows-latest] - bwc_version : ["2.11.0"] + bwc_version : ["2.9.0","2.10.0","2.11.0","2.12.0-SNAPSHOT"] opensearch_version : [ "3.0.0-SNAPSHOT" ] -# exclude: -# - os: windows-latest -# bwc_version: "2.0.1" -# - os: windows-latest -# bwc_version: "2.1.0" -# - os: windows-latest -# bwc_version: "2.2.1" -# - os: windows-latest -# bwc_version: "2.3.0" name: NeuralSearch Restart-Upgrade BWC Tests runs-on: ${{ matrix.os }} diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java index 86b322f8b..26a4eb421 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java @@ -26,6 +26,9 @@ public class SemanticSearch extends AbstractRestartUpgradeRestTestCase{ private static final String TEST_FIELD = "passage_text"; private static final String TEXT= "Hello world"; + //Test restart-upgrade Semantic Search + //Create Text Embedding Processor, Ingestion Pipeline and add document + //Validate process , pipeline and document count in restart-upgrade scenario public void testSemanticSearch() throws Exception{ waitForClusterHealthGreen(NODES_BWC_CLUSTER); @@ -84,10 +87,5 @@ protected void createPipelineProcessor(String modelId, String pipelineName, Proc String requestBody=Files.readString(Path.of(classLoader.getResource("processor/PipelineConfiguration.json").toURI())); createPipelineProcessor(requestBody,pipelineName,modelId); } - - private Map getTotalHits(Map searchResponseAsMap) { - Map hitsMap = (Map) searchResponseAsMap.get("hits"); - return (Map) hitsMap.get("total"); - } } diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java index e91bd5834..713c05d1b 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java @@ -22,6 +22,9 @@ public class SemanticSearch extends AbstractRollingUpgradeTestCase{ private static final String TEXT_UPGRADED= "Hello world upgraded"; private static final int NUM_DOCS = 1; + //Test rolling-upgrade Semantic Search + //Create Text Embedding Processor, Ingestion Pipeline and add document + //Validate process , pipeline and document count in rolling-upgrade scenario public void testSemanticSearch() throws Exception{ waitForClusterHealthGreen(NODES_BWC_CLUSTER); switch (getClusterType()){ diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java index 08b584b33..a273b3f22 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java @@ -22,7 +22,11 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; -import org.apache.hc.core5.http.*; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.HttpHeaders; +import org.apache.hc.core5.http.ParseException; import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.http.io.entity.StringEntity; import org.apache.hc.core5.http.message.BasicHeader; @@ -850,26 +854,26 @@ protected Map getIngestionPipeline(String pipelineName) { return (Map) responseMap.get(pipelineName); } - /** - * Get indices - */ - @SneakyThrows - protected void getIndices() { - Request request = new Request("GET", "/_cat/indices"); - Response response = client().performRequest(request); - String responseBody = EntityUtils.toString(response.getEntity()); - logger.info("Indices are ============================"); - logger.info(responseBody); - } - - @SneakyThrows - protected void getShards() { - Request request = new Request("GET", "/_cat/shards"); - Response response = client().performRequest(request); - String responseBody = EntityUtils.toString(response.getEntity()); - logger.info("Shards are ============================"); - logger.info(responseBody); - } + // /** + // * Get indices + // */ + // @SneakyThrows + // protected void getIndices() { + // Request request = new Request("GET", "/_cat/indices"); + // Response response = client().performRequest(request); + // String responseBody = EntityUtils.toString(response.getEntity()); + // logger.info("Indices are ============================"); + // logger.info(responseBody); + // } + // + // @SneakyThrows + // protected void getShards() { + // Request request = new Request("GET", "/_cat/shards"); + // Response response = client().performRequest(request); + // String responseBody = EntityUtils.toString(response.getEntity()); + // logger.info("Shards are ============================"); + // logger.info(responseBody); + // } /** * Enumeration for types of pipeline processors, used to lookup resources like create diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java b/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java index daeb35a33..de217d828 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java @@ -16,7 +16,12 @@ import static org.opensearch.neuralsearch.TestUtils.SKIP_DELETE_MODEL_INDEX; import java.io.IOException; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import org.apache.hc.client5.http.auth.AuthScope; From f84bb56f735ffc5eec020eda93ac32cf41d25560 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 26 Dec 2023 20:47:08 -0800 Subject: [PATCH 21/45] Updating Pipeline Configuration Signed-off-by: Varun Jain --- qa/build.gradle | 7 ------- .../processor/PipelineConfiguration.json | 7 +------ .../processor/PipelineConfiguration.json | 7 +------ .../neuralsearch/BaseNeuralSearchIT.java | 21 ------------------- 4 files changed, 2 insertions(+), 40 deletions(-) diff --git a/qa/build.gradle b/qa/build.gradle index f5ecfb615..86ad41315 100644 --- a/qa/build.gradle +++ b/qa/build.gradle @@ -20,13 +20,6 @@ test.enabled = false assemble.enabled = false dependenciesInfo.enabled = false -repositories { - mavenLocal() - maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } - mavenCentral() - maven { url "https://plugins.gradle.org/m2/" } -} - configurations { zipArchive } diff --git a/qa/restart-upgrade/src/test/resources/processor/PipelineConfiguration.json b/qa/restart-upgrade/src/test/resources/processor/PipelineConfiguration.json index 5d0bfe90f..73646329f 100644 --- a/qa/restart-upgrade/src/test/resources/processor/PipelineConfiguration.json +++ b/qa/restart-upgrade/src/test/resources/processor/PipelineConfiguration.json @@ -5,12 +5,7 @@ "text_embedding": { "model_id": "%s", "field_map": { - "title": "title_knn", - "favor_list": "favor_list_knn", - "favorites": { - "game": "game_knn", - "movie": "movie_knn" - } + "passage_text": "passage_embedding" } } } diff --git a/qa/rolling-upgrade/src/test/resources/processor/PipelineConfiguration.json b/qa/rolling-upgrade/src/test/resources/processor/PipelineConfiguration.json index 5d0bfe90f..73646329f 100644 --- a/qa/rolling-upgrade/src/test/resources/processor/PipelineConfiguration.json +++ b/qa/rolling-upgrade/src/test/resources/processor/PipelineConfiguration.json @@ -5,12 +5,7 @@ "text_embedding": { "model_id": "%s", "field_map": { - "title": "title_knn", - "favor_list": "favor_list_knn", - "favorites": { - "game": "game_knn", - "movie": "movie_knn" - } + "passage_text": "passage_embedding" } } } diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java index a273b3f22..3e0a504ff 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java @@ -854,27 +854,6 @@ protected Map getIngestionPipeline(String pipelineName) { return (Map) responseMap.get(pipelineName); } - // /** - // * Get indices - // */ - // @SneakyThrows - // protected void getIndices() { - // Request request = new Request("GET", "/_cat/indices"); - // Response response = client().performRequest(request); - // String responseBody = EntityUtils.toString(response.getEntity()); - // logger.info("Indices are ============================"); - // logger.info(responseBody); - // } - // - // @SneakyThrows - // protected void getShards() { - // Request request = new Request("GET", "/_cat/shards"); - // Response response = client().performRequest(request); - // String responseBody = EntityUtils.toString(response.getEntity()); - // logger.info("Shards are ============================"); - // logger.info(responseBody); - // } - /** * Enumeration for types of pipeline processors, used to lookup resources like create * processor request as those are type specific From d6d7f9ba32e3465c9001910d0ad26194ec4e3224 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Wed, 27 Dec 2023 11:18:53 -0800 Subject: [PATCH 22/45] Remove KNN delete models Signed-off-by: Varun Jain --- .../OpenSearchSecureRestTestCase.java | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java b/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java index de217d828..d42ae62eb 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java @@ -50,7 +50,12 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.rest.RestStatus; -import org.opensearch.core.xcontent.*; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.search.SearchHit; import org.opensearch.test.rest.OpenSearchRestTestCase; @@ -209,7 +214,6 @@ private boolean isIndexCleanupRequired(final String index) { } private void wipeIndexContent(String indexName) throws IOException, ParseException { - deleteModels(getModelIds()); deleteAllDocs(indexName); } @@ -228,18 +232,6 @@ private List getModelIds() throws IOException, ParseException { return Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toList()); } - private void deleteModels(final List modelIds) throws IOException { - for (final String testModelID : modelIds) { - final String restURIGetModel = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, testModelID); - final Response getModelResponse = adminClient().performRequest(new Request("GET", restURIGetModel)); - if (RestStatus.OK != RestStatus.fromCode(getModelResponse.getStatusLine().getStatusCode())) { - continue; - } - final String restURIDeleteModel = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, testModelID); - adminClient().performRequest(new Request("DELETE", restURIDeleteModel)); - } - } - private void deleteAllDocs(final String indexName) throws IOException { final String restURIDeleteByQuery = String.join("/", indexName, "_delete_by_query"); final Request request = new Request("POST", restURIDeleteByQuery); From 20ec55f259029d21d4eda8b92c2ef8dd3f3b5f52 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Wed, 27 Dec 2023 11:31:24 -0800 Subject: [PATCH 23/45] Remove unecessary KNN code Signed-off-by: Varun Jain --- .../OpenSearchSecureRestTestCase.java | 52 ------------------- 1 file changed, 52 deletions(-) diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java b/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java index d42ae62eb..84966770c 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java @@ -7,7 +7,6 @@ import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_PER_ROUTE; import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_TOTAL; -import static org.opensearch.knn.common.KNNConstants.MODELS; import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; import static org.opensearch.neuralsearch.TestUtils.NEURAL_SEARCH_BWC_PREFIX; import static org.opensearch.neuralsearch.TestUtils.OPENDISTRO_SECURITY; @@ -16,7 +15,6 @@ import static org.opensearch.neuralsearch.TestUtils.SKIP_DELETE_MODEL_INDEX; import java.io.IOException; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -34,13 +32,11 @@ import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpHost; import org.apache.hc.core5.http.ParseException; -import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.http.message.BasicHeader; import org.apache.hc.core5.http.nio.ssl.TlsStrategy; import org.apache.hc.core5.ssl.SSLContextBuilder; import org.apache.hc.core5.util.Timeout; import org.junit.After; -import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.RestClient; @@ -48,16 +44,10 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; -import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.knn.plugin.KNNPlugin; -import org.opensearch.search.SearchHit; import org.opensearch.test.rest.OpenSearchRestTestCase; /** @@ -198,10 +188,6 @@ public void deleteExternalIndices() throws IOException, ParseException { .collect(Collectors.toList()); for (final String indexName : externalIndices) { - if (isIndexCleanupRequired(indexName)) { - wipeIndexContent(indexName); - continue; - } if (!skipDeleteIndex(indexName)) { adminClient().performRequest(new Request("DELETE", "/" + indexName)); } @@ -209,44 +195,6 @@ public void deleteExternalIndices() throws IOException, ParseException { } } - private boolean isIndexCleanupRequired(final String index) { - return MODEL_INDEX_NAME.equals(index) && !getSkipDeleteModelIndexFlag(); - } - - private void wipeIndexContent(String indexName) throws IOException, ParseException { - deleteAllDocs(indexName); - } - - private List getModelIds() throws IOException, ParseException { - final String restURIGetModels = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, "_search"); - final Response response = adminClient().performRequest(new Request("GET", restURIGetModels)); - - assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); - - final String responseBody = EntityUtils.toString(response.getEntity()); - assertNotNull(responseBody); - - final XContentParser parser = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody); - final SearchResponse searchResponse = SearchResponse.fromXContent(parser); - - return Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toList()); - } - - private void deleteAllDocs(final String indexName) throws IOException { - final String restURIDeleteByQuery = String.join("/", indexName, "_delete_by_query"); - final Request request = new Request("POST", restURIDeleteByQuery); - final XContentBuilder matchAllDocsQuery = XContentFactory.jsonBuilder() - .startObject() - .startObject("query") - .startObject("match_all") - .endObject() - .endObject() - .endObject(); - - request.setJsonEntity(matchAllDocsQuery.toString()); - adminClient().performRequest(request); - } - private boolean getSkipDeleteModelIndexFlag() { return Boolean.parseBoolean(System.getProperty(SKIP_DELETE_MODEL_INDEX, "false")); } From 4a7ea91d5d2d408234b0489f32e684ba026816c0 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Wed, 27 Dec 2023 12:51:49 -0800 Subject: [PATCH 24/45] Addressing comments of naveen Signed-off-by: Varun Jain --- ...backwards_compatibility_tests_workflow.yml | 60 ------------------- gradle.properties | 3 +- qa/build.gradle | 39 +++++++----- qa/restart-upgrade/build.gradle | 16 +---- .../AbstractRestartUpgradeRestTestCase.java | 2 +- qa/rolling-upgrade/build.gradle | 15 ----- .../bwc/AbstractRollingUpgradeTestCase.java | 2 +- 7 files changed, 29 insertions(+), 108 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 6500011f4..2ef6b2e1c 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -32,36 +32,6 @@ jobs: with: java-version: ${{ matrix.java }} - - if: startsWith(matrix.os,'ubuntu') - name: Install dependencies on ubuntu - run: | - sudo apt-get install libopenblas-dev gfortran -y - - - if: startsWith(matrix.os,'windows') - name: Install MinGW Using Scoop on Windows - run: | - iex "& {$(irm get.scoop.sh)} -RunAsAdmin" - scoop bucket add main - scoop install mingw - - - if: startsWith(matrix.os,'windows') - name: Add MinGW to PATH on Windows - run: | - echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH - Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" - refreshenv - - - if: startsWith(matrix.os,'windows') - name: Download OpenBLAS on Windows - run: | - curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip - mkdir OpenBLAS - Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\ - mkdir ./src/main/resources/windowsDependencies - cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/ - rm .\OpenBLAS-0.3.21-x64.zip - rm -r .\OpenBLAS\ - - if: startsWith(matrix.os,'windows') name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Windows run: | @@ -97,36 +67,6 @@ jobs: with: java-version: ${{ matrix.java }} - - if: startsWith(matrix.os,'ubuntu') - name: Install dependencies on ubuntu - run: | - sudo apt-get install libopenblas-dev gfortran -y - - - if: startsWith(matrix.os,'windows') - name: Install MinGW Using Scoop on Windows - run: | - iex "& {$(irm get.scoop.sh)} -RunAsAdmin" - scoop bucket add main - scoop install mingw - - - if: startsWith(matrix.os,'windows') - name: Add MinGW to PATH on Windows - run: | - echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH - Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" - refreshenv - - - if: startsWith(matrix.os,'windows') - name: Download OpenBLAS on Windows - run: | - curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip - mkdir OpenBLAS - Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\ - mkdir ./src/main/resources/windowsDependencies - cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/ - rm .\OpenBLAS-0.3.21-x64.zip - rm -r .\OpenBLAS\ - - if: startsWith(matrix.os,'windows') name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Windows run: | diff --git a/gradle.properties b/gradle.properties index 242771ae4..b4dc3433d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,8 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # -version=2.0.0 -systemProp.bwc.version=2.11.0 +systemProp.bwc.version=2.12.0-SNAPSHOT # For fixing Spotless check with Java 17 org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ diff --git a/qa/build.gradle b/qa/build.gradle index 86ad41315..ccaba4e24 100644 --- a/qa/build.gradle +++ b/qa/build.gradle @@ -31,7 +31,6 @@ dependencies { zipArchive group: 'org.opensearch.plugin', name:'opensearch-knn', version: "${opensearch_build}" zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}" compileOnly fileTree(dir: knnJarDirectory, include: '*.jar') - //api group: 'commons-lang', name: 'commons-lang', version: '2.6' api "org.apache.logging.log4j:log4j-api:${versions.log4j}" api "org.apache.logging.log4j:log4j-core:${versions.log4j}" api "junit:junit:${versions.junit}" @@ -72,19 +71,11 @@ task deletetempDirectories { } } -task pullMlCommonsBwcPlugin { - doLast { - copy { - from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-ml")) - into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml") - } - } -} - -task pullKnnBwcPlugin { - dependsOn "deletetempDirectories" +// Task to pull opensearch artifact from archive +task pullOpensearchArtifact { + dependsOn "deletetempDirectories" - doLast { + doLast{ ext{ if (isSnapshot) { srcUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${neural_search_bwc_version_no_qualifier}/latest/${os_platform}/x64/${artifact_type}/dist/opensearch/opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}" @@ -105,6 +96,24 @@ task pullKnnBwcPlugin { } into tmp_dir.absolutePath } + } +} + +// Task to pull ml plugin from archive +task pullMlCommonsBwcPlugin { + doLast { + copy { + from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-ml")) + into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml") + } + } +} + +// Task to pull KNN plugin from archive +task pullKnnBwcPlugin { + dependsOn "pullOpensearchArtifact" + + doLast { copy { from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-knn")) into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn") @@ -123,6 +132,7 @@ task pullBwcPlugin { } } +// Task to zip ml-commons plugin from archive task zipBwcMlCommonsPlugin(type: Zip) { dependsOn "pullMlCommonsBwcPlugin" from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml")) @@ -133,6 +143,7 @@ task zipBwcMlCommonsPlugin(type: Zip) { } } +// Task to zip knn plugin from archive task zipBwcKnnPlugin(type: Zip) { dependsOn "zipBwcMlCommonsPlugin" dependsOn "pullKnnBwcPlugin" @@ -144,7 +155,7 @@ task zipBwcKnnPlugin(type: Zip) { } } -// Task to zip plugin from archive +// Task to zip neural search plugin from archive task zipBwcPlugin(type: Zip) { dependsOn "zipBwcKnnPlugin" dependsOn "pullBwcPlugin" diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index ea8f3b830..c7bd6ae74 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -13,7 +13,6 @@ String default_bwc_version = System.getProperty("bwc.version") String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") String baseName = "neuralSearchBwcCluster-restart" -String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version List> plugins = [provider(new Callable(){ @Override @@ -37,7 +36,7 @@ List> plugins = [provider(new Callable(){ } }), rootProject.tasks.bundlePlugin.archiveFile] -// Creates a test cluster of previous version and loads k-NN plugin of bwcVersion +// Creates a test cluster of previous version and loads neuralsearch plugin of bwcVersion testClusters { "${baseName}" { testDistribution = "ARCHIVE" @@ -49,22 +48,11 @@ testClusters { plugin(project.tasks.zipBwcPlugin.archiveFile) setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' - environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" - if (Os.isFamily(Os.FAMILY_WINDOWS)) { - // While running on Windows OS, setting the PATH environment variable to include the paths to dlls of JNI libraries and windows dependencies - environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies") - systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" - } else { - systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" - } - } } // Task to run BWC tests against the old cluster task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { - dependsOn "zipBwcMlCommonsPlugin" - dependsOn "zipBwcKnnPlugin" dependsOn "zipBwcPlugin" useCluster testClusters."${baseName}" systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster' @@ -83,8 +71,6 @@ task testRestartUpgrade(type: StandaloneRestIntegTestTask) { dependsOn rootProject.tasks.assemble useCluster testClusters."${baseName}" doFirst { - testClusters."${baseName}".environment("LD_LIBRARY_PATH", "$rootDir/jni/release") - testClusters."${baseName}".systemProperty("java.library.path", "$rootDir/jni/release") testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion(plugins) } systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster' diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java index aa2fbbe0b..25f591d33 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java @@ -21,7 +21,7 @@ public abstract class AbstractRestartUpgradeRestTestCase extends BaseNeuralSearc @Before protected void setIndex() { - // Creating index name by concatenating "knn-bwc-" prefix with test method name + // Creating index name by concatenating "neural-bwc-" prefix with test method name // for all the tests in this sub-project testIndex = NEURAL_SEARCH_BWC_PREFIX + getTestName().toLowerCase(Locale.ROOT); } diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 76560a3cf..d7546f657 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -13,7 +13,6 @@ apply from : "$rootDir/qa/build.gradle" String default_bwc_version = System.getProperty("bwc.version") String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") -String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version String baseName = "neuralSearchBwcCluster-rolling" List> plugins = [provider(new Callable(){ @@ -50,22 +49,12 @@ testClusters { jvmArgs("-Xms1g", "-Xmx4g") setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' - environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" - if (Os.isFamily(Os.FAMILY_WINDOWS)) { - // While running on Windows OS, setting the PATH environment variable to include the paths to dlls of JNI libraries and windows dependencies - environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies") - systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" - } else { - systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" - } } } // Task to run BWC tests against the old cluster task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { - dependsOn "zipBwcMlCommonsPlugin" - dependsOn "zipBwcKnnPlugin" dependsOn "zipBwcPlugin" useCluster testClusters."${baseName}" systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster' @@ -102,8 +91,6 @@ task testAgainstTwoThirdsUpgradedCluster(type: StandaloneRestIntegTestTask) { dependsOn "testAgainstOneThirdUpgradedCluster" useCluster testClusters."${baseName}" doFirst { - testClusters."${baseName}".getNodes().getAt("${baseName}" + "-1").environment("LD_LIBRARY_PATH", "$rootDir/jni/release") - testClusters."${baseName}".getNodes().getAt("${baseName}" + "-1").systemProperty("java.library.path", "$rootDir/jni/release") testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins) } systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster' @@ -121,8 +108,6 @@ task testRollingUpgrade(type: StandaloneRestIntegTestTask) { dependsOn "testAgainstTwoThirdsUpgradedCluster" useCluster testClusters."${baseName}" doFirst { - testClusters."${baseName}".getNodes().getAt("${baseName}" + "-2").environment("LD_LIBRARY_PATH", "$rootDir/jni/release") - testClusters."${baseName}".getNodes().getAt("${baseName}" + "-2").systemProperty("java.library.path", "$rootDir/jni/release") testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins) } mustRunAfter "testAgainstOneThirdUpgradedCluster" diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java index 42418f901..49cdf39e8 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java @@ -25,7 +25,7 @@ public abstract class AbstractRollingUpgradeTestCase extends BaseNeuralSearchIT @Before protected void setIndex() { - // Creating index name by concatenating "knn-bwc-" prefix with test method name + // Creating index name by concatenating "neural-bwc-" prefix with test method name // for all the tests in this sub-project testIndex = NEURAL_SEARCH_BWC_PREFIX + getTestName().toLowerCase(Locale.ROOT); } From 563caf88f47eff410e1ccb61403cfe39cd1e3f2b Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Wed, 27 Dec 2023 13:00:56 -0800 Subject: [PATCH 25/45] Addressing comments of naveen Signed-off-by: Varun Jain --- qa/rolling-upgrade/build.gradle | 2 -- 1 file changed, 2 deletions(-) diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index d7546f657..e7f959dc5 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -72,8 +72,6 @@ task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) { dependsOn rootProject.tasks.assemble dependsOn "testAgainstOldCluster" doFirst { - testClusters."${baseName}".getNodes().getAt("${baseName}" + "-0").environment("LD_LIBRARY_PATH", "$rootDir/jni/release") - testClusters."${baseName}".getNodes().getAt("${baseName}" + "-0").systemProperty("java.library.path", "$rootDir/jni/release") testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins) } systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster' From 88823b3c53aa379c8b47304fe73080547ebbc201 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Wed, 27 Dec 2023 13:42:10 -0800 Subject: [PATCH 26/45] Addressing comments of Naveen Signed-off-by: Varun Jain --- qa/build.gradle | 2 +- qa/restart-upgrade/build.gradle | 5 ++++- qa/rolling-upgrade/build.gradle | 3 +++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/qa/build.gradle b/qa/build.gradle index ccaba4e24..0ce535ea0 100644 --- a/qa/build.gradle +++ b/qa/build.gradle @@ -145,8 +145,8 @@ task zipBwcMlCommonsPlugin(type: Zip) { // Task to zip knn plugin from archive task zipBwcKnnPlugin(type: Zip) { - dependsOn "zipBwcMlCommonsPlugin" dependsOn "pullKnnBwcPlugin" + dependsOn "zipBwcMlCommonsPlugin" from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn")) destinationDirectory = tmp_dir archiveFileName = "opensearch-knn-${neural_search_bwc_version_no_qualifier}.zip" diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index c7bd6ae74..d585e388d 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -13,6 +13,7 @@ String default_bwc_version = System.getProperty("bwc.version") String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") String baseName = "neuralSearchBwcCluster-restart" +String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version List> plugins = [provider(new Callable(){ @Override @@ -36,7 +37,7 @@ List> plugins = [provider(new Callable(){ } }), rootProject.tasks.bundlePlugin.archiveFile] -// Creates a test cluster of previous version and loads neuralsearch plugin of bwcVersion +// Creates a test cluster of previous version and loads k-NN plugin of bwcVersion testClusters { "${baseName}" { testDistribution = "ARCHIVE" @@ -48,6 +49,8 @@ testClusters { plugin(project.tasks.zipBwcPlugin.archiveFile) setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' + environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" + systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" } } diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index e7f959dc5..f2dbb80f8 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -13,6 +13,7 @@ apply from : "$rootDir/qa/build.gradle" String default_bwc_version = System.getProperty("bwc.version") String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") +String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version String baseName = "neuralSearchBwcCluster-rolling" List> plugins = [provider(new Callable(){ @@ -49,6 +50,8 @@ testClusters { jvmArgs("-Xms1g", "-Xmx4g") setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' + environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" + systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" } } From de02df2aac028900033f1f419669a08d8f4a9a72 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Wed, 27 Dec 2023 20:41:53 -0800 Subject: [PATCH 27/45] Addressing martin comments Signed-off-by: Varun Jain --- .../neuralsearch/bwc/SemanticSearch.java | 2 ++ .../resources/processor/IndexMappings.json | 4 ++-- .../neuralsearch/bwc/SemanticSearch.java | 2 ++ .../resources/processor/IndexMappings.json | 4 ++-- .../neuralsearch/BaseNeuralSearchIT.java | 21 +++++++++++++++++++ 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java index 26a4eb421..89df4e7db 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java @@ -47,6 +47,8 @@ public void testSemanticSearch() throws Exception{ assertNotNull(pipeline); String modelId=getModelId(pipeline, TEXT_EMBEDDING_PROCESSOR); validateTestIndex(modelId); + deletePipeline(PIPELINE_NAME); + deleteModel(modelId); deleteIndex(testIndex); } } diff --git a/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json b/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json index 70ecf8335..2e01a9c51 100644 --- a/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json +++ b/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json @@ -6,8 +6,8 @@ "refresh_interval": "30s", "default_pipeline": "%s" }, - "number_of_shards": 1, - "number_of_replicas": 0 + "number_of_shards": 6, + "number_of_replicas": 2 }, "mappings": { "properties": { diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java index 713c05d1b..b87b19961 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java @@ -57,6 +57,8 @@ public void testSemanticSearch() throws Exception{ int totalDocsCountUpgraded=3*NUM_DOCS; addDocument(testIndex, "2",TEST_FIELD,TEXT_UPGRADED); validateTestIndexOnUpgrade(totalDocsCountUpgraded, modelId, TEXT_UPGRADED); + deletePipeline(PIPELINE_NAME); + deleteModel(modelId); deleteIndex(testIndex); break; } diff --git a/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json b/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json index 03a683079..ae5b3d2e3 100644 --- a/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json +++ b/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json @@ -6,8 +6,8 @@ "refresh_interval": "30s", "default_pipeline": "%s" }, - "number_of_shards": 1, - "number_of_replicas": 0 + "number_of_shards": 6, + "number_of_replicas": 2 }, "mappings": { "properties": { diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java index 3e0a504ff..df0050476 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java @@ -828,6 +828,11 @@ protected void waitForClusterHealthGreen(String numOfNodes) throws IOException { /** * Add a single Doc to an index + * + * @param index name of the index + * @param docId + * @param fieldName name of the field + * @param text to be added */ protected void addDocument(String index, String docId, String fieldName, String text) throws IOException { Request request = new Request("PUT", "/" + index + "/_doc/" + docId + "?refresh=true"); @@ -843,6 +848,7 @@ protected void addDocument(String index, String docId, String fieldName, String /** * Get ingest pipeline + * @param pipelineName of the ingest pipeline */ @SneakyThrows protected Map getIngestionPipeline(String pipelineName) { @@ -854,6 +860,21 @@ protected Map getIngestionPipeline(String pipelineName) { return (Map) responseMap.get(pipelineName); } + /** + * Delete pipeline + * + * @param pipelineName of the pipeline + */ + @SneakyThrows + protected Map deletePipeline(String pipelineName) { + Request request = new Request("DELETE", "/_ingest/pipeline/" + pipelineName); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + String responseBody = EntityUtils.toString(response.getEntity()); + Map responseMap = createParser(XContentType.JSON.xContent(), responseBody).map(); + return responseMap; + } + /** * Enumeration for types of pipeline processors, used to lookup resources like create * processor request as those are type specific From d7c6151a3401e304f3ed0ed1d012465690eed8cb Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Wed, 27 Dec 2023 21:16:20 -0800 Subject: [PATCH 28/45] Addressing comments of martin Signed-off-by: Varun Jain --- .../src/test/resources/processor/IndexMappings.json | 2 +- .../src/test/resources/processor/IndexMappings.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json b/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json index 2e01a9c51..b62164732 100644 --- a/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json +++ b/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json @@ -7,7 +7,7 @@ "default_pipeline": "%s" }, "number_of_shards": 6, - "number_of_replicas": 2 + "number_of_replicas": 0 }, "mappings": { "properties": { diff --git a/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json b/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json index ae5b3d2e3..e9a2da2a1 100644 --- a/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json +++ b/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json @@ -7,7 +7,7 @@ "default_pipeline": "%s" }, "number_of_shards": 6, - "number_of_replicas": 2 + "number_of_replicas": 0 }, "mappings": { "properties": { From 820917ba6c52c0cdca87ba79965aa2028f0ff75e Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Thu, 28 Dec 2023 10:31:50 -0800 Subject: [PATCH 29/45] Apply formatting.xml in all lines Signed-off-by: Varun Jain --- .github/draft-release-notes-config.yml | 2 +- .github/workflows/add-untriaged.yml | 38 +++++----- ...backwards_compatibility_tests_workflow.yml | 2 +- .github/workflows/delete_backport_branch.yml | 6 +- .github/workflows/links.yml | 2 +- DEVELOPER_GUIDE.md | 14 ++-- README.md | 2 +- build.gradle | 12 +-- .../formatterConfig.xml | 0 formatter/license-header.txt | 8 ++ gradle/formatting.gradle | 32 ++++++++ qa/build.gradle | 2 +- .../AbstractRestartUpgradeRestTestCase.java | 19 +++-- .../neuralsearch/bwc/SemanticSearch.java | 56 +++++++------- .../CreateModelGroupRequestBody.json | 2 +- .../resources/processor/IndexMappings.json | 1 - .../processor/PipelineConfiguration.json | 2 +- .../processor/UploadModelRequestBody.json | 2 +- qa/rolling-upgrade/build.gradle | 2 +- .../bwc/AbstractRollingUpgradeTestCase.java | 20 ++--- .../neuralsearch/bwc/SemanticSearch.java | 73 ++++++++++--------- .../CreateModelGroupRequestBody.json | 2 +- .../resources/processor/IndexMappings.json | 2 +- .../processor/PipelineConfiguration.json | 2 +- .../processor/UploadModelRequestBody.json | 2 +- repositories.gradle | 2 +- settings.gradle | 2 +- .../neuralsearch/common/VectorUtil.java | 5 +- .../ml/MLCommonsClientAccessor.java | 5 +- .../neuralsearch/plugin/NeuralSearch.java | 5 +- .../processor/CompoundTopDocs.java | 5 +- .../processor/InferenceProcessor.java | 5 +- .../NeuralQueryEnricherProcessor.java | 8 +- .../processor/NormalizationProcessor.java | 5 +- .../NormalizationProcessorWorkflow.java | 5 +- .../processor/SparseEncodingProcessor.java | 5 +- .../processor/TextEmbeddingProcessor.java | 5 +- .../TextImageEmbeddingProcessor.java | 5 +- ...ithmeticMeanScoreCombinationTechnique.java | 5 +- ...eometricMeanScoreCombinationTechnique.java | 5 +- ...HarmonicMeanScoreCombinationTechnique.java | 5 +- .../combination/ScoreCombinationFactory.java | 5 +- .../ScoreCombinationTechnique.java | 5 +- .../combination/ScoreCombinationUtil.java | 5 +- .../processor/combination/ScoreCombiner.java | 5 +- .../NormalizationProcessorFactory.java | 5 +- .../SparseEncodingProcessorFactory.java | 9 ++- .../TextEmbeddingProcessorFactory.java | 9 ++- .../TextImageEmbeddingProcessorFactory.java | 5 +- .../L2ScoreNormalizationTechnique.java | 5 +- .../MinMaxScoreNormalizationTechnique.java | 5 +- .../ScoreNormalizationFactory.java | 5 +- .../ScoreNormalizationTechnique.java | 5 +- .../normalization/ScoreNormalizer.java | 5 +- .../neuralsearch/query/HybridQuery.java | 5 +- .../query/HybridQueryBuilder.java | 5 +- .../neuralsearch/query/HybridQueryScorer.java | 5 +- .../neuralsearch/query/HybridQueryWeight.java | 5 +- .../query/NeuralQueryBuilder.java | 5 +- .../query/NeuralSparseQueryBuilder.java | 5 +- .../visitor/NeuralSearchQueryVisitor.java | 5 +- .../search/HitsThresholdChecker.java | 5 +- .../search/HybridTopScoreDocCollector.java | 5 +- .../query/HybridQueryPhaseSearcher.java | 5 +- .../util/HybridSearchResultFormatUtil.java | 5 +- .../settings/NeuralSearchSettings.java | 5 +- .../util/NeuralSearchClusterUtil.java | 5 +- .../neuralsearch/util/RetryUtil.java | 5 +- .../neuralsearch/util/TokenWeightUtil.java | 5 +- .../neuralsearch/NeuralSearchIT.java | 5 +- .../neuralsearch/NeuralSearchTests.java | 5 +- ...ValidateDependentPluginInstallationIT.java | 5 +- .../neuralsearch/common/VectorUtilTests.java | 5 +- .../constants/TestCommonConstants.java | 5 +- .../ml/MLCommonsClientAccessorTests.java | 5 +- .../plugin/NeuralSearchTests.java | 5 +- .../processor/CompoundTopDocsTests.java | 5 +- .../NeuralQueryEnricherProcessorIT.java | 5 +- .../NeuralQueryEnricherProcessorTests.java | 5 +- .../processor/NormalizationProcessorIT.java | 5 +- .../NormalizationProcessorTests.java | 5 +- .../NormalizationProcessorWorkflowTests.java | 5 +- .../processor/ScoreCombinationIT.java | 5 +- .../ScoreCombinationTechniqueTests.java | 5 +- .../processor/ScoreNormalizationIT.java | 5 +- .../ScoreNormalizationTechniqueTests.java | 5 +- .../processor/SparseEncodingProcessIT.java | 5 +- .../SparseEncodingProcessorTests.java | 25 ++++++- .../processor/TextEmbeddingProcessorIT.java | 5 +- .../TextEmbeddingProcessorTests.java | 14 +++- .../TextImageEmbeddingProcessorIT.java | 5 +- .../TextImageEmbeddingProcessorTests.java | 5 +- ...ticMeanScoreCombinationTechniqueTests.java | 5 +- .../BaseScoreCombinationTechniqueTests.java | 5 +- ...ricMeanScoreCombinationTechniqueTests.java | 5 +- ...nicMeanScoreCombinationTechniqueTests.java | 5 +- .../ScoreCombinationFactoryTests.java | 5 +- .../ScoreCombinationUtilTests.java | 5 +- .../NormalizationProcessorFactoryTests.java | 5 +- ...xtImageEmbeddingProcessorFactoryTests.java | 5 +- .../L2ScoreNormalizationTechniqueTests.java | 5 +- ...inMaxScoreNormalizationTechniqueTests.java | 5 +- .../ScoreNormalizationFactoryTests.java | 5 +- .../query/HybridQueryBuilderTests.java | 5 +- .../neuralsearch/query/HybridQueryIT.java | 5 +- .../query/HybridQueryScorerTests.java | 5 +- .../neuralsearch/query/HybridQueryTests.java | 5 +- .../query/HybridQueryWeightTests.java | 5 +- .../query/NeuralQueryBuilderTests.java | 5 +- .../neuralsearch/query/NeuralQueryIT.java | 5 +- .../query/NeuralSparseQueryBuilderTests.java | 5 +- .../query/NeuralSparseQueryIT.java | 5 +- .../query/OpenSearchQueryTestCase.java | 5 +- .../NeuralSearchQueryVisitorTests.java | 5 +- .../search/HitsTresholdCheckerTests.java | 5 +- .../HybridTopScoreDocCollectorTests.java | 5 +- .../query/HybridQueryPhaseSearcherTests.java | 5 +- .../HybridSearchResultFormatUtilTests.java | 5 +- .../util/NeuralSearchClusterTestUtils.java | 5 +- .../util/NeuralSearchClusterUtilTests.java | 5 +- .../util/TokenWeightUtilTests.java | 5 +- .../CreateModelGroupRequestBody.json | 2 +- .../resources/processor/IndexMappings.json | 1 - .../SearchRequestPipelineConfiguration.json | 2 +- .../SparseEncodingIndexMappings.json | 2 +- .../SparseEncodingPipelineConfiguration.json | 2 +- .../UploadSparseEncodingModelRequestBody.json | 2 +- .../neuralsearch/BaseNeuralSearchIT.java | 5 +- .../neuralsearch/BaseSparseEncodingIT.java | 5 +- .../OpenSearchSecureRestTestCase.java | 5 +- .../opensearch/neuralsearch/TestUtils.java | 5 +- 131 files changed, 602 insertions(+), 248 deletions(-) rename formatterConfig.xml => formatter/formatterConfig.xml (100%) create mode 100644 formatter/license-header.txt create mode 100644 gradle/formatting.gradle diff --git a/.github/draft-release-notes-config.yml b/.github/draft-release-notes-config.yml index d168727b4..3fe43c8d4 100644 --- a/.github/draft-release-notes-config.yml +++ b/.github/draft-release-notes-config.yml @@ -37,4 +37,4 @@ categories: - 'Maintenance' - title: 'Refactoring' labels: - - 'Refactoring' \ No newline at end of file + - 'Refactoring' diff --git a/.github/workflows/add-untriaged.yml b/.github/workflows/add-untriaged.yml index 9dcc7020d..15b9a5565 100644 --- a/.github/workflows/add-untriaged.yml +++ b/.github/workflows/add-untriaged.yml @@ -1,19 +1,19 @@ -name: Apply 'untriaged' label during issue lifecycle - -on: - issues: - types: [opened, reopened, transferred] - -jobs: - apply-label: - runs-on: ubuntu-latest - steps: - - uses: actions/github-script@v6 - with: - script: | - github.rest.issues.addLabels({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - labels: ['untriaged'] - }) +name: Apply 'untriaged' label during issue lifecycle + +on: + issues: + types: [opened, reopened, transferred] + +jobs: + apply-label: + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@v6 + with: + script: | + github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: ['untriaged'] + }) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 2ef6b2e1c..9b1ca720a 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -77,4 +77,4 @@ jobs: name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Ubuntu run: | echo "Running rolling-upgrade backwards compatibility tests ..." - ./gradlew :qa:rolling-upgrade:testRollingUpgrade -Dtests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE \ No newline at end of file + ./gradlew :qa:rolling-upgrade:testRollingUpgrade -Dtests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml index d654df6b4..f24f022b0 100644 --- a/.github/workflows/delete_backport_branch.yml +++ b/.github/workflows/delete_backport_branch.yml @@ -1,9 +1,9 @@ name: Delete merged branch of the backport PRs -on: +on: pull_request: types: - closed - + jobs: delete-branch: runs-on: ubuntu-latest @@ -12,4 +12,4 @@ jobs: - name: Delete merged branch uses: SvanBoxel/delete-merged-branch@main env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml index 3d0b81ad5..f41cc14bc 100644 --- a/.github/workflows/links.yml +++ b/.github/workflows/links.yml @@ -20,4 +20,4 @@ jobs: env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - name: Fail if there were link errors - run: exit ${{ steps.lychee.outputs.exit_code }} \ No newline at end of file + run: exit ${{ steps.lychee.outputs.exit_code }} diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 0ce29d811..55678007e 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -33,7 +33,7 @@ git clone https://github.com/[your username]/neural-search.git #### JDK 11 -OpenSearch builds using Java 11 at a minimum. This means you must have a JDK 11 installed with the environment variable +OpenSearch builds using Java 11 at a minimum. This means you must have a JDK 11 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 11 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-11`. One easy way to get Java 11 on *nix is to use [sdkman](https://sdkman.io/). @@ -83,10 +83,10 @@ Please follow these formatting guidelines: ## Build -OpenSearch neural-search uses a [Gradle](https://docs.gradle.org/6.6.1/userguide/userguide.html) wrapper for its build. +OpenSearch neural-search uses a [Gradle](https://docs.gradle.org/6.6.1/userguide/userguide.html) wrapper for its build. Run `gradlew` on Unix systems. -Build OpenSearch neural-search using `gradlew build` +Build OpenSearch neural-search using `gradlew build` ``` ./gradlew build @@ -221,8 +221,8 @@ See [CONTRIBUTING](CONTRIBUTING.md). ## Backports -The Github workflow in [`backport.yml`](.github/workflows/backport.yml) creates backport PRs automatically when the -original PR with an appropriate label `backport ` is merged to main with the backport workflow -run successfully on the PR. For example, if a PR on main needs to be backported to `2.x` branch, add a label -`backport 2.x` to the PR and make sure the backport workflow runs on the PR along with other checks. Once this PR is +The Github workflow in [`backport.yml`](.github/workflows/backport.yml) creates backport PRs automatically when the +original PR with an appropriate label `backport ` is merged to main with the backport workflow +run successfully on the PR. For example, if a PR on main needs to be backported to `2.x` branch, add a label +`backport 2.x` to the PR and make sure the backport workflow runs on the PR along with other checks. Once this PR is merged to main, the workflow will create a backport PR to the `2.x` branch. diff --git a/README.md b/README.md index b40318f01..979a0c51d 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ![PRs welcome!](https://img.shields.io/badge/PRs-welcome!-success) ## OpenSearch Neural Search -**OpenSearch Neural Search** is an OpenSearch plugin that adds dense neural retrieval into the OpenSearch ecosystem. +**OpenSearch Neural Search** is an OpenSearch plugin that adds dense neural retrieval into the OpenSearch ecosystem. The plugin provides the capability for indexing documents and doing neural search on the indexed documents. ## Project Resources diff --git a/build.gradle b/build.gradle index 9d9115327..4b1b8e340 100644 --- a/build.gradle +++ b/build.gradle @@ -18,6 +18,7 @@ apply plugin: 'opensearch.pluginzip' apply plugin: 'jacoco' apply plugin: "com.diffplug.spotless" apply plugin: 'io.freefair.lombok' +apply from: 'gradle/formatting.gradle' def pluginName = 'opensearch-neural-search' def pluginDescription = 'A plugin that adds dense neural retrieval into the OpenSearch ecosytem' @@ -313,16 +314,6 @@ run { useCluster testClusters.integTest } -spotless { - java { - removeUnusedImports() - importOrder 'java', 'javax', 'org', 'com' - eclipse().configFile rootProject.file('formatterConfig.xml') - trimTrailingWhitespace() - endWithNewline() - } -} - jacocoTestReport { dependsOn integTest, test reports { @@ -331,6 +322,7 @@ jacocoTestReport { } } +check.dependsOn spotlessCheck check.dependsOn jacocoTestCoverageVerification jacocoTestCoverageVerification.dependsOn jacocoTestReport diff --git a/formatterConfig.xml b/formatter/formatterConfig.xml similarity index 100% rename from formatterConfig.xml rename to formatter/formatterConfig.xml diff --git a/formatter/license-header.txt b/formatter/license-header.txt new file mode 100644 index 000000000..098d3cb0b --- /dev/null +++ b/formatter/license-header.txt @@ -0,0 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ \ No newline at end of file diff --git a/gradle/formatting.gradle b/gradle/formatting.gradle new file mode 100644 index 000000000..60ac2368b --- /dev/null +++ b/gradle/formatting.gradle @@ -0,0 +1,32 @@ +allprojects { + project.apply plugin: "com.diffplug.spotless" + spotless { + java { + // Normally this isn't necessary, but we have Java sources in + // non-standard places + target '**/*.java' + + removeUnusedImports() + eclipse().configFile rootProject.file('formatter/formatterConfig.xml') + trimTrailingWhitespace() + endWithNewline(); + + custom 'Refuse wildcard imports', { + // Wildcard imports can't be resolved; fail the build + if (it =~ /\s+import .*\*;/) { + throw new AssertionError("Do not use wildcard imports. 'spotlessApply' cannot resolve this issue.") + } + } + } + format 'misc', { + target '*.md', '*.gradle', '**/*.json', '**/*.yaml', '**/*.yml', '**/*.svg' + + trimTrailingWhitespace() + endWithNewline() + } + format("license", { + licenseHeaderFile("${rootProject.file("formatter/license-header.txt")}", "package "); + target("src/*/java/**/*.java") + }) + } +} \ No newline at end of file diff --git a/qa/build.gradle b/qa/build.gradle index 0ce535ea0..cb812644f 100644 --- a/qa/build.gradle +++ b/qa/build.gradle @@ -171,4 +171,4 @@ task zipBwcPlugin(type: Zip) { task bwcTestSuite { dependsOn ":qa:restart-upgrade:testRestartUpgrade" dependsOn ":qa:rolling-upgrade:testRollingUpgrade" -} \ No newline at end of file +} diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java index 25f591d33..2ce037575 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.bwc; import java.util.Locale; @@ -44,12 +47,12 @@ protected boolean preserveTemplatesUponCompletion() { @Override protected final Settings restClientSettings() { return Settings.builder() - .put(super.restClientSettings()) - // increase the timeout here to 90 seconds to handle long waits for a green - // cluster health. the waits for green need to be longer than a minute to - // account for delayed shards - .put(OpenSearchRestTestCase.CLIENT_SOCKET_TIMEOUT, CLIENT_TIMEOUT_VALUE) - .build(); + .put(super.restClientSettings()) + // increase the timeout here to 90 seconds to handle long waits for a green + // cluster health. the waits for green need to be longer than a minute to + // account for delayed shards + .put(OpenSearchRestTestCase.CLIENT_SOCKET_TIMEOUT, CLIENT_TIMEOUT_VALUE) + .build(); } protected static final boolean isRunningAgainstOldCluster() { @@ -59,4 +62,4 @@ protected static final boolean isRunningAgainstOldCluster() { protected final Optional getBWCVersion() { return Optional.ofNullable(System.getProperty(BWC_VERSION, null)); } -} \ No newline at end of file +} diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java index 89df4e7db..0492fe077 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java @@ -1,16 +1,17 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.bwc; import com.carrotsearch.randomizedtesting.RandomizedTest; import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; -import java.util.ArrayList; - import static org.opensearch.neuralsearch.TestUtils.NODES_BWC_CLUSTER; @@ -18,34 +19,33 @@ import static org.opensearch.neuralsearch.TestUtils.TEXT_EMBEDDING_PROCESSOR; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; - -public class SemanticSearch extends AbstractRestartUpgradeRestTestCase{ +public class SemanticSearch extends AbstractRestartUpgradeRestTestCase { private static final String PIPELINE_NAME = "nlp-pipeline"; private static String DOC_ID = "0"; private static final String TEST_FIELD = "passage_text"; - private static final String TEXT= "Hello world"; + private static final String TEXT = "Hello world"; - //Test restart-upgrade Semantic Search - //Create Text Embedding Processor, Ingestion Pipeline and add document - //Validate process , pipeline and document count in restart-upgrade scenario - public void testSemanticSearch() throws Exception{ + // Test restart-upgrade Semantic Search + // Create Text Embedding Processor, Ingestion Pipeline and add document + // Validate process , pipeline and document count in restart-upgrade scenario + public void testSemanticSearch() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); - if (isRunningAgainstOldCluster()){ - String modelId= uploadTextEmbeddingModel(); + if (isRunningAgainstOldCluster()) { + String modelId = uploadTextEmbeddingModel(); loadModel(modelId); - createPipelineProcessor(modelId,PIPELINE_NAME); + createPipelineProcessor(modelId, PIPELINE_NAME); createIndexWithConfiguration( - testIndex, - Files.readString(Path.of(classLoader.getResource("processor/IndexMappings.json").toURI())), - PIPELINE_NAME + testIndex, + Files.readString(Path.of(classLoader.getResource("processor/IndexMappings.json").toURI())), + PIPELINE_NAME ); - addDocument(testIndex, DOC_ID,TEST_FIELD,TEXT); - }else { - Map pipeline= getIngestionPipeline(PIPELINE_NAME); + addDocument(testIndex, DOC_ID, TEST_FIELD, TEXT); + } else { + Map pipeline = getIngestionPipeline(PIPELINE_NAME); assertNotNull(pipeline); - String modelId=getModelId(pipeline, TEXT_EMBEDDING_PROCESSOR); + String modelId = getModelId(pipeline, TEXT_EMBEDDING_PROCESSOR); validateTestIndex(modelId); deletePipeline(PIPELINE_NAME); deleteModel(modelId); @@ -53,10 +53,9 @@ public void testSemanticSearch() throws Exception{ } } - private void validateTestIndex(String modelId) throws Exception { - int docCount=getDocCount(testIndex); - assertEquals(1,docCount); + int docCount = getDocCount(testIndex); + assertEquals(1, docCount); loadModel(modelId); NeuralQueryBuilder neuralQueryBuilder = new NeuralQueryBuilder(); neuralQueryBuilder.fieldName("passage_embedding"); @@ -67,7 +66,6 @@ private void validateTestIndex(String modelId) throws Exception { assertNotNull(response); } - private String uploadTextEmbeddingModel() throws Exception { String requestBody = Files.readString(Path.of(classLoader.getResource("processor/UploadModelRequestBody.json").toURI())); return registerModelGroupAndGetModelId(requestBody); @@ -75,10 +73,10 @@ private String uploadTextEmbeddingModel() throws Exception { private String registerModelGroupAndGetModelId(String requestBody) throws Exception { String modelGroupRegisterRequestBody = Files.readString( - Path.of(classLoader.getResource("processor/CreateModelGroupRequestBody.json").toURI()) + Path.of(classLoader.getResource("processor/CreateModelGroupRequestBody.json").toURI()) ).replace("", "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)); - String modelGroupId=registerModelGroup(modelGroupRegisterRequestBody); + String modelGroupId = registerModelGroup(modelGroupRegisterRequestBody); requestBody = requestBody.replace("", modelGroupId); @@ -86,8 +84,8 @@ private String registerModelGroupAndGetModelId(String requestBody) throws Except } protected void createPipelineProcessor(String modelId, String pipelineName, ProcessorType processorType) throws Exception { - String requestBody=Files.readString(Path.of(classLoader.getResource("processor/PipelineConfiguration.json").toURI())); - createPipelineProcessor(requestBody,pipelineName,modelId); + String requestBody = Files.readString(Path.of(classLoader.getResource("processor/PipelineConfiguration.json").toURI())); + createPipelineProcessor(requestBody, pipelineName, modelId); } - + } diff --git a/qa/restart-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json b/qa/restart-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json index 91f68e222..51511222c 100644 --- a/qa/restart-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json +++ b/qa/restart-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json @@ -1,4 +1,4 @@ { "name": "", "description": "This is a public model group" -} \ No newline at end of file +} diff --git a/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json b/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json index b62164732..cf8a5c317 100644 --- a/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json +++ b/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json @@ -30,4 +30,3 @@ } } } - diff --git a/qa/restart-upgrade/src/test/resources/processor/PipelineConfiguration.json b/qa/restart-upgrade/src/test/resources/processor/PipelineConfiguration.json index 73646329f..ee4e253a6 100644 --- a/qa/restart-upgrade/src/test/resources/processor/PipelineConfiguration.json +++ b/qa/restart-upgrade/src/test/resources/processor/PipelineConfiguration.json @@ -10,4 +10,4 @@ } } ] -} \ No newline at end of file +} diff --git a/qa/restart-upgrade/src/test/resources/processor/UploadModelRequestBody.json b/qa/restart-upgrade/src/test/resources/processor/UploadModelRequestBody.json index 6769e78c7..95f9c9cb5 100644 --- a/qa/restart-upgrade/src/test/resources/processor/UploadModelRequestBody.json +++ b/qa/restart-upgrade/src/test/resources/processor/UploadModelRequestBody.json @@ -12,4 +12,4 @@ "all_config": "{\"architectures\":[\"BertModel\"],\"max_position_embeddings\":512,\"model_type\":\"bert\",\"num_attention_heads\":12,\"num_hidden_layers\":6}" }, "url": "https://github.com/opensearch-project/ml-commons/blob/2.x/ml-algorithms/src/test/resources/org/opensearch/ml/engine/algorithms/text_embedding/traced_small_model.zip?raw=true" -} \ No newline at end of file +} diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index f2dbb80f8..00422f654 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -118,4 +118,4 @@ task testRollingUpgrade(type: StandaloneRestIntegTestTask) { nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") systemProperty 'tests.security.manager', 'false' -} \ No newline at end of file +} diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java index 49cdf39e8..090818991 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.bwc; import java.util.Locale; @@ -11,7 +14,6 @@ import org.opensearch.common.settings.Settings; import org.opensearch.neuralsearch.BaseNeuralSearchIT; import org.opensearch.test.rest.OpenSearchRestTestCase; -import static org.opensearch.neuralsearch.TestUtils.CLIENT_TIMEOUT_VALUE; import static org.opensearch.neuralsearch.TestUtils.OLD_CLUSTER; import static org.opensearch.neuralsearch.TestUtils.MIXED_CLUSTER; import static org.opensearch.neuralsearch.TestUtils.UPGRADED_CLUSTER; @@ -48,12 +50,12 @@ protected boolean preserveTemplatesUponCompletion() { @Override protected final Settings restClientSettings() { return Settings.builder() - .put(super.restClientSettings()) - // increase the timeout here to 90 seconds to handle long waits for a green - // cluster health. the waits for green need to be longer than a minute to - // account for delayed shards - .put(OpenSearchRestTestCase.CLIENT_SOCKET_TIMEOUT, "120s") - .build(); + .put(super.restClientSettings()) + // increase the timeout here to 90 seconds to handle long waits for a green + // cluster health. the waits for green need to be longer than a minute to + // account for delayed shards + .put(OpenSearchRestTestCase.CLIENT_SOCKET_TIMEOUT, "120s") + .build(); } protected enum ClusterType { @@ -87,4 +89,4 @@ protected final Optional getBWCVersion() { return Optional.ofNullable(System.getProperty(BWC_VERSION, null)); } -} \ No newline at end of file +} diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java index b87b19961..b810bff8c 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.bwc; import com.carrotsearch.randomizedtesting.RandomizedTest; @@ -14,48 +17,48 @@ import static org.opensearch.neuralsearch.TestUtils.TEXT_EMBEDDING_PROCESSOR; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; -public class SemanticSearch extends AbstractRollingUpgradeTestCase{ +public class SemanticSearch extends AbstractRollingUpgradeTestCase { private static final String PIPELINE_NAME = "nlp-pipeline"; private static final String TEST_FIELD = "test-field"; - private static final String TEXT= "Hello world"; - private static final String TEXT_MIXED= "Hello world mixed"; - private static final String TEXT_UPGRADED= "Hello world upgraded"; + private static final String TEXT = "Hello world"; + private static final String TEXT_MIXED = "Hello world mixed"; + private static final String TEXT_UPGRADED = "Hello world upgraded"; private static final int NUM_DOCS = 1; - //Test rolling-upgrade Semantic Search - //Create Text Embedding Processor, Ingestion Pipeline and add document - //Validate process , pipeline and document count in rolling-upgrade scenario - public void testSemanticSearch() throws Exception{ + // Test rolling-upgrade Semantic Search + // Create Text Embedding Processor, Ingestion Pipeline and add document + // Validate process , pipeline and document count in rolling-upgrade scenario + public void testSemanticSearch() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); - switch (getClusterType()){ + switch (getClusterType()) { case OLD: - String modelId= uploadTextEmbeddingModel(); + String modelId = uploadTextEmbeddingModel(); loadModel(modelId); - createPipelineProcessor(modelId,PIPELINE_NAME); + createPipelineProcessor(modelId, PIPELINE_NAME); createIndexWithConfiguration( - testIndex, - Files.readString(Path.of(classLoader.getResource("processor/IndexMappings.json").toURI())), - PIPELINE_NAME + testIndex, + Files.readString(Path.of(classLoader.getResource("processor/IndexMappings.json").toURI())), + PIPELINE_NAME ); - addDocument(testIndex, "0",TEST_FIELD,TEXT); + addDocument(testIndex, "0", TEST_FIELD, TEXT); break; case MIXED: - modelId=getModelId(PIPELINE_NAME); + modelId = getModelId(PIPELINE_NAME); int totalDocsCountMixed; - if (isFirstMixedRound()){ - totalDocsCountMixed=NUM_DOCS; + if (isFirstMixedRound()) { + totalDocsCountMixed = NUM_DOCS; validateTestIndexOnUpgrade(totalDocsCountMixed, modelId, TEXT); - addDocument(testIndex, "1",TEST_FIELD,TEXT_MIXED); - - }else{ - totalDocsCountMixed=2*NUM_DOCS; + addDocument(testIndex, "1", TEST_FIELD, TEXT_MIXED); + + } else { + totalDocsCountMixed = 2 * NUM_DOCS; validateTestIndexOnUpgrade(totalDocsCountMixed, modelId, TEXT_MIXED); } break; case UPGRADED: - modelId=getModelId(PIPELINE_NAME); - int totalDocsCountUpgraded=3*NUM_DOCS; - addDocument(testIndex, "2",TEST_FIELD,TEXT_UPGRADED); + modelId = getModelId(PIPELINE_NAME); + int totalDocsCountUpgraded = 3 * NUM_DOCS; + addDocument(testIndex, "2", TEST_FIELD, TEXT_UPGRADED); validateTestIndexOnUpgrade(totalDocsCountUpgraded, modelId, TEXT_UPGRADED); deletePipeline(PIPELINE_NAME); deleteModel(modelId); @@ -66,8 +69,8 @@ public void testSemanticSearch() throws Exception{ } private void validateTestIndexOnUpgrade(int numberOfDocs, String modelId, String text) throws Exception { - int docCount=getDocCount(testIndex); - assertEquals(numberOfDocs,docCount); + int docCount = getDocCount(testIndex); + assertEquals(numberOfDocs, docCount); loadModel(modelId); NeuralQueryBuilder neuralQueryBuilder = new NeuralQueryBuilder(); neuralQueryBuilder.fieldName("passage_embedding"); @@ -85,10 +88,10 @@ private String uploadTextEmbeddingModel() throws Exception { private String registerModelGroupAndGetModelId(String requestBody) throws Exception { String modelGroupRegisterRequestBody = Files.readString( - Path.of(classLoader.getResource("processor/CreateModelGroupRequestBody.json").toURI()) + Path.of(classLoader.getResource("processor/CreateModelGroupRequestBody.json").toURI()) ).replace("", "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)); - String modelGroupId=registerModelGroup(modelGroupRegisterRequestBody); + String modelGroupId = registerModelGroup(modelGroupRegisterRequestBody); requestBody = requestBody.replace("", modelGroupId); @@ -96,13 +99,13 @@ private String registerModelGroupAndGetModelId(String requestBody) throws Except } protected void createPipelineProcessor(String modelId, String pipelineName, ProcessorType processorType) throws Exception { - String requestBody=Files.readString(Path.of(classLoader.getResource("processor/PipelineConfiguration.json").toURI())); - createPipelineProcessor(requestBody,pipelineName,modelId); + String requestBody = Files.readString(Path.of(classLoader.getResource("processor/PipelineConfiguration.json").toURI())); + createPipelineProcessor(requestBody, pipelineName, modelId); } - private String getModelId(String pipelineName){ - Map pipeline = getIngestionPipeline(pipelineName); + private String getModelId(String pipelineName) { + Map pipeline = getIngestionPipeline(pipelineName); assertNotNull(pipeline); - return TestUtils.getModelId(pipeline,TEXT_EMBEDDING_PROCESSOR); + return TestUtils.getModelId(pipeline, TEXT_EMBEDDING_PROCESSOR); } } diff --git a/qa/rolling-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json b/qa/rolling-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json index 91f68e222..51511222c 100644 --- a/qa/rolling-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json +++ b/qa/rolling-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json @@ -1,4 +1,4 @@ { "name": "", "description": "This is a public model group" -} \ No newline at end of file +} diff --git a/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json b/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json index e9a2da2a1..cf8a5c317 100644 --- a/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json +++ b/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json @@ -29,4 +29,4 @@ } } } -} \ No newline at end of file +} diff --git a/qa/rolling-upgrade/src/test/resources/processor/PipelineConfiguration.json b/qa/rolling-upgrade/src/test/resources/processor/PipelineConfiguration.json index 73646329f..ee4e253a6 100644 --- a/qa/rolling-upgrade/src/test/resources/processor/PipelineConfiguration.json +++ b/qa/rolling-upgrade/src/test/resources/processor/PipelineConfiguration.json @@ -10,4 +10,4 @@ } } ] -} \ No newline at end of file +} diff --git a/qa/rolling-upgrade/src/test/resources/processor/UploadModelRequestBody.json b/qa/rolling-upgrade/src/test/resources/processor/UploadModelRequestBody.json index 6769e78c7..95f9c9cb5 100644 --- a/qa/rolling-upgrade/src/test/resources/processor/UploadModelRequestBody.json +++ b/qa/rolling-upgrade/src/test/resources/processor/UploadModelRequestBody.json @@ -12,4 +12,4 @@ "all_config": "{\"architectures\":[\"BertModel\"],\"max_position_embeddings\":512,\"model_type\":\"bert\",\"num_attention_heads\":12,\"num_hidden_layers\":6}" }, "url": "https://github.com/opensearch-project/ml-commons/blob/2.x/ml-algorithms/src/test/resources/org/opensearch/ml/engine/algorithms/text_embedding/traced_small_model.zip?raw=true" -} \ No newline at end of file +} diff --git a/repositories.gradle b/repositories.gradle index 0ce55eed3..16f39204b 100644 --- a/repositories.gradle +++ b/repositories.gradle @@ -8,4 +8,4 @@ repositories { maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } mavenCentral() maven { url "https://plugins.gradle.org/m2/" } -} \ No newline at end of file +} diff --git a/settings.gradle b/settings.gradle index 5bfca1add..232fd316c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -12,4 +12,4 @@ rootProject.name = 'neural-search' include ":qa" include ":qa:rolling-upgrade" -include ":qa:restart-upgrade" \ No newline at end of file +include ":qa:restart-upgrade" diff --git a/src/main/java/org/opensearch/neuralsearch/common/VectorUtil.java b/src/main/java/org/opensearch/neuralsearch/common/VectorUtil.java index 2198f7e9e..93eac30e5 100644 --- a/src/main/java/org/opensearch/neuralsearch/common/VectorUtil.java +++ b/src/main/java/org/opensearch/neuralsearch/common/VectorUtil.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.common; import java.util.List; diff --git a/src/main/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessor.java b/src/main/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessor.java index b9a95dea8..93c0c509f 100644 --- a/src/main/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessor.java +++ b/src/main/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessor.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.ml; import static org.opensearch.neuralsearch.processor.TextImageEmbeddingProcessor.INPUT_IMAGE; diff --git a/src/main/java/org/opensearch/neuralsearch/plugin/NeuralSearch.java b/src/main/java/org/opensearch/neuralsearch/plugin/NeuralSearch.java index f244419b3..250a1fd67 100644 --- a/src/main/java/org/opensearch/neuralsearch/plugin/NeuralSearch.java +++ b/src/main/java/org/opensearch/neuralsearch/plugin/NeuralSearch.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.plugin; import static org.opensearch.neuralsearch.settings.NeuralSearchSettings.NEURAL_SEARCH_HYBRID_SEARCH_DISABLED; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/CompoundTopDocs.java b/src/main/java/org/opensearch/neuralsearch/processor/CompoundTopDocs.java index b7aa43c64..6c54f101f 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/CompoundTopDocs.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/CompoundTopDocs.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import static org.opensearch.neuralsearch.search.util.HybridSearchResultFormatUtil.isHybridQueryDelimiterElement; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/InferenceProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/InferenceProcessor.java index e66c3f7f3..54b0ce6b3 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/InferenceProcessor.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/InferenceProcessor.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import java.util.ArrayList; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessor.java index c95e0f526..0a7cb6932 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessor.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessor.java @@ -1,12 +1,14 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; -import static org.opensearch.ingest.ConfigurationUtils.*; -import static org.opensearch.neuralsearch.processor.TextEmbeddingProcessor.TYPE; +import static org.opensearch.ingest.ConfigurationUtils.readOptionalStringProperty; import java.util.Map; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessor.java index 0d6742dbe..99c1d985f 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessor.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessor.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import static org.opensearch.neuralsearch.search.util.HybridSearchResultFormatUtil.isHybridQueryStartStopElement; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflow.java b/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflow.java index b8bc86de5..6d12077f0 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflow.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflow.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import java.util.Arrays; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessor.java index d1433e983..da596bda2 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessor.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessor.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import java.util.List; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessor.java index 3e87188e8..38b5ee81e 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessor.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessor.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import java.util.List; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessor.java index 9a252bace..53ca5d838 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessor.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessor.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import java.util.HashMap; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechnique.java index e656beca3..037a0f88f 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechnique.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.combination; import java.util.List; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechnique.java index 2a78d5ac6..95b4e2352 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechnique.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.combination; import java.util.List; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechnique.java index 0b45fb616..51576f210 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechnique.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.combination; import java.util.List; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactory.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactory.java index f05d24823..5541a4664 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactory.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactory.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.combination; import java.util.Map; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationTechnique.java index 6e0a5db65..da3de2d00 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationTechnique.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.combination; public interface ScoreCombinationTechnique { diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtil.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtil.java index c92abba4d..27a44e636 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtil.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtil.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.combination; import java.util.Arrays; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombiner.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombiner.java index 22745c50a..514d096fa 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombiner.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombiner.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.combination; import java.util.ArrayList; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactory.java b/src/main/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactory.java index 92ab2b1df..c809ccfca 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactory.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactory.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.factory; import static org.opensearch.ingest.ConfigurationUtils.readOptionalMap; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/factory/SparseEncodingProcessorFactory.java b/src/main/java/org/opensearch/neuralsearch/processor/factory/SparseEncodingProcessorFactory.java index 609ee07be..71fd3eae4 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/factory/SparseEncodingProcessorFactory.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/factory/SparseEncodingProcessorFactory.java @@ -1,13 +1,18 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.factory; import static org.opensearch.ingest.ConfigurationUtils.readMap; import static org.opensearch.ingest.ConfigurationUtils.readStringProperty; -import static org.opensearch.neuralsearch.processor.TextEmbeddingProcessor.*; +import static org.opensearch.neuralsearch.processor.TextEmbeddingProcessor.TYPE; +import static org.opensearch.neuralsearch.processor.TextEmbeddingProcessor.MODEL_ID_FIELD; +import static org.opensearch.neuralsearch.processor.TextEmbeddingProcessor.FIELD_MAP_FIELD; import java.util.Map; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/factory/TextEmbeddingProcessorFactory.java b/src/main/java/org/opensearch/neuralsearch/processor/factory/TextEmbeddingProcessorFactory.java index adf6f6d21..6fcae7a79 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/factory/TextEmbeddingProcessorFactory.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/factory/TextEmbeddingProcessorFactory.java @@ -1,13 +1,18 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.factory; import static org.opensearch.ingest.ConfigurationUtils.readMap; import static org.opensearch.ingest.ConfigurationUtils.readStringProperty; -import static org.opensearch.neuralsearch.processor.TextEmbeddingProcessor.*; +import static org.opensearch.neuralsearch.processor.TextEmbeddingProcessor.TYPE; +import static org.opensearch.neuralsearch.processor.TextEmbeddingProcessor.MODEL_ID_FIELD; +import static org.opensearch.neuralsearch.processor.TextEmbeddingProcessor.FIELD_MAP_FIELD; import java.util.Map; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactory.java b/src/main/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactory.java index 3820d84c7..953864ff8 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactory.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactory.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.factory; import static org.opensearch.ingest.ConfigurationUtils.readMap; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechnique.java index 7f7d18546..4d6fdb454 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechnique.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.normalization; import java.util.ArrayList; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechnique.java index bd1112840..1aeb10091 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechnique.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.normalization; import java.util.Arrays; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactory.java b/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactory.java index 667c237c7..b392ab977 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactory.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactory.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.normalization; import java.util.Map; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationTechnique.java index 8dd124804..3c1b5ea8c 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationTechnique.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.normalization; import java.util.List; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizer.java b/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizer.java index e2e3385eb..7ec071f55 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizer.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizer.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.normalization; import java.util.List; diff --git a/src/main/java/org/opensearch/neuralsearch/query/HybridQuery.java b/src/main/java/org/opensearch/neuralsearch/query/HybridQuery.java index 2c79c56e5..62f27bda0 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/HybridQuery.java +++ b/src/main/java/org/opensearch/neuralsearch/query/HybridQuery.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.query; import java.io.IOException; diff --git a/src/main/java/org/opensearch/neuralsearch/query/HybridQueryBuilder.java b/src/main/java/org/opensearch/neuralsearch/query/HybridQueryBuilder.java index a0019c826..37d576e9c 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/HybridQueryBuilder.java +++ b/src/main/java/org/opensearch/neuralsearch/query/HybridQueryBuilder.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.query; import java.io.IOException; diff --git a/src/main/java/org/opensearch/neuralsearch/query/HybridQueryScorer.java b/src/main/java/org/opensearch/neuralsearch/query/HybridQueryScorer.java index 109a50d05..266911f81 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/HybridQueryScorer.java +++ b/src/main/java/org/opensearch/neuralsearch/query/HybridQueryScorer.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.query; import java.io.IOException; diff --git a/src/main/java/org/opensearch/neuralsearch/query/HybridQueryWeight.java b/src/main/java/org/opensearch/neuralsearch/query/HybridQueryWeight.java index 605892ea0..57c84ffe1 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/HybridQueryWeight.java +++ b/src/main/java/org/opensearch/neuralsearch/query/HybridQueryWeight.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.query; import java.io.IOException; diff --git a/src/main/java/org/opensearch/neuralsearch/query/NeuralQueryBuilder.java b/src/main/java/org/opensearch/neuralsearch/query/NeuralQueryBuilder.java index 2e0e06adc..02abafc6a 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/NeuralQueryBuilder.java +++ b/src/main/java/org/opensearch/neuralsearch/query/NeuralQueryBuilder.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.query; import static org.opensearch.knn.index.query.KNNQueryBuilder.FILTER_FIELD; diff --git a/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java b/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java index 039d6a319..94c20430d 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java +++ b/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.query; import java.io.IOException; diff --git a/src/main/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitor.java b/src/main/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitor.java index 54b478bdc..9bf7b0e50 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitor.java +++ b/src/main/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitor.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.query.visitor; import java.util.Map; diff --git a/src/main/java/org/opensearch/neuralsearch/search/HitsThresholdChecker.java b/src/main/java/org/opensearch/neuralsearch/search/HitsThresholdChecker.java index c8c52320f..392ee9fef 100644 --- a/src/main/java/org/opensearch/neuralsearch/search/HitsThresholdChecker.java +++ b/src/main/java/org/opensearch/neuralsearch/search/HitsThresholdChecker.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.search; import java.util.Locale; diff --git a/src/main/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollector.java b/src/main/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollector.java index 556f09bfd..3f16d6911 100644 --- a/src/main/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollector.java +++ b/src/main/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollector.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.search; import java.io.IOException; diff --git a/src/main/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcher.java b/src/main/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcher.java index 539e0e22e..a29196aaf 100644 --- a/src/main/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcher.java +++ b/src/main/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcher.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.search.query; import static org.opensearch.neuralsearch.search.util.HybridSearchResultFormatUtil.createDelimiterElementForHybridSearchResults; diff --git a/src/main/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtil.java b/src/main/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtil.java index b345a105a..70cb0fa98 100644 --- a/src/main/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtil.java +++ b/src/main/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtil.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.search.util; import java.util.Objects; diff --git a/src/main/java/org/opensearch/neuralsearch/settings/NeuralSearchSettings.java b/src/main/java/org/opensearch/neuralsearch/settings/NeuralSearchSettings.java index 4ac88f75f..00d726ff4 100644 --- a/src/main/java/org/opensearch/neuralsearch/settings/NeuralSearchSettings.java +++ b/src/main/java/org/opensearch/neuralsearch/settings/NeuralSearchSettings.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.settings; import org.opensearch.common.settings.Setting; diff --git a/src/main/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtil.java b/src/main/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtil.java index 1f7dd9d4a..3fbe82787 100644 --- a/src/main/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtil.java +++ b/src/main/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtil.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.util; import org.opensearch.Version; diff --git a/src/main/java/org/opensearch/neuralsearch/util/RetryUtil.java b/src/main/java/org/opensearch/neuralsearch/util/RetryUtil.java index 5c1d486d4..5923c14be 100644 --- a/src/main/java/org/opensearch/neuralsearch/util/RetryUtil.java +++ b/src/main/java/org/opensearch/neuralsearch/util/RetryUtil.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.util; import java.util.List; diff --git a/src/main/java/org/opensearch/neuralsearch/util/TokenWeightUtil.java b/src/main/java/org/opensearch/neuralsearch/util/TokenWeightUtil.java index 76ce0fa16..bb98e7d32 100644 --- a/src/main/java/org/opensearch/neuralsearch/util/TokenWeightUtil.java +++ b/src/main/java/org/opensearch/neuralsearch/util/TokenWeightUtil.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.util; import java.util.ArrayList; diff --git a/src/test/java/org/opensearch/neuralsearch/NeuralSearchIT.java b/src/test/java/org/opensearch/neuralsearch/NeuralSearchIT.java index 38915e4b5..0b4d91ff5 100644 --- a/src/test/java/org/opensearch/neuralsearch/NeuralSearchIT.java +++ b/src/test/java/org/opensearch/neuralsearch/NeuralSearchIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch; import java.io.IOException; diff --git a/src/test/java/org/opensearch/neuralsearch/NeuralSearchTests.java b/src/test/java/org/opensearch/neuralsearch/NeuralSearchTests.java index 905b32aaa..435910e75 100644 --- a/src/test/java/org/opensearch/neuralsearch/NeuralSearchTests.java +++ b/src/test/java/org/opensearch/neuralsearch/NeuralSearchTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch; import org.opensearch.knn.common.KNNConstants; diff --git a/src/test/java/org/opensearch/neuralsearch/ValidateDependentPluginInstallationIT.java b/src/test/java/org/opensearch/neuralsearch/ValidateDependentPluginInstallationIT.java index b277f9a8f..0302352f6 100644 --- a/src/test/java/org/opensearch/neuralsearch/ValidateDependentPluginInstallationIT.java +++ b/src/test/java/org/opensearch/neuralsearch/ValidateDependentPluginInstallationIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch; import java.io.BufferedReader; diff --git a/src/test/java/org/opensearch/neuralsearch/common/VectorUtilTests.java b/src/test/java/org/opensearch/neuralsearch/common/VectorUtilTests.java index 1eba51e4f..4b960b6c0 100644 --- a/src/test/java/org/opensearch/neuralsearch/common/VectorUtilTests.java +++ b/src/test/java/org/opensearch/neuralsearch/common/VectorUtilTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.common; import java.util.Collections; diff --git a/src/test/java/org/opensearch/neuralsearch/constants/TestCommonConstants.java b/src/test/java/org/opensearch/neuralsearch/constants/TestCommonConstants.java index 185934b07..4e6269d06 100644 --- a/src/test/java/org/opensearch/neuralsearch/constants/TestCommonConstants.java +++ b/src/test/java/org/opensearch/neuralsearch/constants/TestCommonConstants.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.constants; import java.util.List; diff --git a/src/test/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessorTests.java b/src/test/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessorTests.java index 68d5f79eb..1d8f03ee1 100644 --- a/src/test/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessorTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.ml; import static org.mockito.Mockito.mock; diff --git a/src/test/java/org/opensearch/neuralsearch/plugin/NeuralSearchTests.java b/src/test/java/org/opensearch/neuralsearch/plugin/NeuralSearchTests.java index 32901cf12..b501782ad 100644 --- a/src/test/java/org/opensearch/neuralsearch/plugin/NeuralSearchTests.java +++ b/src/test/java/org/opensearch/neuralsearch/plugin/NeuralSearchTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.plugin; import static org.mockito.Mockito.mock; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/CompoundTopDocsTests.java b/src/test/java/org/opensearch/neuralsearch/processor/CompoundTopDocsTests.java index a5bdda1e3..120334c4b 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/CompoundTopDocsTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/CompoundTopDocsTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import java.util.Arrays; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java index 236a7199c..5d77c0f51 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import static org.opensearch.neuralsearch.TestUtils.TEST_DIMENSION; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorTests.java b/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorTests.java index f6de3e58d..693ec516d 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import java.util.Collections; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorIT.java index df3493bc5..bf2ddd456 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import static org.opensearch.neuralsearch.TestUtils.RELATION_EQUAL_TO; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorTests.java b/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorTests.java index 41348ec49..153eae5d0 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import static org.mockito.ArgumentMatchers.any; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflowTests.java b/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflowTests.java index 95c2ba0c2..4a0fb7bf8 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflowTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflowTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import static org.mockito.Mockito.spy; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationIT.java b/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationIT.java index 1aca59a5c..dfadb5019 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import static org.hamcrest.Matchers.allOf; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationTechniqueTests.java index 78c4e1139..f3d5876bc 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationTechniqueTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import static org.opensearch.neuralsearch.TestUtils.DELTA_FOR_SCORE_ASSERTION; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationIT.java b/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationIT.java index 41f961ef4..7047d7ca3 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import static org.opensearch.neuralsearch.TestUtils.TEST_DIMENSION; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationTechniqueTests.java index 123e57e74..28abe5b6c 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationTechniqueTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import java.util.List; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessIT.java b/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessIT.java index e575f8dcb..21687b717 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import java.nio.file.Files; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessorTests.java b/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessorTests.java index 7ddbfcc21..4a237881a 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessorTests.java @@ -1,15 +1,32 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.any; + +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import static org.mockito.Mockito.verify; -import java.util.*; +import java.util.Map; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + import java.util.function.BiConsumer; import java.util.stream.IntStream; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java index 43fd0b3d9..56684aeaf 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import java.nio.file.Files; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorTests.java b/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorTests.java index f1936d812..387fc244e 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorTests.java @@ -1,13 +1,23 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.isNull; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.isA; import java.util.ArrayList; import java.util.HashMap; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java index d96bc14d9..2c6aa9cdf 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import java.nio.file.Files; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorTests.java b/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorTests.java index 604e4d075..23a146d95 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor; import static org.mockito.ArgumentMatchers.anyMap; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechniqueTests.java index 125930007..98b3b1e11 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechniqueTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.combination; import static org.opensearch.neuralsearch.processor.combination.ArithmeticMeanScoreCombinationTechnique.PARAM_NAME_WEIGHTS; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/combination/BaseScoreCombinationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/combination/BaseScoreCombinationTechniqueTests.java index 6697ccd3e..fa96a9014 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/combination/BaseScoreCombinationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/combination/BaseScoreCombinationTechniqueTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.combination; import java.util.Arrays; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechniqueTests.java index 3f70c229f..6bac2c996 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechniqueTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.combination; import static org.opensearch.neuralsearch.processor.combination.ArithmeticMeanScoreCombinationTechnique.PARAM_NAME_WEIGHTS; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechniqueTests.java index 7b1b07f64..231e0853d 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechniqueTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.combination; import static org.opensearch.neuralsearch.processor.combination.ArithmeticMeanScoreCombinationTechnique.PARAM_NAME_WEIGHTS; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactoryTests.java b/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactoryTests.java index ee46bf0a1..24fadab94 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactoryTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactoryTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.combination; import static org.hamcrest.Matchers.containsString; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtilTests.java b/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtilTests.java index ca13cb2eb..646251e5b 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtilTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtilTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.combination; import static org.hamcrest.Matchers.allOf; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactoryTests.java b/src/test/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactoryTests.java index efd1cce4f..c84c045d0 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactoryTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactoryTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.factory; import static org.hamcrest.Matchers.containsString; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactoryTests.java b/src/test/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactoryTests.java index 564373315..b81eb6f64 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactoryTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactoryTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.factory; import static org.hamcrest.Matchers.allOf; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechniqueTests.java index 5732687ca..ea289d168 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechniqueTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.normalization; import java.util.Arrays; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechniqueTests.java index 1b38b7bb4..76e0e9f91 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechniqueTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.normalization; import java.util.List; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactoryTests.java b/src/test/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactoryTests.java index bf1489fe3..8cdf5a925 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactoryTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactoryTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.processor.normalization; import static org.hamcrest.Matchers.containsString; diff --git a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryBuilderTests.java b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryBuilderTests.java index 5e53dd69d..accbe0deb 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryBuilderTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryBuilderTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.query; import static org.hamcrest.Matchers.containsString; diff --git a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryIT.java b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryIT.java index 76db24df3..d3d4345ab 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryIT.java +++ b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.query; import static org.hamcrest.Matchers.allOf; diff --git a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryScorerTests.java b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryScorerTests.java index 3777c2d14..661faff5e 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryScorerTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryScorerTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.query; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; diff --git a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryTests.java b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryTests.java index a1d632755..01d02c89b 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.query; import static org.hamcrest.Matchers.containsString; diff --git a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryWeightTests.java b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryWeightTests.java index 50198af46..dd7b5672b 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryWeightTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryWeightTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.query; import static org.mockito.ArgumentMatchers.eq; diff --git a/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryBuilderTests.java b/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryBuilderTests.java index 068be51ef..21e612700 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryBuilderTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryBuilderTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.query; import static org.mockito.ArgumentMatchers.any; diff --git a/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryIT.java b/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryIT.java index a9e77262e..8ddd78c1a 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryIT.java +++ b/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.query; import static org.opensearch.neuralsearch.TestUtils.DELTA_FOR_SCORE_ASSERTION; diff --git a/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilderTests.java b/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilderTests.java index e95794b1c..4b1b926ce 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilderTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilderTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.query; import static org.mockito.ArgumentMatchers.any; diff --git a/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryIT.java b/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryIT.java index 0ec35ef81..d994f9b9a 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryIT.java +++ b/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.query; import static org.opensearch.neuralsearch.TestUtils.objectToFloat; diff --git a/src/test/java/org/opensearch/neuralsearch/query/OpenSearchQueryTestCase.java b/src/test/java/org/opensearch/neuralsearch/query/OpenSearchQueryTestCase.java index 5b73e12ec..39e7360b2 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/OpenSearchQueryTestCase.java +++ b/src/test/java/org/opensearch/neuralsearch/query/OpenSearchQueryTestCase.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.query; import static java.util.Collections.emptyMap; diff --git a/src/test/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitorTests.java b/src/test/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitorTests.java index 7570ece54..fe52f36ff 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitorTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.query.visitor; import java.util.HashMap; diff --git a/src/test/java/org/opensearch/neuralsearch/search/HitsTresholdCheckerTests.java b/src/test/java/org/opensearch/neuralsearch/search/HitsTresholdCheckerTests.java index 0a6a12c88..7771becb1 100644 --- a/src/test/java/org/opensearch/neuralsearch/search/HitsTresholdCheckerTests.java +++ b/src/test/java/org/opensearch/neuralsearch/search/HitsTresholdCheckerTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.search; import java.util.stream.IntStream; diff --git a/src/test/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollectorTests.java b/src/test/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollectorTests.java index 06bbfc416..3b2bf5a75 100644 --- a/src/test/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollectorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollectorTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.search; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; diff --git a/src/test/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcherTests.java b/src/test/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcherTests.java index 55a16c4bb..1eecd7661 100644 --- a/src/test/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcherTests.java +++ b/src/test/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcherTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.search.query; import static org.hamcrest.Matchers.containsString; diff --git a/src/test/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtilTests.java b/src/test/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtilTests.java index 65971b6d6..febc0dacc 100644 --- a/src/test/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtilTests.java +++ b/src/test/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtilTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.search.util; import static org.junit.Assert.assertEquals; diff --git a/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterTestUtils.java b/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterTestUtils.java index 30399cfea..71432a043 100644 --- a/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterTestUtils.java +++ b/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterTestUtils.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.util; import static org.mockito.Mockito.mock; diff --git a/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtilTests.java b/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtilTests.java index f85be25d5..e1c53a6a8 100644 --- a/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtilTests.java +++ b/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtilTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.util; import static org.opensearch.neuralsearch.util.NeuralSearchClusterTestUtils.mockClusterService; diff --git a/src/test/java/org/opensearch/neuralsearch/util/TokenWeightUtilTests.java b/src/test/java/org/opensearch/neuralsearch/util/TokenWeightUtilTests.java index a4bc2c495..a3c43e147 100644 --- a/src/test/java/org/opensearch/neuralsearch/util/TokenWeightUtilTests.java +++ b/src/test/java/org/opensearch/neuralsearch/util/TokenWeightUtilTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch.util; import java.util.List; diff --git a/src/test/resources/processor/CreateModelGroupRequestBody.json b/src/test/resources/processor/CreateModelGroupRequestBody.json index 91f68e222..51511222c 100644 --- a/src/test/resources/processor/CreateModelGroupRequestBody.json +++ b/src/test/resources/processor/CreateModelGroupRequestBody.json @@ -1,4 +1,4 @@ { "name": "", "description": "This is a public model group" -} \ No newline at end of file +} diff --git a/src/test/resources/processor/IndexMappings.json b/src/test/resources/processor/IndexMappings.json index 02de6c6af..13faad6c4 100644 --- a/src/test/resources/processor/IndexMappings.json +++ b/src/test/resources/processor/IndexMappings.json @@ -87,4 +87,3 @@ } } } - diff --git a/src/test/resources/processor/SearchRequestPipelineConfiguration.json b/src/test/resources/processor/SearchRequestPipelineConfiguration.json index 44d3b3ef0..5cd9988cb 100644 --- a/src/test/resources/processor/SearchRequestPipelineConfiguration.json +++ b/src/test/resources/processor/SearchRequestPipelineConfiguration.json @@ -8,4 +8,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/test/resources/processor/SparseEncodingIndexMappings.json b/src/test/resources/processor/SparseEncodingIndexMappings.json index 87dee278e..9748e8f3d 100644 --- a/src/test/resources/processor/SparseEncodingIndexMappings.json +++ b/src/test/resources/processor/SparseEncodingIndexMappings.json @@ -23,4 +23,4 @@ } } } -} \ No newline at end of file +} diff --git a/src/test/resources/processor/SparseEncodingPipelineConfiguration.json b/src/test/resources/processor/SparseEncodingPipelineConfiguration.json index 82d13c8fe..04a4baf80 100644 --- a/src/test/resources/processor/SparseEncodingPipelineConfiguration.json +++ b/src/test/resources/processor/SparseEncodingPipelineConfiguration.json @@ -15,4 +15,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/test/resources/processor/UploadSparseEncodingModelRequestBody.json b/src/test/resources/processor/UploadSparseEncodingModelRequestBody.json index c45334bae..50b4b8a9b 100644 --- a/src/test/resources/processor/UploadSparseEncodingModelRequestBody.json +++ b/src/test/resources/processor/UploadSparseEncodingModelRequestBody.json @@ -7,4 +7,4 @@ "model_group_id": "", "model_content_hash_value": "b345e9e943b62c405a8dd227ef2c46c84c5ff0a0b71b584be9132b37bce91a9a", "url": "https://github.com/opensearch-project/ml-commons/raw/main/ml-algorithms/src/test/resources/org/opensearch/ml/engine/algorithms/sparse_encoding/sparse_demo.zip" -} \ No newline at end of file +} diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java index df0050476..d38b3fc11 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch; import static org.opensearch.neuralsearch.common.VectorUtil.vectorAsListToArray; diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java index 811af20d0..cf6e0b164 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch; import java.nio.file.Files; diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java b/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java index 84966770c..2de6505f4 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch; import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_PER_ROUTE; diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java b/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java index 52b208e5d..ec40efb4a 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.neuralsearch; import static org.junit.Assert.assertEquals; From 69a41bccf7d2054e10776cc208e48e8325022d28 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Thu, 28 Dec 2023 11:09:55 -0800 Subject: [PATCH 30/45] Removing extra spacess from formatting.gradle Signed-off-by: Varun Jain --- gradle/formatting.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/formatting.gradle b/gradle/formatting.gradle index 60ac2368b..60cace641 100644 --- a/gradle/formatting.gradle +++ b/gradle/formatting.gradle @@ -19,7 +19,7 @@ allprojects { } } format 'misc', { - target '*.md', '*.gradle', '**/*.json', '**/*.yaml', '**/*.yml', '**/*.svg' + target '*.md', '**/*.gradle', '**/*.json', '**/*.yaml', '**/*.yml', '**/*.svg' trimTrailingWhitespace() endWithNewline() @@ -29,4 +29,4 @@ allprojects { target("src/*/java/**/*.java") }) } -} \ No newline at end of file +} From 2a71941bcf321e636072853f6a03ef829d9933b9 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Thu, 28 Dec 2023 11:21:42 -0800 Subject: [PATCH 31/45] Addressing martin comment Signed-off-by: Varun Jain --- formatter/license-header.txt | 4 ---- .../neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java | 4 ---- .../java/org/opensearch/neuralsearch/bwc/SemanticSearch.java | 4 ---- .../neuralsearch/bwc/AbstractRollingUpgradeTestCase.java | 4 ---- .../java/org/opensearch/neuralsearch/bwc/SemanticSearch.java | 4 ---- .../java/org/opensearch/neuralsearch/common/VectorUtil.java | 4 ---- .../opensearch/neuralsearch/ml/MLCommonsClientAccessor.java | 4 ---- .../java/org/opensearch/neuralsearch/plugin/NeuralSearch.java | 4 ---- .../opensearch/neuralsearch/processor/CompoundTopDocs.java | 4 ---- .../opensearch/neuralsearch/processor/InferenceProcessor.java | 4 ---- .../neuralsearch/processor/NeuralQueryEnricherProcessor.java | 4 ---- .../neuralsearch/processor/NormalizationProcessor.java | 4 ---- .../processor/NormalizationProcessorWorkflow.java | 4 ---- .../neuralsearch/processor/SparseEncodingProcessor.java | 4 ---- .../neuralsearch/processor/TextEmbeddingProcessor.java | 4 ---- .../neuralsearch/processor/TextImageEmbeddingProcessor.java | 4 ---- .../combination/ArithmeticMeanScoreCombinationTechnique.java | 4 ---- .../combination/GeometricMeanScoreCombinationTechnique.java | 4 ---- .../combination/HarmonicMeanScoreCombinationTechnique.java | 4 ---- .../processor/combination/ScoreCombinationFactory.java | 4 ---- .../processor/combination/ScoreCombinationTechnique.java | 4 ---- .../processor/combination/ScoreCombinationUtil.java | 4 ---- .../neuralsearch/processor/combination/ScoreCombiner.java | 4 ---- .../processor/factory/NormalizationProcessorFactory.java | 4 ---- .../processor/factory/SparseEncodingProcessorFactory.java | 4 ---- .../processor/factory/TextEmbeddingProcessorFactory.java | 4 ---- .../processor/factory/TextImageEmbeddingProcessorFactory.java | 4 ---- .../normalization/L2ScoreNormalizationTechnique.java | 4 ---- .../normalization/MinMaxScoreNormalizationTechnique.java | 4 ---- .../processor/normalization/ScoreNormalizationFactory.java | 4 ---- .../processor/normalization/ScoreNormalizationTechnique.java | 4 ---- .../neuralsearch/processor/normalization/ScoreNormalizer.java | 4 ---- .../java/org/opensearch/neuralsearch/query/HybridQuery.java | 4 ---- .../org/opensearch/neuralsearch/query/HybridQueryBuilder.java | 4 ---- .../org/opensearch/neuralsearch/query/HybridQueryScorer.java | 4 ---- .../org/opensearch/neuralsearch/query/HybridQueryWeight.java | 4 ---- .../org/opensearch/neuralsearch/query/NeuralQueryBuilder.java | 4 ---- .../neuralsearch/query/NeuralSparseQueryBuilder.java | 4 ---- .../neuralsearch/query/visitor/NeuralSearchQueryVisitor.java | 4 ---- .../opensearch/neuralsearch/search/HitsThresholdChecker.java | 4 ---- .../neuralsearch/search/HybridTopScoreDocCollector.java | 4 ---- .../neuralsearch/search/query/HybridQueryPhaseSearcher.java | 4 ---- .../search/util/HybridSearchResultFormatUtil.java | 4 ---- .../neuralsearch/settings/NeuralSearchSettings.java | 4 ---- .../opensearch/neuralsearch/util/NeuralSearchClusterUtil.java | 4 ---- src/main/java/org/opensearch/neuralsearch/util/RetryUtil.java | 4 ---- .../org/opensearch/neuralsearch/util/TokenWeightUtil.java | 4 ---- src/test/java/org/opensearch/neuralsearch/NeuralSearchIT.java | 4 ---- .../java/org/opensearch/neuralsearch/NeuralSearchTests.java | 4 ---- .../neuralsearch/ValidateDependentPluginInstallationIT.java | 4 ---- .../org/opensearch/neuralsearch/common/VectorUtilTests.java | 4 ---- .../neuralsearch/constants/TestCommonConstants.java | 4 ---- .../neuralsearch/ml/MLCommonsClientAccessorTests.java | 4 ---- .../org/opensearch/neuralsearch/plugin/NeuralSearchTests.java | 4 ---- .../neuralsearch/processor/CompoundTopDocsTests.java | 4 ---- .../processor/NeuralQueryEnricherProcessorIT.java | 4 ---- .../processor/NeuralQueryEnricherProcessorTests.java | 4 ---- .../neuralsearch/processor/NormalizationProcessorIT.java | 4 ---- .../neuralsearch/processor/NormalizationProcessorTests.java | 4 ---- .../processor/NormalizationProcessorWorkflowTests.java | 4 ---- .../opensearch/neuralsearch/processor/ScoreCombinationIT.java | 4 ---- .../processor/ScoreCombinationTechniqueTests.java | 4 ---- .../neuralsearch/processor/ScoreNormalizationIT.java | 4 ---- .../processor/ScoreNormalizationTechniqueTests.java | 4 ---- .../neuralsearch/processor/SparseEncodingProcessIT.java | 4 ---- .../neuralsearch/processor/SparseEncodingProcessorTests.java | 4 ---- .../neuralsearch/processor/TextEmbeddingProcessorIT.java | 4 ---- .../neuralsearch/processor/TextEmbeddingProcessorTests.java | 4 ---- .../neuralsearch/processor/TextImageEmbeddingProcessorIT.java | 4 ---- .../processor/TextImageEmbeddingProcessorTests.java | 4 ---- .../ArithmeticMeanScoreCombinationTechniqueTests.java | 4 ---- .../combination/BaseScoreCombinationTechniqueTests.java | 4 ---- .../GeometricMeanScoreCombinationTechniqueTests.java | 4 ---- .../HarmonicMeanScoreCombinationTechniqueTests.java | 4 ---- .../processor/combination/ScoreCombinationFactoryTests.java | 4 ---- .../processor/combination/ScoreCombinationUtilTests.java | 4 ---- .../processor/factory/NormalizationProcessorFactoryTests.java | 4 ---- .../factory/TextImageEmbeddingProcessorFactoryTests.java | 4 ---- .../normalization/L2ScoreNormalizationTechniqueTests.java | 4 ---- .../normalization/MinMaxScoreNormalizationTechniqueTests.java | 4 ---- .../normalization/ScoreNormalizationFactoryTests.java | 4 ---- .../neuralsearch/query/HybridQueryBuilderTests.java | 4 ---- .../java/org/opensearch/neuralsearch/query/HybridQueryIT.java | 4 ---- .../opensearch/neuralsearch/query/HybridQueryScorerTests.java | 4 ---- .../org/opensearch/neuralsearch/query/HybridQueryTests.java | 4 ---- .../opensearch/neuralsearch/query/HybridQueryWeightTests.java | 4 ---- .../neuralsearch/query/NeuralQueryBuilderTests.java | 4 ---- .../java/org/opensearch/neuralsearch/query/NeuralQueryIT.java | 4 ---- .../neuralsearch/query/NeuralSparseQueryBuilderTests.java | 4 ---- .../opensearch/neuralsearch/query/NeuralSparseQueryIT.java | 4 ---- .../neuralsearch/query/OpenSearchQueryTestCase.java | 4 ---- .../query/visitor/NeuralSearchQueryVisitorTests.java | 4 ---- .../neuralsearch/search/HitsTresholdCheckerTests.java | 4 ---- .../neuralsearch/search/HybridTopScoreDocCollectorTests.java | 4 ---- .../search/query/HybridQueryPhaseSearcherTests.java | 4 ---- .../search/util/HybridSearchResultFormatUtilTests.java | 4 ---- .../neuralsearch/util/NeuralSearchClusterTestUtils.java | 4 ---- .../neuralsearch/util/NeuralSearchClusterUtilTests.java | 4 ---- .../opensearch/neuralsearch/util/TokenWeightUtilTests.java | 4 ---- .../java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java | 4 ---- .../org/opensearch/neuralsearch/BaseSparseEncodingIT.java | 4 ---- .../opensearch/neuralsearch/OpenSearchSecureRestTestCase.java | 4 ---- .../java/org/opensearch/neuralsearch/TestUtils.java | 4 ---- 103 files changed, 412 deletions(-) diff --git a/formatter/license-header.txt b/formatter/license-header.txt index 098d3cb0b..1ce60bda7 100644 --- a/formatter/license-header.txt +++ b/formatter/license-header.txt @@ -1,8 +1,4 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ \ No newline at end of file diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java index 2ce037575..c1665529c 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.bwc; diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java index 0492fe077..48742ce16 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.bwc; diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java index 090818991..7712580ad 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.bwc; diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java index b810bff8c..5c8300271 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.bwc; diff --git a/src/main/java/org/opensearch/neuralsearch/common/VectorUtil.java b/src/main/java/org/opensearch/neuralsearch/common/VectorUtil.java index 93eac30e5..bfbb2e6d9 100644 --- a/src/main/java/org/opensearch/neuralsearch/common/VectorUtil.java +++ b/src/main/java/org/opensearch/neuralsearch/common/VectorUtil.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.common; diff --git a/src/main/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessor.java b/src/main/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessor.java index 93c0c509f..e12211d28 100644 --- a/src/main/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessor.java +++ b/src/main/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessor.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.ml; diff --git a/src/main/java/org/opensearch/neuralsearch/plugin/NeuralSearch.java b/src/main/java/org/opensearch/neuralsearch/plugin/NeuralSearch.java index 250a1fd67..aacb8d2e6 100644 --- a/src/main/java/org/opensearch/neuralsearch/plugin/NeuralSearch.java +++ b/src/main/java/org/opensearch/neuralsearch/plugin/NeuralSearch.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.plugin; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/CompoundTopDocs.java b/src/main/java/org/opensearch/neuralsearch/processor/CompoundTopDocs.java index 6c54f101f..9d3c3adb5 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/CompoundTopDocs.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/CompoundTopDocs.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/InferenceProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/InferenceProcessor.java index 54b0ce6b3..c34cd0b42 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/InferenceProcessor.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/InferenceProcessor.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessor.java index 0a7cb6932..b20b889cb 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessor.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessor.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessor.java index 99c1d985f..0563c92a0 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessor.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessor.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflow.java b/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflow.java index 6d12077f0..c19c2ae2b 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflow.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflow.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessor.java index da596bda2..8acf95bf7 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessor.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessor.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessor.java index 38b5ee81e..c1b8f92a6 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessor.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessor.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessor.java index 53ca5d838..09fcf3d97 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessor.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessor.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechnique.java index 037a0f88f..001f1670d 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechnique.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.combination; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechnique.java index 95b4e2352..c4b6dfb3f 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechnique.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.combination; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechnique.java index 51576f210..f5195f79f 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechnique.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.combination; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactory.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactory.java index 5541a4664..23d8e01be 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactory.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactory.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.combination; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationTechnique.java index da3de2d00..dbeabe94b 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationTechnique.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.combination; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtil.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtil.java index 27a44e636..a915057df 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtil.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtil.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.combination; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombiner.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombiner.java index 514d096fa..c9e0551e2 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombiner.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombiner.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.combination; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactory.java b/src/main/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactory.java index c809ccfca..0af46a4a4 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactory.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactory.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.factory; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/factory/SparseEncodingProcessorFactory.java b/src/main/java/org/opensearch/neuralsearch/processor/factory/SparseEncodingProcessorFactory.java index 71fd3eae4..75e00e85a 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/factory/SparseEncodingProcessorFactory.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/factory/SparseEncodingProcessorFactory.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.factory; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/factory/TextEmbeddingProcessorFactory.java b/src/main/java/org/opensearch/neuralsearch/processor/factory/TextEmbeddingProcessorFactory.java index 6fcae7a79..7802cb1f6 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/factory/TextEmbeddingProcessorFactory.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/factory/TextEmbeddingProcessorFactory.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.factory; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactory.java b/src/main/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactory.java index 953864ff8..b98f4fcc0 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactory.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactory.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.factory; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechnique.java index 4d6fdb454..2bb6bbed7 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechnique.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.normalization; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechnique.java index 1aeb10091..4fdf3c0a6 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechnique.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.normalization; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactory.java b/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactory.java index b392ab977..ca6ad20d6 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactory.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactory.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.normalization; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationTechnique.java index 3c1b5ea8c..0b784c678 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationTechnique.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.normalization; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizer.java b/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizer.java index 7ec071f55..263115f8f 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizer.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizer.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.normalization; diff --git a/src/main/java/org/opensearch/neuralsearch/query/HybridQuery.java b/src/main/java/org/opensearch/neuralsearch/query/HybridQuery.java index 62f27bda0..8846f6977 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/HybridQuery.java +++ b/src/main/java/org/opensearch/neuralsearch/query/HybridQuery.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.query; diff --git a/src/main/java/org/opensearch/neuralsearch/query/HybridQueryBuilder.java b/src/main/java/org/opensearch/neuralsearch/query/HybridQueryBuilder.java index 37d576e9c..b0104639b 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/HybridQueryBuilder.java +++ b/src/main/java/org/opensearch/neuralsearch/query/HybridQueryBuilder.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.query; diff --git a/src/main/java/org/opensearch/neuralsearch/query/HybridQueryScorer.java b/src/main/java/org/opensearch/neuralsearch/query/HybridQueryScorer.java index 266911f81..1620eeb71 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/HybridQueryScorer.java +++ b/src/main/java/org/opensearch/neuralsearch/query/HybridQueryScorer.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.query; diff --git a/src/main/java/org/opensearch/neuralsearch/query/HybridQueryWeight.java b/src/main/java/org/opensearch/neuralsearch/query/HybridQueryWeight.java index 57c84ffe1..69ee5015f 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/HybridQueryWeight.java +++ b/src/main/java/org/opensearch/neuralsearch/query/HybridQueryWeight.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.query; diff --git a/src/main/java/org/opensearch/neuralsearch/query/NeuralQueryBuilder.java b/src/main/java/org/opensearch/neuralsearch/query/NeuralQueryBuilder.java index 02abafc6a..cda01767e 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/NeuralQueryBuilder.java +++ b/src/main/java/org/opensearch/neuralsearch/query/NeuralQueryBuilder.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.query; diff --git a/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java b/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java index 94c20430d..226594a87 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java +++ b/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.query; diff --git a/src/main/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitor.java b/src/main/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitor.java index 9bf7b0e50..9dab0a695 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitor.java +++ b/src/main/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitor.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.query.visitor; diff --git a/src/main/java/org/opensearch/neuralsearch/search/HitsThresholdChecker.java b/src/main/java/org/opensearch/neuralsearch/search/HitsThresholdChecker.java index 392ee9fef..3c6a33d86 100644 --- a/src/main/java/org/opensearch/neuralsearch/search/HitsThresholdChecker.java +++ b/src/main/java/org/opensearch/neuralsearch/search/HitsThresholdChecker.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.search; diff --git a/src/main/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollector.java b/src/main/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollector.java index 3f16d6911..8b7a12d29 100644 --- a/src/main/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollector.java +++ b/src/main/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollector.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.search; diff --git a/src/main/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcher.java b/src/main/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcher.java index a29196aaf..bf05fdc9d 100644 --- a/src/main/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcher.java +++ b/src/main/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcher.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.search.query; diff --git a/src/main/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtil.java b/src/main/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtil.java index 70cb0fa98..162647b11 100644 --- a/src/main/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtil.java +++ b/src/main/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtil.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.search.util; diff --git a/src/main/java/org/opensearch/neuralsearch/settings/NeuralSearchSettings.java b/src/main/java/org/opensearch/neuralsearch/settings/NeuralSearchSettings.java index 00d726ff4..bf887830d 100644 --- a/src/main/java/org/opensearch/neuralsearch/settings/NeuralSearchSettings.java +++ b/src/main/java/org/opensearch/neuralsearch/settings/NeuralSearchSettings.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.settings; diff --git a/src/main/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtil.java b/src/main/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtil.java index 3fbe82787..d7d77b823 100644 --- a/src/main/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtil.java +++ b/src/main/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtil.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.util; diff --git a/src/main/java/org/opensearch/neuralsearch/util/RetryUtil.java b/src/main/java/org/opensearch/neuralsearch/util/RetryUtil.java index 5923c14be..d638fb9c8 100644 --- a/src/main/java/org/opensearch/neuralsearch/util/RetryUtil.java +++ b/src/main/java/org/opensearch/neuralsearch/util/RetryUtil.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.util; diff --git a/src/main/java/org/opensearch/neuralsearch/util/TokenWeightUtil.java b/src/main/java/org/opensearch/neuralsearch/util/TokenWeightUtil.java index bb98e7d32..e36b42cd6 100644 --- a/src/main/java/org/opensearch/neuralsearch/util/TokenWeightUtil.java +++ b/src/main/java/org/opensearch/neuralsearch/util/TokenWeightUtil.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.util; diff --git a/src/test/java/org/opensearch/neuralsearch/NeuralSearchIT.java b/src/test/java/org/opensearch/neuralsearch/NeuralSearchIT.java index 0b4d91ff5..58f05fd07 100644 --- a/src/test/java/org/opensearch/neuralsearch/NeuralSearchIT.java +++ b/src/test/java/org/opensearch/neuralsearch/NeuralSearchIT.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch; diff --git a/src/test/java/org/opensearch/neuralsearch/NeuralSearchTests.java b/src/test/java/org/opensearch/neuralsearch/NeuralSearchTests.java index 435910e75..3220ce8c6 100644 --- a/src/test/java/org/opensearch/neuralsearch/NeuralSearchTests.java +++ b/src/test/java/org/opensearch/neuralsearch/NeuralSearchTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch; diff --git a/src/test/java/org/opensearch/neuralsearch/ValidateDependentPluginInstallationIT.java b/src/test/java/org/opensearch/neuralsearch/ValidateDependentPluginInstallationIT.java index 0302352f6..ba9521303 100644 --- a/src/test/java/org/opensearch/neuralsearch/ValidateDependentPluginInstallationIT.java +++ b/src/test/java/org/opensearch/neuralsearch/ValidateDependentPluginInstallationIT.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch; diff --git a/src/test/java/org/opensearch/neuralsearch/common/VectorUtilTests.java b/src/test/java/org/opensearch/neuralsearch/common/VectorUtilTests.java index 4b960b6c0..a06e8f84d 100644 --- a/src/test/java/org/opensearch/neuralsearch/common/VectorUtilTests.java +++ b/src/test/java/org/opensearch/neuralsearch/common/VectorUtilTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.common; diff --git a/src/test/java/org/opensearch/neuralsearch/constants/TestCommonConstants.java b/src/test/java/org/opensearch/neuralsearch/constants/TestCommonConstants.java index 4e6269d06..3cf04bb05 100644 --- a/src/test/java/org/opensearch/neuralsearch/constants/TestCommonConstants.java +++ b/src/test/java/org/opensearch/neuralsearch/constants/TestCommonConstants.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.constants; diff --git a/src/test/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessorTests.java b/src/test/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessorTests.java index 1d8f03ee1..d9b816597 100644 --- a/src/test/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessorTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.ml; diff --git a/src/test/java/org/opensearch/neuralsearch/plugin/NeuralSearchTests.java b/src/test/java/org/opensearch/neuralsearch/plugin/NeuralSearchTests.java index b501782ad..2a66f6992 100644 --- a/src/test/java/org/opensearch/neuralsearch/plugin/NeuralSearchTests.java +++ b/src/test/java/org/opensearch/neuralsearch/plugin/NeuralSearchTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.plugin; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/CompoundTopDocsTests.java b/src/test/java/org/opensearch/neuralsearch/processor/CompoundTopDocsTests.java index 120334c4b..0096f7f94 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/CompoundTopDocsTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/CompoundTopDocsTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java index 5d77c0f51..3c8adf5b3 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorTests.java b/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorTests.java index 693ec516d..df5c4ad60 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorIT.java index bf2ddd456..5a48b9938 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorIT.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorTests.java b/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorTests.java index 153eae5d0..fb4a3dc57 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflowTests.java b/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflowTests.java index 4a0fb7bf8..f34f8f59b 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflowTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflowTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationIT.java b/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationIT.java index dfadb5019..d6467b065 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationIT.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationTechniqueTests.java index f3d5876bc..d2c1ddb4f 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationTechniqueTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationIT.java b/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationIT.java index 7047d7ca3..9f83a3608 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationIT.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationTechniqueTests.java index 28abe5b6c..34f0af48c 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationTechniqueTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessIT.java b/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessIT.java index 21687b717..416707956 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessIT.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessorTests.java b/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessorTests.java index 4a237881a..815ea851b 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessorTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java index 56684aeaf..cfdb6ebaf 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorTests.java b/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorTests.java index 387fc244e..8c2f1c1be 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java index 2c6aa9cdf..fa83ae4e0 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorTests.java b/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorTests.java index 23a146d95..431d6a440 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechniqueTests.java index 98b3b1e11..7d3b3fb61 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechniqueTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.combination; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/combination/BaseScoreCombinationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/combination/BaseScoreCombinationTechniqueTests.java index fa96a9014..ed06b0876 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/combination/BaseScoreCombinationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/combination/BaseScoreCombinationTechniqueTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.combination; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechniqueTests.java index 6bac2c996..495e2f4cd 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechniqueTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.combination; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechniqueTests.java index 231e0853d..0c6e1f81d 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechniqueTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.combination; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactoryTests.java b/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactoryTests.java index 24fadab94..b36a6b492 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactoryTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactoryTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.combination; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtilTests.java b/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtilTests.java index 646251e5b..9e00e3833 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtilTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtilTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.combination; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactoryTests.java b/src/test/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactoryTests.java index c84c045d0..9895d5b97 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactoryTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactoryTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.factory; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactoryTests.java b/src/test/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactoryTests.java index b81eb6f64..fa91d61a5 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactoryTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactoryTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.factory; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechniqueTests.java index ea289d168..bccc0820a 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechniqueTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.normalization; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechniqueTests.java index 76e0e9f91..1fe1edf25 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechniqueTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.normalization; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactoryTests.java b/src/test/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactoryTests.java index 8cdf5a925..d9dcd5540 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactoryTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactoryTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.processor.normalization; diff --git a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryBuilderTests.java b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryBuilderTests.java index accbe0deb..7bae96780 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryBuilderTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryBuilderTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.query; diff --git a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryIT.java b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryIT.java index d3d4345ab..da042ab0d 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryIT.java +++ b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryIT.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.query; diff --git a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryScorerTests.java b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryScorerTests.java index 661faff5e..1a2e3f26e 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryScorerTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryScorerTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.query; diff --git a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryTests.java b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryTests.java index 01d02c89b..dfbc97b69 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.query; diff --git a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryWeightTests.java b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryWeightTests.java index dd7b5672b..dad497a57 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryWeightTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryWeightTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.query; diff --git a/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryBuilderTests.java b/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryBuilderTests.java index 21e612700..dd63abbea 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryBuilderTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryBuilderTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.query; diff --git a/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryIT.java b/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryIT.java index 8ddd78c1a..47f5cbb06 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryIT.java +++ b/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryIT.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.query; diff --git a/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilderTests.java b/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilderTests.java index 4b1b926ce..2220ce326 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilderTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilderTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.query; diff --git a/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryIT.java b/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryIT.java index d994f9b9a..afabd46c9 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryIT.java +++ b/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryIT.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.query; diff --git a/src/test/java/org/opensearch/neuralsearch/query/OpenSearchQueryTestCase.java b/src/test/java/org/opensearch/neuralsearch/query/OpenSearchQueryTestCase.java index 39e7360b2..97e1f7c5d 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/OpenSearchQueryTestCase.java +++ b/src/test/java/org/opensearch/neuralsearch/query/OpenSearchQueryTestCase.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.query; diff --git a/src/test/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitorTests.java b/src/test/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitorTests.java index fe52f36ff..e513ab035 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitorTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.query.visitor; diff --git a/src/test/java/org/opensearch/neuralsearch/search/HitsTresholdCheckerTests.java b/src/test/java/org/opensearch/neuralsearch/search/HitsTresholdCheckerTests.java index 7771becb1..0fbef90b8 100644 --- a/src/test/java/org/opensearch/neuralsearch/search/HitsTresholdCheckerTests.java +++ b/src/test/java/org/opensearch/neuralsearch/search/HitsTresholdCheckerTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.search; diff --git a/src/test/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollectorTests.java b/src/test/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollectorTests.java index 3b2bf5a75..b296a5c3a 100644 --- a/src/test/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollectorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollectorTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.search; diff --git a/src/test/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcherTests.java b/src/test/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcherTests.java index 1eecd7661..e609eec05 100644 --- a/src/test/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcherTests.java +++ b/src/test/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcherTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.search.query; diff --git a/src/test/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtilTests.java b/src/test/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtilTests.java index febc0dacc..16f2f10ce 100644 --- a/src/test/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtilTests.java +++ b/src/test/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtilTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.search.util; diff --git a/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterTestUtils.java b/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterTestUtils.java index 71432a043..6219a26b5 100644 --- a/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterTestUtils.java +++ b/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterTestUtils.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.util; diff --git a/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtilTests.java b/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtilTests.java index e1c53a6a8..07975bd7c 100644 --- a/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtilTests.java +++ b/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtilTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.util; diff --git a/src/test/java/org/opensearch/neuralsearch/util/TokenWeightUtilTests.java b/src/test/java/org/opensearch/neuralsearch/util/TokenWeightUtilTests.java index a3c43e147..887d8fc17 100644 --- a/src/test/java/org/opensearch/neuralsearch/util/TokenWeightUtilTests.java +++ b/src/test/java/org/opensearch/neuralsearch/util/TokenWeightUtilTests.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch.util; diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java index d38b3fc11..52719239d 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch; diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java index cf6e0b164..4e547e090 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch; diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java b/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java index 2de6505f4..133f42daf 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch; diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java b/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java index ec40efb4a..99e396395 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java @@ -1,10 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.neuralsearch; From 3cd927b7b0f7dd7dab378808abdbd8ad76cedcba Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Thu, 28 Dec 2023 15:20:09 -0800 Subject: [PATCH 32/45] Addressing Jack comments Signed-off-by: Varun Jain --- qa/build.gradle | 61 +++++++++++++------ qa/restart-upgrade/build.gradle | 30 +-------- ...anticSearch.java => SemanticSearchIT.java} | 2 +- qa/rolling-upgrade/build.gradle | 35 +---------- ...anticSearch.java => SemanticSearchIT.java} | 2 +- 5 files changed, 49 insertions(+), 81 deletions(-) rename qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/{SemanticSearch.java => SemanticSearchIT.java} (97%) rename qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/{SemanticSearch.java => SemanticSearchIT.java} (98%) diff --git a/qa/build.gradle b/qa/build.gradle index cb812644f..885aef7d3 100644 --- a/qa/build.gradle +++ b/qa/build.gradle @@ -4,9 +4,10 @@ */ import org.apache.tools.ant.taskdefs.condition.Os -import org.w3c.dom.Text import java.nio.file.Files +import java.util.concurrent.Callable +import java.nio.file.Path apply plugin: 'opensearch.testclusters' apply plugin: 'opensearch.build' @@ -60,7 +61,31 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { file_ext = "zip" } -task deletetempDirectories { +ext{ + plugins = [provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.matching{include "**/opensearch-ml-plugin-${opensearch_build}.zip"}.getSingleFile() + } + } + } + }), provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.matching{include "**/opensearch-knn-${opensearch_build}.zip"}.getSingleFile() + } + } + } + }), rootProject.tasks.bundlePlugin.archiveFile] +} + +task deleteTempDirectories { doFirst { File[] tempFiles = tmp_dir.listFiles() for (File child : tempFiles) { @@ -73,7 +98,7 @@ task deletetempDirectories { // Task to pull opensearch artifact from archive task pullOpensearchArtifact { - dependsOn "deletetempDirectories" + dependsOn "deleteTempDirectories" doLast{ ext{ @@ -90,9 +115,9 @@ task pullOpensearchArtifact { ) copy { if (Os.isFamily(Os.FAMILY_WINDOWS)) { - from zipTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) + from zipTree(Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) } else { - from tarTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) + from tarTree(Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) } into tmp_dir.absolutePath } @@ -103,8 +128,8 @@ task pullOpensearchArtifact { task pullMlCommonsBwcPlugin { doLast { copy { - from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-ml")) - into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml") + from(Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-ml")) + into Path.of(tmp_dir.absolutePath, "opensearch-ml") } } } @@ -115,8 +140,8 @@ task pullKnnBwcPlugin { doLast { copy { - from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-knn")) - into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn") + from(Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-knn")) + into Path.of(tmp_dir.absolutePath, "opensearch-knn") } } } @@ -125,21 +150,21 @@ task pullKnnBwcPlugin { task pullBwcPlugin { doLast { copy { - from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-neural-search")) - into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search") + from(Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-neural-search")) + into Path.of(tmp_dir.absolutePath, "opensearch-neural-search") } - delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}"), java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}") + delete Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}"), java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}") } } // Task to zip ml-commons plugin from archive task zipBwcMlCommonsPlugin(type: Zip) { dependsOn "pullMlCommonsBwcPlugin" - from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml")) + from(Path.of(tmp_dir.absolutePath, "opensearch-ml")) destinationDirectory = tmp_dir archiveFileName = "opensearch-ml-${neural_search_bwc_version_no_qualifier}.zip" doLast { - delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml") + delete Path.of(tmp_dir.absolutePath, "opensearch-ml") } } @@ -147,11 +172,11 @@ task zipBwcMlCommonsPlugin(type: Zip) { task zipBwcKnnPlugin(type: Zip) { dependsOn "pullKnnBwcPlugin" dependsOn "zipBwcMlCommonsPlugin" - from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn")) + from(Path.of(tmp_dir.absolutePath, "opensearch-knn")) destinationDirectory = tmp_dir archiveFileName = "opensearch-knn-${neural_search_bwc_version_no_qualifier}.zip" doLast { - delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn") + delete Path.of(tmp_dir.absolutePath, "opensearch-knn") } } @@ -159,11 +184,11 @@ task zipBwcKnnPlugin(type: Zip) { task zipBwcPlugin(type: Zip) { dependsOn "zipBwcKnnPlugin" dependsOn "pullBwcPlugin" - from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search")) + from(Path.of(tmp_dir.absolutePath, "opensearch-neural-search")) destinationDirectory = tmp_dir archiveFileName = "opensearch-neural-search-${neural_search_bwc_version_no_qualifier}.zip" doLast { - delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search") + delete Path.of(tmp_dir.absolutePath, "opensearch-neural-search") } } diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index d585e388d..957c2a45d 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -4,38 +4,12 @@ */ import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask -import org.apache.tools.ant.taskdefs.condition.Os -import java.util.concurrent.Callable apply from : "$rootDir/qa/build.gradle" String default_bwc_version = System.getProperty("bwc.version") String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) -boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") String baseName = "neuralSearchBwcCluster-restart" -String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version - -List> plugins = [provider(new Callable(){ - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - return configurations.zipArchive.asFileTree.matching{include "**/opensearch-ml-plugin-${opensearch_build}.zip"}.getSingleFile() - } - } - } -}), provider(new Callable(){ - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - return configurations.zipArchive.asFileTree.matching{include "**/opensearch-knn-${opensearch_build}.zip"}.getSingleFile() - } - } - } -}), rootProject.tasks.bundlePlugin.archiveFile] // Creates a test cluster of previous version and loads k-NN plugin of bwcVersion testClusters { @@ -49,8 +23,6 @@ testClusters { plugin(project.tasks.zipBwcPlugin.archiveFile) setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' - environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" - systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" } } @@ -74,7 +46,7 @@ task testRestartUpgrade(type: StandaloneRestIntegTestTask) { dependsOn rootProject.tasks.assemble useCluster testClusters."${baseName}" doFirst { - testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion(plugins) + testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion(project.ext.plugins) } systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster' systemProperty 'tests.skip_delete_model_index', 'true' diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java similarity index 97% rename from qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java rename to qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java index 48742ce16..fb852b6ff 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -15,7 +15,7 @@ import static org.opensearch.neuralsearch.TestUtils.TEXT_EMBEDDING_PROCESSOR; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; -public class SemanticSearch extends AbstractRestartUpgradeRestTestCase { +public class SemanticSearchIT extends AbstractRestartUpgradeRestTestCase { private static final String PIPELINE_NAME = "nlp-pipeline"; private static String DOC_ID = "0"; diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 00422f654..db219ad3d 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -4,40 +4,13 @@ */ import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask -import org.apache.tools.ant.taskdefs.condition.Os - -import java.util.concurrent.Callable apply from : "$rootDir/qa/build.gradle" String default_bwc_version = System.getProperty("bwc.version") String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) -boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") -String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version String baseName = "neuralSearchBwcCluster-rolling" -List> plugins = [provider(new Callable(){ - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - return configurations.zipArchive.asFileTree.matching{include "**/opensearch-ml-plugin-${opensearch_build}.zip"}.getSingleFile() - } - } - } -}), provider(new Callable(){ - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - return configurations.zipArchive.asFileTree.matching{include "**/opensearch-knn-${opensearch_build}.zip"}.getSingleFile() - } - } - } -}), rootProject.tasks.bundlePlugin.archiveFile] - // Creates a test cluster of previous version and loads k-NN plugin of bwcVersion testClusters { "${baseName}" { @@ -50,8 +23,6 @@ testClusters { jvmArgs("-Xms1g", "-Xmx4g") setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' - environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" - systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" } } @@ -75,7 +46,7 @@ task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) { dependsOn rootProject.tasks.assemble dependsOn "testAgainstOldCluster" doFirst { - testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins) + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(project.ext.plugins) } systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster' systemProperty 'tests.rest.first_round', 'true' @@ -92,7 +63,7 @@ task testAgainstTwoThirdsUpgradedCluster(type: StandaloneRestIntegTestTask) { dependsOn "testAgainstOneThirdUpgradedCluster" useCluster testClusters."${baseName}" doFirst { - testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins) + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(project.ext.plugins) } systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster' systemProperty 'tests.rest.first_round', 'false' @@ -109,7 +80,7 @@ task testRollingUpgrade(type: StandaloneRestIntegTestTask) { dependsOn "testAgainstTwoThirdsUpgradedCluster" useCluster testClusters."${baseName}" doFirst { - testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins) + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(project.ext.plugins) } mustRunAfter "testAgainstOneThirdUpgradedCluster" systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster' diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java similarity index 98% rename from qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java rename to qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java index 5c8300271..936c7dc11 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearch.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -13,7 +13,7 @@ import static org.opensearch.neuralsearch.TestUtils.TEXT_EMBEDDING_PROCESSOR; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; -public class SemanticSearch extends AbstractRollingUpgradeTestCase { +public class SemanticSearchIT extends AbstractRollingUpgradeTestCase { private static final String PIPELINE_NAME = "nlp-pipeline"; private static final String TEST_FIELD = "test-field"; private static final String TEXT = "Hello world"; From 3a3d17536bd468111289527aaf4fa83f3e111b1d Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Fri, 29 Dec 2023 14:44:28 -0800 Subject: [PATCH 33/45] Addressing Jack comments Signed-off-by: Varun Jain --- qa/build.gradle | 18 +++++----- qa/restart-upgrade/build.gradle | 10 +++--- .../AbstractRestartUpgradeRestTestCase.java | 6 ++-- .../neuralsearch/bwc/SemanticSearchIT.java | 27 +++++++-------- .../CreateModelGroupRequestBody.json | 2 +- .../processor/UploadModelRequestBody.json | 2 +- qa/rolling-upgrade/build.gradle | 8 ++--- .../bwc/AbstractRollingUpgradeTestCase.java | 6 ++-- .../neuralsearch/bwc/SemanticSearchIT.java | 34 +++++++++---------- .../CreateModelGroupRequestBody.json | 2 +- .../processor/UploadModelRequestBody.json | 2 +- .../neuralsearch/BaseNeuralSearchIT.java | 12 +++---- 12 files changed, 63 insertions(+), 66 deletions(-) diff --git a/qa/build.gradle b/qa/build.gradle index 885aef7d3..511dcc442 100644 --- a/qa/build.gradle +++ b/qa/build.gradle @@ -157,8 +157,8 @@ task pullBwcPlugin { } } -// Task to zip ml-commons plugin from archive -task zipBwcMlCommonsPlugin(type: Zip) { +// Task to unzip ml-commons plugin from archive +task unZipBwcMlCommonsPlugin(type: Zip) { dependsOn "pullMlCommonsBwcPlugin" from(Path.of(tmp_dir.absolutePath, "opensearch-ml")) destinationDirectory = tmp_dir @@ -168,10 +168,10 @@ task zipBwcMlCommonsPlugin(type: Zip) { } } -// Task to zip knn plugin from archive -task zipBwcKnnPlugin(type: Zip) { +// Task to unzip knn plugin from archive +task unZipBwcKnnPlugin(type: Zip) { dependsOn "pullKnnBwcPlugin" - dependsOn "zipBwcMlCommonsPlugin" + dependsOn "unZipBwcMlCommonsPlugin" from(Path.of(tmp_dir.absolutePath, "opensearch-knn")) destinationDirectory = tmp_dir archiveFileName = "opensearch-knn-${neural_search_bwc_version_no_qualifier}.zip" @@ -180,9 +180,9 @@ task zipBwcKnnPlugin(type: Zip) { } } -// Task to zip neural search plugin from archive -task zipBwcPlugin(type: Zip) { - dependsOn "zipBwcKnnPlugin" +// Task to unzip neural search plugin from archive +task unZipBwcPlugin(type: Zip) { + dependsOn "unZipBwcKnnPlugin" dependsOn "pullBwcPlugin" from(Path.of(tmp_dir.absolutePath, "opensearch-neural-search")) destinationDirectory = tmp_dir @@ -194,6 +194,6 @@ task zipBwcPlugin(type: Zip) { task bwcTestSuite { - dependsOn ":qa:restart-upgrade:testRestartUpgrade" + dependsOn ":qa:restart-upgrade:testAgainstNewCluster" dependsOn ":qa:rolling-upgrade:testRollingUpgrade" } diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index 957c2a45d..b817c1395 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -18,9 +18,9 @@ testClusters { versions = [neural_search_bwc_version, opensearch_version] numberOfNodes = 3 jvmArgs("-Xms1g", "-Xmx4g") - plugin(project.tasks.zipBwcMlCommonsPlugin.archiveFile) - plugin(project.tasks.zipBwcKnnPlugin.archiveFile) - plugin(project.tasks.zipBwcPlugin.archiveFile) + plugin(project.tasks.unZipBwcMlCommonsPlugin.archiveFile) + plugin(project.tasks.unZipBwcKnnPlugin.archiveFile) + plugin(project.tasks.unZipBwcPlugin.archiveFile) setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' } @@ -28,7 +28,7 @@ testClusters { // Task to run BWC tests against the old cluster task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { - dependsOn "zipBwcPlugin" + dependsOn "unZipBwcPlugin" useCluster testClusters."${baseName}" systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster' systemProperty 'tests.is_old_cluster', 'true' @@ -41,7 +41,7 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { } // All nodes are upgraded to latest version and run the tests -task testRestartUpgrade(type: StandaloneRestIntegTestTask) { +task testAgainstNewCluster(type: StandaloneRestIntegTestTask) { dependsOn "testAgainstOldCluster" dependsOn rootProject.tasks.assemble useCluster testClusters."${baseName}" diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java index c1665529c..d82f523ed 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java @@ -16,13 +16,13 @@ import org.opensearch.test.rest.OpenSearchRestTestCase; public abstract class AbstractRestartUpgradeRestTestCase extends BaseNeuralSearchIT { - protected static String testIndex; + // protected static String testIndex; @Before - protected void setIndex() { + protected String getIndexNameForTest() { // Creating index name by concatenating "neural-bwc-" prefix with test method name // for all the tests in this sub-project - testIndex = NEURAL_SEARCH_BWC_PREFIX + getTestName().toLowerCase(Locale.ROOT); + return NEURAL_SEARCH_BWC_PREFIX + getTestName().toLowerCase(Locale.ROOT); } @Override diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java index fb852b6ff..d2c0b0c12 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -18,9 +18,9 @@ public class SemanticSearchIT extends AbstractRestartUpgradeRestTestCase { private static final String PIPELINE_NAME = "nlp-pipeline"; - private static String DOC_ID = "0"; private static final String TEST_FIELD = "passage_text"; private static final String TEXT = "Hello world"; + private static final String TEXT_1 = "Hello world a"; // Test restart-upgrade Semantic Search // Create Text Embedding Processor, Ingestion Pipeline and add document @@ -33,32 +33,33 @@ public void testSemanticSearch() throws Exception { loadModel(modelId); createPipelineProcessor(modelId, PIPELINE_NAME); createIndexWithConfiguration( - testIndex, + getIndexNameForTest(), Files.readString(Path.of(classLoader.getResource("processor/IndexMappings.json").toURI())), PIPELINE_NAME ); - addDocument(testIndex, DOC_ID, TEST_FIELD, TEXT); + addDocument(getIndexNameForTest(), "0", TEST_FIELD, TEXT); } else { Map pipeline = getIngestionPipeline(PIPELINE_NAME); assertNotNull(pipeline); String modelId = getModelId(pipeline, TEXT_EMBEDDING_PROCESSOR); + addDocument(getIndexNameForTest(), "1", TEST_FIELD, TEXT_1); validateTestIndex(modelId); deletePipeline(PIPELINE_NAME); deleteModel(modelId); - deleteIndex(testIndex); + deleteIndex(getIndexNameForTest()); } } private void validateTestIndex(String modelId) throws Exception { - int docCount = getDocCount(testIndex); - assertEquals(1, docCount); + int docCount = getDocCount(getIndexNameForTest()); + assertEquals(2, docCount); loadModel(modelId); NeuralQueryBuilder neuralQueryBuilder = new NeuralQueryBuilder(); neuralQueryBuilder.fieldName("passage_embedding"); neuralQueryBuilder.modelId(modelId); neuralQueryBuilder.queryText(TEXT); neuralQueryBuilder.k(1); - Map response = search(testIndex, neuralQueryBuilder, 1); + Map response = search(getIndexNameForTest(), neuralQueryBuilder, 1); assertNotNull(response); } @@ -70,13 +71,11 @@ private String uploadTextEmbeddingModel() throws Exception { private String registerModelGroupAndGetModelId(String requestBody) throws Exception { String modelGroupRegisterRequestBody = Files.readString( Path.of(classLoader.getResource("processor/CreateModelGroupRequestBody.json").toURI()) - ).replace("", "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)); - - String modelGroupId = registerModelGroup(modelGroupRegisterRequestBody); - - requestBody = requestBody.replace("", modelGroupId); - - return uploadModelId(requestBody); + ); + String modelGroupId = registerModelGroup( + String.format(LOCALE, modelGroupRegisterRequestBody, "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)) + ); + return uploadModelId(String.format(LOCALE, requestBody, modelGroupId)); } protected void createPipelineProcessor(String modelId, String pipelineName, ProcessorType processorType) throws Exception { diff --git a/qa/restart-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json b/qa/restart-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json index 51511222c..2dd0852a4 100644 --- a/qa/restart-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json +++ b/qa/restart-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json @@ -1,4 +1,4 @@ { - "name": "", + "name": "%s", "description": "This is a public model group" } diff --git a/qa/restart-upgrade/src/test/resources/processor/UploadModelRequestBody.json b/qa/restart-upgrade/src/test/resources/processor/UploadModelRequestBody.json index 95f9c9cb5..17fab7588 100644 --- a/qa/restart-upgrade/src/test/resources/processor/UploadModelRequestBody.json +++ b/qa/restart-upgrade/src/test/resources/processor/UploadModelRequestBody.json @@ -4,7 +4,7 @@ "model_format": "TORCH_SCRIPT", "model_task_type": "text_embedding", "model_content_hash_value": "e13b74006290a9d0f58c1376f9629d4ebc05a0f9385f40db837452b167ae9021", - "model_group_id": "", + "model_group_id": "%s", "model_config": { "model_type": "bert", "embedding_dimension": 768, diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index db219ad3d..51f03695c 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -17,9 +17,9 @@ testClusters { testDistribution = "ARCHIVE" versions = [neural_search_bwc_version, opensearch_version] numberOfNodes = 3 - plugin(project.tasks.zipBwcMlCommonsPlugin.archiveFile) - plugin(project.tasks.zipBwcKnnPlugin.archiveFile) - plugin(project.tasks.zipBwcPlugin.archiveFile) + plugin(project.tasks.unZipBwcMlCommonsPlugin.archiveFile) + plugin(project.tasks.unZipBwcKnnPlugin.archiveFile) + plugin(project.tasks.unZipBwcPlugin.archiveFile) jvmArgs("-Xms1g", "-Xmx4g") setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' @@ -29,7 +29,7 @@ testClusters { // Task to run BWC tests against the old cluster task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { - dependsOn "zipBwcPlugin" + dependsOn "unZipBwcPlugin" useCluster testClusters."${baseName}" systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster' systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java index 7712580ad..314b73433 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java @@ -19,13 +19,13 @@ import static org.opensearch.neuralsearch.TestUtils.NEURAL_SEARCH_BWC_PREFIX; public abstract class AbstractRollingUpgradeTestCase extends BaseNeuralSearchIT { - protected String testIndex; + // protected String testIndex; @Before - protected void setIndex() { + protected String getIndexNameForTest() { // Creating index name by concatenating "neural-bwc-" prefix with test method name // for all the tests in this sub-project - testIndex = NEURAL_SEARCH_BWC_PREFIX + getTestName().toLowerCase(Locale.ROOT); + return NEURAL_SEARCH_BWC_PREFIX + getTestName().toLowerCase(Locale.ROOT); } @Override diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java index 936c7dc11..3a9c6a9a6 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -19,7 +19,7 @@ public class SemanticSearchIT extends AbstractRollingUpgradeTestCase { private static final String TEXT = "Hello world"; private static final String TEXT_MIXED = "Hello world mixed"; private static final String TEXT_UPGRADED = "Hello world upgraded"; - private static final int NUM_DOCS = 1; + private static final int NUM_DOCS_PER_ROUND = 1; // Test rolling-upgrade Semantic Search // Create Text Embedding Processor, Ingestion Pipeline and add document @@ -32,40 +32,40 @@ public void testSemanticSearch() throws Exception { loadModel(modelId); createPipelineProcessor(modelId, PIPELINE_NAME); createIndexWithConfiguration( - testIndex, + getIndexNameForTest(), Files.readString(Path.of(classLoader.getResource("processor/IndexMappings.json").toURI())), PIPELINE_NAME ); - addDocument(testIndex, "0", TEST_FIELD, TEXT); + addDocument(getIndexNameForTest(), "0", TEST_FIELD, TEXT); break; case MIXED: modelId = getModelId(PIPELINE_NAME); int totalDocsCountMixed; if (isFirstMixedRound()) { - totalDocsCountMixed = NUM_DOCS; + totalDocsCountMixed = NUM_DOCS_PER_ROUND; validateTestIndexOnUpgrade(totalDocsCountMixed, modelId, TEXT); - addDocument(testIndex, "1", TEST_FIELD, TEXT_MIXED); + addDocument(getIndexNameForTest(), "1", TEST_FIELD, TEXT_MIXED); } else { - totalDocsCountMixed = 2 * NUM_DOCS; + totalDocsCountMixed = 2 * NUM_DOCS_PER_ROUND; validateTestIndexOnUpgrade(totalDocsCountMixed, modelId, TEXT_MIXED); } break; case UPGRADED: modelId = getModelId(PIPELINE_NAME); - int totalDocsCountUpgraded = 3 * NUM_DOCS; - addDocument(testIndex, "2", TEST_FIELD, TEXT_UPGRADED); + int totalDocsCountUpgraded = 3 * NUM_DOCS_PER_ROUND; + addDocument(getIndexNameForTest(), "2", TEST_FIELD, TEXT_UPGRADED); validateTestIndexOnUpgrade(totalDocsCountUpgraded, modelId, TEXT_UPGRADED); deletePipeline(PIPELINE_NAME); deleteModel(modelId); - deleteIndex(testIndex); + deleteIndex(getIndexNameForTest()); break; } } private void validateTestIndexOnUpgrade(int numberOfDocs, String modelId, String text) throws Exception { - int docCount = getDocCount(testIndex); + int docCount = getDocCount(getIndexNameForTest()); assertEquals(numberOfDocs, docCount); loadModel(modelId); NeuralQueryBuilder neuralQueryBuilder = new NeuralQueryBuilder(); @@ -73,7 +73,7 @@ private void validateTestIndexOnUpgrade(int numberOfDocs, String modelId, String neuralQueryBuilder.modelId(modelId); neuralQueryBuilder.queryText(text); neuralQueryBuilder.k(1); - Map response = search(testIndex, neuralQueryBuilder, 1); + Map response = search(getIndexNameForTest(), neuralQueryBuilder, 1); assertNotNull(response); } @@ -85,13 +85,11 @@ private String uploadTextEmbeddingModel() throws Exception { private String registerModelGroupAndGetModelId(String requestBody) throws Exception { String modelGroupRegisterRequestBody = Files.readString( Path.of(classLoader.getResource("processor/CreateModelGroupRequestBody.json").toURI()) - ).replace("", "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)); - - String modelGroupId = registerModelGroup(modelGroupRegisterRequestBody); - - requestBody = requestBody.replace("", modelGroupId); - - return uploadModelId(requestBody); + ); + String modelGroupId = registerModelGroup( + String.format(LOCALE, modelGroupRegisterRequestBody, "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)) + ); + return uploadModelId(String.format(LOCALE, requestBody, modelGroupId)); } protected void createPipelineProcessor(String modelId, String pipelineName, ProcessorType processorType) throws Exception { diff --git a/qa/rolling-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json b/qa/rolling-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json index 51511222c..2dd0852a4 100644 --- a/qa/rolling-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json +++ b/qa/rolling-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json @@ -1,4 +1,4 @@ { - "name": "", + "name": "%s", "description": "This is a public model group" } diff --git a/qa/rolling-upgrade/src/test/resources/processor/UploadModelRequestBody.json b/qa/rolling-upgrade/src/test/resources/processor/UploadModelRequestBody.json index 95f9c9cb5..17fab7588 100644 --- a/qa/rolling-upgrade/src/test/resources/processor/UploadModelRequestBody.json +++ b/qa/rolling-upgrade/src/test/resources/processor/UploadModelRequestBody.json @@ -4,7 +4,7 @@ "model_format": "TORCH_SCRIPT", "model_task_type": "text_embedding", "model_content_hash_value": "e13b74006290a9d0f58c1376f9629d4ebc05a0f9385f40db837452b167ae9021", - "model_group_id": "", + "model_group_id": "%s", "model_config": { "model_type": "bert", "embedding_dimension": 768, diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java index 52719239d..8580c5d64 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java @@ -839,15 +839,13 @@ protected void addDocument(String index, String docId, String fieldName, String XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field(fieldName, text).endObject(); request.setJsonEntity(builder.toString()); client().performRequest(request); - - request = new Request("POST", "/" + index + "/_refresh"); - Response response = client().performRequest(request); - assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); } /** - * Get ingest pipeline - * @param pipelineName of the ingest pipeline + * Get ingest pipeline + * @param pipelineName of the ingest pipeline + * + * @return get pipeline response as a map object */ @SneakyThrows protected Map getIngestionPipeline(String pipelineName) { @@ -863,6 +861,8 @@ protected Map getIngestionPipeline(String pipelineName) { * Delete pipeline * * @param pipelineName of the pipeline + * + * @return delete pipeline response as a map object */ @SneakyThrows protected Map deletePipeline(String pipelineName) { From cba8193fc9f7836cedeac1fc8a11ea72fdca313c Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Fri, 29 Dec 2023 14:50:40 -0800 Subject: [PATCH 34/45] Addressing Jack's comments Signed-off-by: Varun Jain --- .github/workflows/backwards_compatibility_tests_workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 9b1ca720a..699cfcec0 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -36,13 +36,13 @@ jobs: name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Windows run: | echo "Running restart-upgrade backwards compatibility tests ..." - ./gradlew :qa:restart-upgrade:testRestartUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}' + ./gradlew :qa:restart-upgrade:testAgainstNewCluster -D'tests.bwc.version=${{ matrix.bwc_version }}' - if: startsWith(matrix.os,'ubuntu') name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Ubuntu run: | echo "Running restart-upgrade backwards compatibility tests ..." - ./gradlew :qa:restart-upgrade:testRestartUpgrade -Dtests.bwc.version=$BWC_VERSION_RESTART_UPGRADE + ./gradlew :qa:restart-upgrade:testAgainstNewCluster -Dtests.bwc.version=$BWC_VERSION_RESTART_UPGRADE Rolling-Upgrade-BWCTests-NeuralSearch: From 8721079082a7531650bb7fdf5a0c359e670cae4d Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Fri, 29 Dec 2023 15:16:33 -0800 Subject: [PATCH 35/45] Fixing Test cases Signed-off-by: Varun Jain --- .../java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java index d2c0b0c12..f2b6088ee 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -42,6 +42,7 @@ public void testSemanticSearch() throws Exception { Map pipeline = getIngestionPipeline(PIPELINE_NAME); assertNotNull(pipeline); String modelId = getModelId(pipeline, TEXT_EMBEDDING_PROCESSOR); + loadModel(modelId); addDocument(getIndexNameForTest(), "1", TEST_FIELD, TEXT_1); validateTestIndex(modelId); deletePipeline(PIPELINE_NAME); From e371d583c2ebb5733f580cf80f6c7dcc21ea57c6 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Mon, 1 Jan 2024 21:26:31 -0800 Subject: [PATCH 36/45] Addressing comments of Navneet Signed-off-by: Varun Jain --- ...backwards_compatibility_tests_workflow.yml | 23 ++++--------------- CHANGELOG.md | 1 + build.gradle | 3 +-- gradle.properties | 4 ++++ gradle/formatting.gradle | 2 +- .../AbstractRestartUpgradeRestTestCase.java | 1 - .../neuralsearch/bwc/SemanticSearchIT.java | 2 +- .../resources/processor/IndexMappings.json | 4 ++-- .../bwc/AbstractRollingUpgradeTestCase.java | 1 - .../neuralsearch/bwc/SemanticSearchIT.java | 2 +- .../resources/processor/IndexMappings.json | 4 ++-- .../processor/TextEmbeddingProcessorIT.java | 2 +- .../TextImageEmbeddingProcessorIT.java | 2 +- .../CreateModelGroupRequestBody.json | 2 +- .../processor/UploadModelRequestBody.json | 2 +- .../neuralsearch/BaseNeuralSearchIT.java | 19 +++++++-------- .../neuralsearch/BaseSparseEncodingIT.java | 2 +- 17 files changed, 32 insertions(+), 44 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 699cfcec0..200f08501 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -32,18 +32,10 @@ jobs: with: java-version: ${{ matrix.java }} - - if: startsWith(matrix.os,'windows') - name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Windows + - name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-$BWC_VERSION_RESTART_UPGRADE to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}} run: | echo "Running restart-upgrade backwards compatibility tests ..." - ./gradlew :qa:restart-upgrade:testAgainstNewCluster -D'tests.bwc.version=${{ matrix.bwc_version }}' - - - if: startsWith(matrix.os,'ubuntu') - name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Ubuntu - run: | - echo "Running restart-upgrade backwards compatibility tests ..." - ./gradlew :qa:restart-upgrade:testAgainstNewCluster -Dtests.bwc.version=$BWC_VERSION_RESTART_UPGRADE - + ./gradlew :qa:restart-upgrade:testAgainstNewCluster -D'tests.bwc.version=$BWC_VERSION_RESTART_UPGRADE' Rolling-Upgrade-BWCTests-NeuralSearch: strategy: @@ -67,14 +59,7 @@ jobs: with: java-version: ${{ matrix.java }} - - if: startsWith(matrix.os,'windows') - name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Windows - run: | - echo "Running rolling-upgrade backwards compatibility tests ..." - ./gradlew :qa:rolling-upgrade:testRollingUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}' - - - if: startsWith(matrix.os,'ubuntu') - name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Ubuntu + - name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-$BWC_VERSION_ROLLING_UPGRADE to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}} run: | echo "Running rolling-upgrade backwards compatibility tests ..." - ./gradlew :qa:rolling-upgrade:testRollingUpgrade -Dtests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE + ./gradlew :qa:rolling-upgrade:testRollingUpgrade -D'tests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE' diff --git a/CHANGELOG.md b/CHANGELOG.md index f6862064f..3ae6baab7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,3 +26,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Documentation ### Maintenance ### Refactoring +- Added spotless check in the build ([#515](https://github.com/opensearch-project/neural-search/pull/515)) diff --git a/build.gradle b/build.gradle index 4b1b8e340..ac0ebaa9d 100644 --- a/build.gradle +++ b/build.gradle @@ -135,7 +135,7 @@ allprojects { project.dependencyLicenses.enabled = false project.thirdPartyAudit.enabled = false project.loggerUsageCheck.enabled = false - project.forbiddenApis.ignoreFailures = true + project.forbiddenApis.ignoreFailures = false project.forbiddenPatterns { setEnabled(false) } @@ -143,7 +143,6 @@ allprojects { project.validateNebulaPom.enabled = false project.licenseFile = rootProject.file('LICENSE.txt') project.noticeFile = rootProject.file('NOTICE.txt') - project.forbiddenApis.ignoreFailures = true } } diff --git a/gradle.properties b/gradle.properties index b4dc3433d..98663c787 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,6 +3,10 @@ # SPDX-License-Identifier: Apache-2.0 # +# The BWC version here should always be the latest opensearch version set in +# https://github.com/opensearch-project/OpenSearch/blob/main/libs/core/src/main/java/org/opensearch/Version.java . +# Wired compatibility of OpenSearch works like 3.x version is compatible with 2.(latest-major) version. +# Therefore, to run rolling-upgrade BWC Test on local machine the BWC version here should be set 2.(latest-major). systemProp.bwc.version=2.12.0-SNAPSHOT # For fixing Spotless check with Java 17 diff --git a/gradle/formatting.gradle b/gradle/formatting.gradle index 60cace641..197b848c9 100644 --- a/gradle/formatting.gradle +++ b/gradle/formatting.gradle @@ -26,7 +26,7 @@ allprojects { } format("license", { licenseHeaderFile("${rootProject.file("formatter/license-header.txt")}", "package "); - target("src/*/java/**/*.java") + target("src/*/java/**/*.java","qa/*/java/**/*.java") }) } } diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java index d82f523ed..cf985d759 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java @@ -16,7 +16,6 @@ import org.opensearch.test.rest.OpenSearchRestTestCase; public abstract class AbstractRestartUpgradeRestTestCase extends BaseNeuralSearchIT { - // protected static String testIndex; @Before protected String getIndexNameForTest() { diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java index f2b6088ee..10abdb7f5 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -76,7 +76,7 @@ private String registerModelGroupAndGetModelId(String requestBody) throws Except String modelGroupId = registerModelGroup( String.format(LOCALE, modelGroupRegisterRequestBody, "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)) ); - return uploadModelId(String.format(LOCALE, requestBody, modelGroupId)); + return uploadModel(String.format(LOCALE, requestBody, modelGroupId)); } protected void createPipelineProcessor(String modelId, String pipelineName, ProcessorType processorType) throws Exception { diff --git a/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json b/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json index cf8a5c317..fc4c13646 100644 --- a/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json +++ b/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json @@ -6,8 +6,8 @@ "refresh_interval": "30s", "default_pipeline": "%s" }, - "number_of_shards": 6, - "number_of_replicas": 0 + "number_of_shards": 3, + "number_of_replicas": 1 }, "mappings": { "properties": { diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java index 314b73433..98ce95b72 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java @@ -19,7 +19,6 @@ import static org.opensearch.neuralsearch.TestUtils.NEURAL_SEARCH_BWC_PREFIX; public abstract class AbstractRollingUpgradeTestCase extends BaseNeuralSearchIT { - // protected String testIndex; @Before protected String getIndexNameForTest() { diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java index 3a9c6a9a6..87fde97cf 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -89,7 +89,7 @@ private String registerModelGroupAndGetModelId(String requestBody) throws Except String modelGroupId = registerModelGroup( String.format(LOCALE, modelGroupRegisterRequestBody, "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)) ); - return uploadModelId(String.format(LOCALE, requestBody, modelGroupId)); + return uploadModel(String.format(LOCALE, requestBody, modelGroupId)); } protected void createPipelineProcessor(String modelId, String pipelineName, ProcessorType processorType) throws Exception { diff --git a/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json b/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json index cf8a5c317..fc4c13646 100644 --- a/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json +++ b/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json @@ -6,8 +6,8 @@ "refresh_interval": "30s", "default_pipeline": "%s" }, - "number_of_shards": 6, - "number_of_replicas": 0 + "number_of_shards": 3, + "number_of_replicas": 1 }, "mappings": { "properties": { diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java index cfdb6ebaf..97d1c857c 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java @@ -49,7 +49,7 @@ public void testTextEmbeddingProcessor() throws Exception { private String uploadTextEmbeddingModel() throws Exception { String requestBody = Files.readString(Path.of(classLoader.getResource("processor/UploadModelRequestBody.json").toURI())); - return uploadModel(requestBody); + return registerModelGroupAndUploadModel(requestBody); } private void createTextEmbeddingIndex() throws Exception { diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java index fa83ae4e0..f3cd65694 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java @@ -57,7 +57,7 @@ public void testEmbeddingProcessor_whenIngestingDocumentWithSourceWithoutMatchin private String uploadModel() throws Exception { String requestBody = Files.readString(Path.of(classLoader.getResource("processor/UploadModelRequestBody.json").toURI())); - return uploadModel(requestBody); + return registerModelGroupAndUploadModel(requestBody); } private void createTextImageEmbeddingIndex() throws Exception { diff --git a/src/test/resources/processor/CreateModelGroupRequestBody.json b/src/test/resources/processor/CreateModelGroupRequestBody.json index 51511222c..2dd0852a4 100644 --- a/src/test/resources/processor/CreateModelGroupRequestBody.json +++ b/src/test/resources/processor/CreateModelGroupRequestBody.json @@ -1,4 +1,4 @@ { - "name": "", + "name": "%s", "description": "This is a public model group" } diff --git a/src/test/resources/processor/UploadModelRequestBody.json b/src/test/resources/processor/UploadModelRequestBody.json index 95f9c9cb5..17fab7588 100644 --- a/src/test/resources/processor/UploadModelRequestBody.json +++ b/src/test/resources/processor/UploadModelRequestBody.json @@ -4,7 +4,7 @@ "model_format": "TORCH_SCRIPT", "model_task_type": "text_embedding", "model_content_hash_value": "e13b74006290a9d0f58c1376f9629d4ebc05a0f9385f40db837452b167ae9021", - "model_group_id": "", + "model_group_id": "%s", "model_config": { "model_type": "bert", "embedding_dimension": 768, diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java index 8580c5d64..680d90b65 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java @@ -134,14 +134,13 @@ protected void updateClusterSettings(String settingKey, Object value) { assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); } - protected String uploadModel(String requestBody) throws Exception { - String modelGroupId = registerModelGroup(); + protected String registerModelGroupAndUploadModel(String requestBody) throws Exception { + String modelGroupId = getModelGroupId(); // model group id is dynamically generated, we need to update model update request body after group is registered - requestBody = requestBody.replace("", modelGroupId); - return uploadModelId(requestBody); + return uploadModel(String.format(LOCALE, requestBody, modelGroupId)); } - protected String uploadModelId(String requestBody) throws Exception { + protected String uploadModel(String requestBody) throws Exception { Response uploadResponse = makeRequest( client(), "POST", @@ -204,7 +203,7 @@ protected void loadModel(String modelId) throws Exception { @SneakyThrows protected String prepareModel() { String requestBody = Files.readString(Path.of(classLoader.getResource("processor/UploadModelRequestBody.json").toURI())); - String modelId = uploadModel(requestBody); + String modelId = registerModelGroupAndUploadModel(requestBody); loadModel(modelId); return modelId; } @@ -791,11 +790,13 @@ protected String getDeployedModelId() { } @SneakyThrows - private String registerModelGroup() { + private String getModelGroupId() { String modelGroupRegisterRequestBody = Files.readString( Path.of(classLoader.getResource("processor/CreateModelGroupRequestBody.json").toURI()) - ).replace("", "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)); - return registerModelGroup(modelGroupRegisterRequestBody); + ); + return registerModelGroup( + String.format(LOCALE, modelGroupRegisterRequestBody, "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)) + ); } protected String registerModelGroup(String modelGroupRegisterRequestBody) throws IOException, ParseException { diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java index 4e547e090..3072bc767 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java @@ -34,7 +34,7 @@ protected String prepareModel() { String requestBody = Files.readString( Path.of(classLoader.getResource("processor/UploadSparseEncodingModelRequestBody.json").toURI()) ); - String modelId = uploadModel(requestBody); + String modelId = registerModelGroupAndUploadModel(requestBody); loadModel(modelId); return modelId; } From 2dcb6315e0102f94a4eb334da60b4a5036b687d0 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Mon, 1 Jan 2024 21:31:39 -0800 Subject: [PATCH 37/45] Addressing comments of Navneet Signed-off-by: Varun Jain --- .../workflows/backwards_compatibility_tests_workflow.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 200f08501..f0490155c 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -32,10 +32,10 @@ jobs: with: java-version: ${{ matrix.java }} - - name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-$BWC_VERSION_RESTART_UPGRADE to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}} + - name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}} run: | echo "Running restart-upgrade backwards compatibility tests ..." - ./gradlew :qa:restart-upgrade:testAgainstNewCluster -D'tests.bwc.version=$BWC_VERSION_RESTART_UPGRADE' + ./gradlew :qa:restart-upgrade:testAgainstNewCluster -D'tests.bwc.version=${{ matrix.bwc_version }}' Rolling-Upgrade-BWCTests-NeuralSearch: strategy: @@ -59,7 +59,7 @@ jobs: with: java-version: ${{ matrix.java }} - - name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-$BWC_VERSION_ROLLING_UPGRADE to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}} + - name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}} run: | echo "Running rolling-upgrade backwards compatibility tests ..." - ./gradlew :qa:rolling-upgrade:testRollingUpgrade -D'tests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE' + ./gradlew :qa:rolling-upgrade:testRollingUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}' From 2ace98fc031546cb046d7c84fa070e683e3727ac Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Mon, 1 Jan 2024 22:25:24 -0800 Subject: [PATCH 38/45] Addressing comments of Navneet Signed-off-by: Varun Jain --- .../processor/UploadSparseEncodingModelRequestBody.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/resources/processor/UploadSparseEncodingModelRequestBody.json b/src/test/resources/processor/UploadSparseEncodingModelRequestBody.json index 50b4b8a9b..5c2a73f6e 100644 --- a/src/test/resources/processor/UploadSparseEncodingModelRequestBody.json +++ b/src/test/resources/processor/UploadSparseEncodingModelRequestBody.json @@ -4,7 +4,7 @@ "function_name": "SPARSE_TOKENIZE", "description": "test model", "model_format": "TORCH_SCRIPT", - "model_group_id": "", + "model_group_id": "%s", "model_content_hash_value": "b345e9e943b62c405a8dd227ef2c46c84c5ff0a0b71b584be9132b37bce91a9a", "url": "https://github.com/opensearch-project/ml-commons/raw/main/ml-algorithms/src/test/resources/org/opensearch/ml/engine/algorithms/sparse_encoding/sparse_demo.zip" } From ef84f9966b3545dc6d85790570e72f2a13de53da Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Mon, 1 Jan 2024 22:40:30 -0800 Subject: [PATCH 39/45] Addressing comments of Navneet Signed-off-by: Varun Jain --- .../java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java | 2 +- .../java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java index 10abdb7f5..6d7e3026d 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -25,7 +25,7 @@ public class SemanticSearchIT extends AbstractRestartUpgradeRestTestCase { // Test restart-upgrade Semantic Search // Create Text Embedding Processor, Ingestion Pipeline and add document // Validate process , pipeline and document count in restart-upgrade scenario - public void testSemanticSearch() throws Exception { + public void testSemanticSearch_E2EFlow() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); if (isRunningAgainstOldCluster()) { diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java index 87fde97cf..3626e16f2 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -24,7 +24,7 @@ public class SemanticSearchIT extends AbstractRollingUpgradeTestCase { // Test rolling-upgrade Semantic Search // Create Text Embedding Processor, Ingestion Pipeline and add document // Validate process , pipeline and document count in rolling-upgrade scenario - public void testSemanticSearch() throws Exception { + public void testSemanticSearch_E2EFlow() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); switch (getClusterType()) { case OLD: From 1ebe6a48df14a2b7adcd6cb4b9da5388f90ea474 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 2 Jan 2024 00:29:32 -0800 Subject: [PATCH 40/45] Removing extra parameter from createPipelineProcessor Signed-off-by: Varun Jain --- .../java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java | 2 +- .../java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java index 6d7e3026d..bd53df01d 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -79,7 +79,7 @@ private String registerModelGroupAndGetModelId(String requestBody) throws Except return uploadModel(String.format(LOCALE, requestBody, modelGroupId)); } - protected void createPipelineProcessor(String modelId, String pipelineName, ProcessorType processorType) throws Exception { + protected void createPipelineProcessor(String modelId, String pipelineName) throws Exception { String requestBody = Files.readString(Path.of(classLoader.getResource("processor/PipelineConfiguration.json").toURI())); createPipelineProcessor(requestBody, pipelineName, modelId); } diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java index 3626e16f2..23f9eb368 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -92,7 +92,7 @@ private String registerModelGroupAndGetModelId(String requestBody) throws Except return uploadModel(String.format(LOCALE, requestBody, modelGroupId)); } - protected void createPipelineProcessor(String modelId, String pipelineName, ProcessorType processorType) throws Exception { + protected void createPipelineProcessor(String modelId, String pipelineName) throws Exception { String requestBody = Files.readString(Path.of(classLoader.getResource("processor/PipelineConfiguration.json").toURI())); createPipelineProcessor(requestBody, pipelineName, modelId); } From 7b3498e00436f4e633f3a7d7905366a85e2ddc4f Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 2 Jan 2024 01:28:44 -0800 Subject: [PATCH 41/45] Fixing bug Signed-off-by: Varun Jain --- .../java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java index 23f9eb368..b9c2c67da 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -15,7 +15,7 @@ public class SemanticSearchIT extends AbstractRollingUpgradeTestCase { private static final String PIPELINE_NAME = "nlp-pipeline"; - private static final String TEST_FIELD = "test-field"; + private static final String TEST_FIELD = "passage_text"; private static final String TEXT = "Hello world"; private static final String TEXT_MIXED = "Hello world mixed"; private static final String TEXT_UPGRADED = "Hello world upgraded"; From 7f3e9e464ef29ae87497b935639a73d6f93ad1f6 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 2 Jan 2024 01:42:28 -0800 Subject: [PATCH 42/45] Increasing number of shards Signed-off-by: Varun Jain --- .../src/test/resources/processor/IndexMappings.json | 2 +- .../src/test/resources/processor/IndexMappings.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json b/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json index fc4c13646..ba5ebfb40 100644 --- a/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json +++ b/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json @@ -6,7 +6,7 @@ "refresh_interval": "30s", "default_pipeline": "%s" }, - "number_of_shards": 3, + "number_of_shards": 6, "number_of_replicas": 1 }, "mappings": { diff --git a/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json b/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json index fc4c13646..ba5ebfb40 100644 --- a/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json +++ b/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json @@ -6,7 +6,7 @@ "refresh_interval": "30s", "default_pipeline": "%s" }, - "number_of_shards": 3, + "number_of_shards": 6, "number_of_replicas": 1 }, "mappings": { From 136bb81f5045debf7fcf9f06f50fae2c91a84038 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 2 Jan 2024 02:11:24 -0800 Subject: [PATCH 43/45] Bug fix of load model id Signed-off-by: Varun Jain --- .../java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java index b9c2c67da..00ff46733 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -54,6 +54,7 @@ public void testSemanticSearch_E2EFlow() throws Exception { case UPGRADED: modelId = getModelId(PIPELINE_NAME); int totalDocsCountUpgraded = 3 * NUM_DOCS_PER_ROUND; + loadModel(modelId); addDocument(getIndexNameForTest(), "2", TEST_FIELD, TEXT_UPGRADED); validateTestIndexOnUpgrade(totalDocsCountUpgraded, modelId, TEXT_UPGRADED); deletePipeline(PIPELINE_NAME); From 96cc6bb415085d59dad1fcba7eda5c366da8f1c0 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 2 Jan 2024 12:41:53 -0800 Subject: [PATCH 44/45] Changing names of tests to TextEmbeddingPRocessor Signed-off-by: Varun Jain --- .../java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java | 2 +- .../java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java index bd53df01d..dc8a94236 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -25,7 +25,7 @@ public class SemanticSearchIT extends AbstractRestartUpgradeRestTestCase { // Test restart-upgrade Semantic Search // Create Text Embedding Processor, Ingestion Pipeline and add document // Validate process , pipeline and document count in restart-upgrade scenario - public void testSemanticSearch_E2EFlow() throws Exception { + public void testTextEmbeddingProcessor_E2EFlow() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); if (isRunningAgainstOldCluster()) { diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java index 00ff46733..41d21aa3f 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -24,7 +24,7 @@ public class SemanticSearchIT extends AbstractRollingUpgradeTestCase { // Test rolling-upgrade Semantic Search // Create Text Embedding Processor, Ingestion Pipeline and add document // Validate process , pipeline and document count in rolling-upgrade scenario - public void testSemanticSearch_E2EFlow() throws Exception { + public void testTextEmbeddingProcessor_E2EFlow() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); switch (getClusterType()) { case OLD: From faa8dec95c0fac784f59f6a32f494e775202f4e0 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Tue, 2 Jan 2024 12:51:50 -0800 Subject: [PATCH 45/45] Updating indexes and replicas Signed-off-by: Varun Jain --- .../src/test/resources/processor/IndexMappings.json | 2 +- .../src/test/resources/processor/IndexMappings.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json b/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json index ba5ebfb40..fc4c13646 100644 --- a/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json +++ b/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json @@ -6,7 +6,7 @@ "refresh_interval": "30s", "default_pipeline": "%s" }, - "number_of_shards": 6, + "number_of_shards": 3, "number_of_replicas": 1 }, "mappings": { diff --git a/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json b/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json index ba5ebfb40..fc4c13646 100644 --- a/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json +++ b/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json @@ -6,7 +6,7 @@ "refresh_interval": "30s", "default_pipeline": "%s" }, - "number_of_shards": 6, + "number_of_shards": 3, "number_of_replicas": 1 }, "mappings": {