Merge branch 'main' into formatting-2

.github/workflows/CI.yml

@@ -21,7 +21,8 @@ jobs:
     needs: Get-CI-Image-Tag
     strategy:
       matrix:
-        java: [11, 17, 21]
+        # Restricting java 21 to 21.0.1 due to ongoing bug in JDK 21.0.2 https://bugs.openjdk.org/browse/JDK-8323659. Once the fix https://github.com/opensearch-project/OpenSearch/pull/11968 get merged this change will be reverted.
+        java: [11, 17, 21.0.1]
         os: [ubuntu-latest]
 
     name: Gradle Check Linux
@@ -33,6 +34,7 @@ jobs:
       # need to switch to root so that github actions can install runner binary on container without permission issues.
       options: --user root
 
+
     steps:
       - name: Checkout neural-search
         uses: actions/checkout@v1
@@ -55,7 +57,8 @@ jobs:
   Check-neural-search-windows:
     strategy:
       matrix:
-        java: [11, 17, 21]
+        # Restricting java 21 to 21.0.1 due to ongoing bug in JDK 21.0.2 https://bugs.openjdk.org/browse/JDK-8323659. Once the fix https://github.com/opensearch-project/OpenSearch/pull/11968 get merged this change will be reverted.
+        java: [11, 17, 21.0.1]
         os: [windows-latest]
 
     name: Gradle Check Windows
@@ -83,7 +86,8 @@ jobs:
     needs: Get-CI-Image-Tag
     strategy:
       matrix:
-        java: [11, 17, 21]
+        # Restricting java 21 to 21.0.1 due to ongoing bug in JDK 21.0.2 https://bugs.openjdk.org/browse/JDK-8323659. Once the fix https://github.com/opensearch-project/OpenSearch/pull/11968 get merged this change will be reverted.
+        java: [11, 17, 21.0.1]
         os: [ubuntu-latest]
 
     name: Pre-commit Linux
@@ -117,7 +121,8 @@ jobs:
   Precommit-neural-search-windows:
     strategy:
       matrix:
-        java: [11, 17, 21]
+        # Restricting java 21 to 21.0.1 due to ongoing bug in JDK 21.0.2 https://bugs.openjdk.org/browse/JDK-8323659. Once the fix https://github.com/opensearch-project/OpenSearch/pull/11968 get merged this change will be reverted.
+        java: [11, 17, 21.0.1]
         os: [windows-latest]
 
     name: Pre-commit Windows

.github/workflows/backwards_compatibility_tests_workflow.yml

@@ -13,7 +13,8 @@ jobs:
   Restart-Upgrade-BWCTests-NeuralSearch:
     strategy:
       matrix:
-        java: [ 11, 17, 21 ]
+        # Restricting java 21 to 21.0.1 due to ongoing bug in JDK 21.0.2 https://bugs.openjdk.org/browse/JDK-8323659. Once the fix https://github.com/opensearch-project/OpenSearch/pull/11968 get merged this change will be reverted.
+        java: [ 11, 17, 21.0.1 ]
         os: [ubuntu-latest,windows-latest]
         bwc_version : ["2.9.0","2.10.0","2.11.0","2.12.0-SNAPSHOT"]
         opensearch_version : [ "3.0.0-SNAPSHOT" ]
@@ -35,7 +36,8 @@ jobs:
       - name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}}
         run: |
           echo "Running restart-upgrade backwards compatibility tests ..."
-          ./gradlew :qa:restart-upgrade:testAgainstNewCluster -D'tests.bwc.version=${{ matrix.bwc_version }}'
+#          Disabling BWC tests due to ongoing build failure. https://github.com/opensearch-project/neural-search/issues/536
+#          ./gradlew :qa:restart-upgrade:testAgainstNewCluster -D'tests.bwc.version=${{ matrix.bwc_version }}'
 
   Rolling-Upgrade-BWCTests-NeuralSearch:
     strategy:
@@ -62,4 +64,5 @@ jobs:
       - name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}}
         run: |
           echo "Running rolling-upgrade backwards compatibility tests ..."
-          ./gradlew :qa:rolling-upgrade:testRollingUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}'
+#          Disabling BWC tests due to ongoing build failure. https://github.com/opensearch-project/neural-search/issues/536
+#         ./gradlew :qa:rolling-upgrade:testRollingUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}'

.github/workflows/test_security.yml

@@ -20,7 +20,8 @@ jobs:
   integ-test-with-security-linux:
     strategy:
       matrix:
-        java: [11, 17, 21]
+        # Restricting java 21 to 21.0.1 due to ongoing bug in JDK 21.0.2 https://bugs.openjdk.org/browse/JDK-8323659. Once the fix https://github.com/opensearch-project/OpenSearch/pull/11968 get merged this change will be reverted.
+        java: [11, 17, 21.0.1]
 
     name: Run Integration Tests on Linux
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Bug Fixes
 - Fixing multiple issues reported in #497 ([#524](https://github.com/opensearch-project/neural-search/pull/524))
 - Fix Flaky test reported in #433 ([#533](https://github.com/opensearch-project/neural-search/pull/533))
+- Enable support for default model id on HybridQueryBuilder ([#541](https://github.com/opensearch-project/neural-search/pull/541))
 ### Infrastructure
 - BWC tests for Neural Search ([#515](https://github.com/opensearch-project/neural-search/pull/515))
 - Github action to run integ tests in secure opensearch cluster ([#535](https://github.com/opensearch-project/neural-search/pull/535))

build.gradle

@@ -66,7 +66,7 @@ ext {
     projectSubstitutions = [:]
 
     configureSecurityPlugin = { OpenSearchCluster cluster ->
-        configurations.zipArchive.asFileTree.each {
+        configurations.secureIntegTestPluginArchive.asFileTree.each {
             if(it.name.contains("opensearch-security")){
                 cluster.plugin(provider(new Callable<RegularFile>() {
                     @Override
@@ -160,6 +160,7 @@ allprojects {
 
 configurations {
     zipArchive
+    secureIntegTestPluginArchive
 }
 
 tasks.register("preparePluginPathDirs") {
@@ -231,7 +232,7 @@ dependencies {
     api "org.opensearch:opensearch:${opensearch_version}"
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-knn', version: "${opensearch_build}"
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}"
-    zipArchive group: 'org.opensearch.plugin', name:'opensearch-security', version: "${opensearch_build}"
+    secureIntegTestPluginArchive group: 'org.opensearch.plugin', name:'opensearch-security', version: "${opensearch_build}"
     compileOnly fileTree(dir: knnJarDirectory, include: '*.jar')
     api group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}"
     testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}"
@@ -337,8 +338,7 @@ testClusters.integTest {
 
     // Install K-NN/ml-commons plugins on the integTest cluster nodes except security
     configurations.zipArchive.asFileTree.each {
-        if(!it.name.contains("opensearch-security")) {
-            plugin(provider(new Callable<RegularFile>(){
+        plugin(provider(new Callable<RegularFile>(){
                 @Override
                 RegularFile call() throws Exception {
                     return new RegularFile() {
@@ -348,8 +348,7 @@ testClusters.integTest {
                         }
                     }
                 }
-            }))
-        }
+        }))
     }
 
     // This installs our neural-search plugin into the testClusters

src/main/java/org/opensearch/neuralsearch/query/HybridQueryBuilder.java

@@ -13,6 +13,7 @@
 import java.util.stream.Collectors;
 
 import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.Query;
 import org.opensearch.common.lucene.search.Queries;
 import org.opensearch.core.ParseField;
@@ -27,6 +28,7 @@
 import org.opensearch.index.query.QueryShardContext;
 import org.opensearch.index.query.QueryShardException;
 import org.opensearch.index.query.Rewriteable;
+import org.opensearch.index.query.QueryBuilderVisitor;
 
 import lombok.Getter;
 import lombok.NoArgsConstructor;
@@ -295,4 +297,18 @@ private Collection<Query> toQueries(Collection<QueryBuilder> queryBuilders, Quer
         }).filter(Objects::nonNull).collect(Collectors.toList());
         return queries;
     }
+
+    /**
+     * visit method to parse the HybridQueryBuilder by a visitor
+     */
+    @Override
+    public void visit(QueryBuilderVisitor visitor) {
+        visitor.accept(this);
+        // getChildVisitor of NeuralSearchQueryVisitor return this.
+        // therefore any argument can be passed. Here we have used Occcur.MUST as an argument.
+        QueryBuilderVisitor subVisitor = visitor.getChildVisitor(Occur.MUST);
+        for (QueryBuilder subQueryBuilder : queries) {
+            subQueryBuilder.visit(subVisitor);
+        }
+    }
 }

src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java

@@ -15,6 +15,7 @@
 import org.junit.Before;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.neuralsearch.BaseNeuralSearchIT;
+import org.opensearch.neuralsearch.query.HybridQueryBuilder;
 import org.opensearch.neuralsearch.query.NeuralQueryBuilder;
 
 import com.google.common.primitives.Floats;
@@ -62,6 +63,25 @@ public void testNeuralQueryEnricherProcessor_whenNoModelIdPassed_thenSuccess() {
 
     }
 
+    @SneakyThrows
+    public void testNeuralQueryEnricherProcessor_whenHybridQueryBuilderAndNoModelIdPassed_thenSuccess() {
+        initializeIndexIfNotExist();
+        String modelId = getDeployedModelId();
+        createSearchRequestProcessor(modelId, search_pipeline);
+        createPipelineProcessor(modelId, ingest_pipeline);
+        updateIndexSettings(index, Settings.builder().put("index.search.default_pipeline", search_pipeline));
+        NeuralQueryBuilder neuralQueryBuilder = new NeuralQueryBuilder();
+        neuralQueryBuilder.fieldName(TEST_KNN_VECTOR_FIELD_NAME_1);
+        neuralQueryBuilder.queryText("Hello World");
+        neuralQueryBuilder.k(1);
+        HybridQueryBuilder hybridQueryBuilder = new HybridQueryBuilder();
+        hybridQueryBuilder.add(neuralQueryBuilder);
+        Map<String, Object> response = search(index, hybridQueryBuilder, 2);
+
+        assertFalse(response.isEmpty());
+
+    }
+
     @SneakyThrows
     private void initializeIndexIfNotExist() {
         if (index.equals(NeuralQueryEnricherProcessorIT.index) && !indexExists(index)) {

src/test/java/org/opensearch/neuralsearch/query/HybridQueryBuilderTests.java

@@ -20,6 +20,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.ArrayList;
 import java.util.function.Supplier;
 
 import org.apache.lucene.search.MatchNoDocsQuery;
@@ -708,6 +709,13 @@ public void testBoost_whenDefaultBoostSet_thenBuildSuccessfully() {
         assertNotNull(hybridQueryBuilder);
     }
 
+    public void testVisit() {
+        HybridQueryBuilder hybridQueryBuilder = new HybridQueryBuilder().add(new NeuralQueryBuilder()).add(new NeuralSparseQueryBuilder());
+        List<QueryBuilder> visitedQueries = new ArrayList<>();
+        hybridQueryBuilder.visit(createTestVisitor(visitedQueries));
+        assertEquals(3, visitedQueries.size());
+    }
+
     private Map<String, Object> getInnerMap(Object innerObject, String queryName, String fieldName) {
         if (!(innerObject instanceof Map)) {
             fail("field name does not map to nested object");

src/test/java/org/opensearch/neuralsearch/query/OpenSearchQueryTestCase.java

@@ -7,6 +7,14 @@
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonMap;
 import static java.util.stream.Collectors.toList;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.search.BooleanClause;
+import org.opensearch.index.query.QueryBuilder;
+import org.opensearch.index.query.QueryBuilderVisitor;
 import static org.opensearch.neuralsearch.settings.NeuralSearchSettings.NEURAL_SEARCH_HYBRID_SEARCH_DISABLED;
 
 import java.io.IOException;
@@ -20,11 +28,6 @@
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Weight;
 import org.opensearch.Version;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.CheckedConsumer;
@@ -230,4 +233,18 @@ public float getMaxScore(int upTo) {
     protected static void initFeatureFlags() {
         System.setProperty(NEURAL_SEARCH_HYBRID_SEARCH_DISABLED.getKey(), "true");
     }
+
+    protected static QueryBuilderVisitor createTestVisitor(List<QueryBuilder> visitedQueries) {
+        return new QueryBuilderVisitor() {
+            @Override
+            public void accept(QueryBuilder qb) {
+                visitedQueries.add(qb);
+            }
+
+            @Override
+            public QueryBuilderVisitor getChildVisitor(BooleanClause.Occur occur) {
+                return this;
+            }
+        };
+    }
 }