From 87839ccc1bf267e7f79e84aefa0ec64485a1da23 Mon Sep 17 00:00:00 2001 From: Sahil Buddharaju Date: Fri, 6 Dec 2024 14:10:34 -0800 Subject: [PATCH 1/8] Added backward compatibility tests for Lucene engine Signed-off-by: Sahil Buddharaju --- qa/restart-upgrade/build.gradle | 19 +++- .../org/opensearch/knn/bwc/IndexingIT.java | 97 ++++++++++++++++--- .../org/opensearch/knn/bwc/IndexingIT.java | 54 ++++++++++- 3 files changed, 152 insertions(+), 18 deletions(-) diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index 628b62352..a4a3db40f 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -86,9 +86,18 @@ testClusters { knn_bwc_version.startsWith("2.14.") || knn_bwc_version.startsWith("2.15.")) { filter { + excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneQuantization" excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexBinaryForceMerge" + excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneByteVector" } } + if (knn_bwc_version.startsWith("2.16.") || + knn_bwc_version.startsWith("2.17.")) { + filter { + excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneQuantization" + } + } + if (versionsBelow2_3.any {knn_bwc_version.startsWith(it) }) { filter { @@ -152,14 +161,22 @@ testClusters { knn_bwc_version.startsWith("2.7.") || knn_bwc_version.startsWith("2.8.") || knn_bwc_version.startsWith("2.9.") || - knn_bwc_version.startsWith("2.10.") || + knn_bwc_version.startsWith("2ls.10.") || knn_bwc_version.startsWith("2.11.") || knn_bwc_version.startsWith("2.12.") || knn_bwc_version.startsWith("2.13.") || knn_bwc_version.startsWith("2.14.") || knn_bwc_version.startsWith("2.15.")) { filter { + excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneQuantization" excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexBinaryForceMerge" + excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneByteVector" + } + } + if (knn_bwc_version.startsWith("2.16.") || + knn_bwc_version.startsWith("2.17.")) { + filter { + excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneQuantization" } } diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index b212d844f..1f3707f79 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -5,14 +5,18 @@ package org.opensearch.knn.bwc; +import com.google.common.collect.ImmutableMap; import org.junit.Assert; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import java.util.Arrays; import java.util.List; import java.util.Map; @@ -22,18 +26,7 @@ import static org.opensearch.knn.TestUtils.NODES_BWC_CLUSTER; import static org.opensearch.knn.TestUtils.PROPERTIES; import static org.opensearch.knn.TestUtils.VECTOR_TYPE; -import static org.opensearch.knn.common.KNNConstants.DIMENSION; -import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; -import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; -import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; -import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; -import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; -import static org.opensearch.knn.common.KNNConstants.NAME; -import static org.opensearch.knn.common.KNNConstants.PARAMETERS; +import static org.opensearch.knn.common.KNNConstants.*; public class IndexingIT extends AbstractRestartUpgradeTestCase { private static final String TEST_FIELD = "test-field"; @@ -126,6 +119,86 @@ public void testKNNIndexLuceneForceMerge() throws Exception { } } + // 2.17 and up + public void testKNNIndexLuceneByteVector() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + + if (isRunningAgainstOldCluster()) { + createKnnIndex( + testIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping( + TEST_FIELD, + DIMENSIONS, + METHOD_HNSW, + LUCENE_NAME, + SpaceType.L2.getValue(), + true, + VectorDataType.BYTE + ) + ); + addKNNByteDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, 50); + // Flush to ensure that index is not re-indexed when node comes back up + flush(testIndex, true); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, 50, 5); + } else { + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, 50, 5); + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, 50, 25); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, 75, 5); + deleteKNNIndex(testIndex); + } + } + + // 2.16 and up + public void testKNNIndexLuceneQuantization() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + + if (isRunningAgainstOldCluster()) { + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(TEST_FIELD) + .field(KNNConstants.TYPE, KNNConstants.TYPE_KNN_VECTOR) + .field(KNNConstants.DIMENSION, DIMENSIONS) + .field(VECTOR_DATA_TYPE_FIELD, VectorDataType.FLOAT) + .field("doc_values", true) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, METHOD_HNSW) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) + .field(KNNConstants.KNN_ENGINE, LUCENE_NAME) + + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, M) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, EF_CONSTRUCTION) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(LUCENE_SQ_BITS, 7) + .field(LUCENE_SQ_CONFIDENCE_INTERVAL, 1.0) + .endObject() + .endObject() + .endObject() + + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), mapping); + + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, 100); + // Flush to ensure that index is not re-indexed when node comes back up + flush(testIndex, true); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, 100, 5); + } else { + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, 100, 5); + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, 100, 50); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, 150, 5); + deleteKNNIndex(testIndex); + } + + } + // Ensure bwc works for binary force merge public void testKNNIndexBinaryForceMerge() throws Exception { int dimension = 40; diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index 10df1a79b..3cfa00c72 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -15,7 +15,9 @@ public class IndexingIT extends AbstractRollingUpgradeTestCase { private static final String ALGO = "hnsw"; - private static final String ENGINE = "faiss"; + private static final String FAISS_NAME = "faiss"; + private static final String LUCENE_NAME = "lucene"; + public void testKNNDefaultIndexSettings() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); @@ -91,7 +93,7 @@ public void testKNNIndexCreation_withMethodMapper() throws Exception { final String upgradedIndex = testIndex + "upgraded"; switch (getClusterType()) { case OLD: - createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, ENGINE)); + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, FAISS_NAME)); int docIdOld = 0; addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); break; @@ -101,7 +103,7 @@ public void testKNNIndexCreation_withMethodMapper() throws Exception { createKnnIndex( firstMixRoundIndex, getKNNDefaultIndexSettings(), - createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, ENGINE) + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, FAISS_NAME) ); addKNNDocs(firstMixRoundIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); } else { @@ -109,14 +111,56 @@ public void testKNNIndexCreation_withMethodMapper() throws Exception { createKnnIndex( otherMixRoundIndex, getKNNDefaultIndexSettings(), - createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, ENGINE) + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, FAISS_NAME) + ); + addKNNDocs(otherMixRoundIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); + } + break; + case UPGRADED: + docIdOld = 0; + createKnnIndex(upgradedIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, FAISS_NAME)); + addKNNDocs(upgradedIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); + + deleteKNNIndex(testIndex); + deleteKNNIndex(firstMixRoundIndex); + deleteKNNIndex(otherMixRoundIndex); + deleteKNNIndex(upgradedIndex); + } + } + + public void testKNNLuceneIndexCreation_withMethodMapper() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + final String firstMixRoundIndex = testIndex + "first-mix-round"; + final String otherMixRoundIndex = testIndex + "other-mix-round"; + final String upgradedIndex = testIndex + "upgraded"; + switch (getClusterType()) { + case OLD: + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, LUCENE_NAME)); + int docIdOld = 0; + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); + break; + case MIXED: + if (isFirstMixedRound()) { + docIdOld = 0; + createKnnIndex( + firstMixRoundIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, LUCENE_NAME) + ); + addKNNDocs(firstMixRoundIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); + } else { + docIdOld = 0; + createKnnIndex( + otherMixRoundIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, LUCENE_NAME) ); addKNNDocs(otherMixRoundIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); } break; case UPGRADED: docIdOld = 0; - createKnnIndex(upgradedIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, ENGINE)); + createKnnIndex(upgradedIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, LUCENE_NAME)); addKNNDocs(upgradedIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); deleteKNNIndex(testIndex); From 715255ae917f7a192363e1c73b3224fe4448f20a Mon Sep 17 00:00:00 2001 From: Sahil Buddharaju Date: Fri, 6 Dec 2024 14:34:49 -0800 Subject: [PATCH 2/8] Fixed Typo error Signed-off-by: Sahil Buddharaju --- qa/restart-upgrade/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index a4a3db40f..db30fb5b4 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -161,7 +161,7 @@ testClusters { knn_bwc_version.startsWith("2.7.") || knn_bwc_version.startsWith("2.8.") || knn_bwc_version.startsWith("2.9.") || - knn_bwc_version.startsWith("2ls.10.") || + knn_bwc_version.startsWith("2.10.") || knn_bwc_version.startsWith("2.11.") || knn_bwc_version.startsWith("2.12.") || knn_bwc_version.startsWith("2.13.") || From dd31356cee4fd02a93f8eeb378e39b81c99dc816 Mon Sep 17 00:00:00 2001 From: Sahil Buddharaju Date: Fri, 6 Dec 2024 14:59:54 -0800 Subject: [PATCH 3/8] Fixed autoformat imports Signed-off-by: Sahil Buddharaju --- .../org/opensearch/knn/bwc/IndexingIT.java | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index 1f3707f79..e3e8c29ac 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -5,18 +5,15 @@ package org.opensearch.knn.bwc; -import com.google.common.collect.ImmutableMap; import org.junit.Assert; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; -import java.util.Arrays; import java.util.List; import java.util.Map; @@ -26,7 +23,23 @@ import static org.opensearch.knn.TestUtils.NODES_BWC_CLUSTER; import static org.opensearch.knn.TestUtils.PROPERTIES; import static org.opensearch.knn.TestUtils.VECTOR_TYPE; -import static org.opensearch.knn.common.KNNConstants.*; +import static org.opensearch.knn.common.KNNConstants.DIMENSION; +import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; +import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_BITS; +import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_CONFIDENCE_INTERVAL; +import static org.opensearch.knn.common.KNNConstants.NAME; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; public class IndexingIT extends AbstractRestartUpgradeTestCase { private static final String TEST_FIELD = "test-field"; From 4bb15ed91f2912365f39b65a2c5b7814533561f3 Mon Sep 17 00:00:00 2001 From: Sahil Buddharaju Date: Fri, 6 Dec 2024 15:27:14 -0800 Subject: [PATCH 4/8] Missed import Signed-off-by: Sahil Buddharaju --- .../src/test/java/org/opensearch/knn/bwc/IndexingIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index e3e8c29ac..e78ddc3e1 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -8,7 +8,6 @@ import org.junit.Assert; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; From cf5ea140296d5bf1a081f90c0267c30415e8bc75 Mon Sep 17 00:00:00 2001 From: Sahil Buddharaju Date: Fri, 6 Dec 2024 15:55:25 -0800 Subject: [PATCH 5/8] Formatting Signed-off-by: Sahil Buddharaju --- .../org/opensearch/knn/bwc/IndexingIT.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index e78ddc3e1..745af58f6 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -170,18 +170,18 @@ public void testKNNIndexLuceneQuantization() throws Exception { .startObject() .startObject("properties") .startObject(TEST_FIELD) - .field(KNNConstants.TYPE, KNNConstants.TYPE_KNN_VECTOR) - .field(KNNConstants.DIMENSION, DIMENSIONS) + .field(VECTOR_TYPE, KNN_VECTOR) + .field(DIMENSION, DIMENSIONS) .field(VECTOR_DATA_TYPE_FIELD, VectorDataType.FLOAT) .field("doc_values", true) - .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, METHOD_HNSW) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) - .field(KNNConstants.KNN_ENGINE, LUCENE_NAME) - - .startObject(KNNConstants.PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_M, M) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, EF_CONSTRUCTION) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) + .field(KNN_ENGINE, LUCENE_NAME) + + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_M, M) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, EF_CONSTRUCTION) .startObject(METHOD_ENCODER_PARAMETER) .field(NAME, ENCODER_SQ) .startObject(PARAMETERS) From 610a041fa6c399e442add4b24f16dc09a99a12bc Mon Sep 17 00:00:00 2001 From: Sahil Buddharaju Date: Mon, 9 Dec 2024 09:01:58 -0800 Subject: [PATCH 6/8] Changelog Signed-off-by: Sahil Buddharaju --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03a8d7974..0364fd1d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,4 +27,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Maintenance * Select index settings based on cluster version[2236](https://github.com/opensearch-project/k-NN/pull/2236) * Added null checks for fieldInfo in ExactSearcher to avoid NPE while running exact search for segments with no vector field (#2278)[https://github.com/opensearch-project/k-NN/pull/2278] +* Added Lucene BWC tests (#2313)[https://github.com/opensearch-project/k-NN/pull/2313] ### Refactoring From e820d2cc5e8e596127b0deed4a903ec3848fcc65 Mon Sep 17 00:00:00 2001 From: Sahil Buddharaju Date: Thu, 12 Dec 2024 12:00:31 -0800 Subject: [PATCH 7/8] Changed tests to allow proper versioning Signed-off-by: Sahil Buddharaju --- qa/restart-upgrade/build.gradle | 44 ++++--- .../org/opensearch/knn/bwc/IndexingIT.java | 112 ++++++++++-------- .../org/opensearch/knn/bwc/IndexingIT.java | 14 +-- 3 files changed, 98 insertions(+), 72 deletions(-) diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index db30fb5b4..3868297bb 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -67,7 +67,21 @@ testClusters { excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testEmptyParametersOnUpgrade" } } - + + if (knn_bwc_version.startsWith("1.") || + knn_bwc_version.startsWith("2.0.") || + knn_bwc_version.startsWith("2.1.") || + knn_bwc_version.startsWith("2.2.") || + knn_bwc_version.startsWith("2.3.") || + knn_bwc_version.startsWith("2.4") || + knn_bwc_version.startsWith("2.5.") || + knn_bwc_version.startsWith("2.6.") || + knn_bwc_version.startsWith("2.7.") || + knn_bwc_version.startsWith("2.8.")) { + filter { + excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneByteVector" + } + } if (knn_bwc_version.startsWith("1.") || knn_bwc_version.startsWith("2.0.") || knn_bwc_version.startsWith("2.1.") || @@ -88,13 +102,6 @@ testClusters { filter { excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneQuantization" excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexBinaryForceMerge" - excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneByteVector" - } - } - if (knn_bwc_version.startsWith("2.16.") || - knn_bwc_version.startsWith("2.17.")) { - filter { - excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneQuantization" } } @@ -150,6 +157,20 @@ testClusters { } } + if (knn_bwc_version.startsWith("1.") || + knn_bwc_version.startsWith("2.0.") || + knn_bwc_version.startsWith("2.1.") || + knn_bwc_version.startsWith("2.2.") || + knn_bwc_version.startsWith("2.3.") || + knn_bwc_version.startsWith("2.4") || + knn_bwc_version.startsWith("2.5.") || + knn_bwc_version.startsWith("2.6.") || + knn_bwc_version.startsWith("2.7.") || + knn_bwc_version.startsWith("2.8.")) { + filter { + excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneByteVector" + } + } if (knn_bwc_version.startsWith("1.") || knn_bwc_version.startsWith("2.0.") || knn_bwc_version.startsWith("2.1.") || @@ -170,13 +191,6 @@ testClusters { filter { excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneQuantization" excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexBinaryForceMerge" - excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneByteVector" - } - } - if (knn_bwc_version.startsWith("2.16.") || - knn_bwc_version.startsWith("2.17.")) { - filter { - excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneQuantization" } } diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index 745af58f6..9f84e4f32 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -5,13 +5,17 @@ package org.opensearch.knn.bwc; +import org.apache.hc.core5.http.io.entity.EntityUtils; import org.junit.Assert; +import org.opensearch.client.Response; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.knn.KNNResult; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.query.KNNQueryBuilder; import java.util.List; import java.util.Map; @@ -131,23 +135,22 @@ public void testKNNIndexLuceneForceMerge() throws Exception { } } - // 2.17 and up public void testKNNIndexLuceneByteVector() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); if (isRunningAgainstOldCluster()) { createKnnIndex( - testIndex, - getKNNDefaultIndexSettings(), - createKnnIndexMapping( - TEST_FIELD, - DIMENSIONS, - METHOD_HNSW, - LUCENE_NAME, - SpaceType.L2.getValue(), - true, - VectorDataType.BYTE - ) + testIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping( + TEST_FIELD, + DIMENSIONS, + METHOD_HNSW, + LUCENE_NAME, + SpaceType.L2.getValue(), + true, + VectorDataType.BYTE + ) ); addKNNByteDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, 50); // Flush to ensure that index is not re-indexed when node comes back up @@ -161,54 +164,63 @@ public void testKNNIndexLuceneByteVector() throws Exception { } } - // 2.16 and up + public void testKNNIndexLuceneQuantization() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); + int k = 4; + int dimension = 2; if (isRunningAgainstOldCluster()) { String mapping = XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(TEST_FIELD) - .field(VECTOR_TYPE, KNN_VECTOR) - .field(DIMENSION, DIMENSIONS) - .field(VECTOR_DATA_TYPE_FIELD, VectorDataType.FLOAT) - .field("doc_values", true) - .startObject(KNN_METHOD) - .field(NAME, METHOD_HNSW) - .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) - .field(KNN_ENGINE, LUCENE_NAME) - - .startObject(PARAMETERS) - .field(METHOD_PARAMETER_M, M) - .field(METHOD_PARAMETER_EF_CONSTRUCTION, EF_CONSTRUCTION) - .startObject(METHOD_ENCODER_PARAMETER) - .field(NAME, ENCODER_SQ) - .startObject(PARAMETERS) - .field(LUCENE_SQ_BITS, 7) - .field(LUCENE_SQ_CONFIDENCE_INTERVAL, 1.0) - .endObject() - .endObject() - .endObject() - - .endObject() - .endObject() - .endObject() - .endObject() - .toString(); + .startObject() + .startObject("properties") + .startObject(TEST_FIELD) + .field(VECTOR_TYPE, KNN_VECTOR) + .field(DIMENSION, dimension) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) + .field(KNN_ENGINE, LUCENE_NAME) + .startObject(PARAMETERS) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .endObject() + .field(METHOD_PARAMETER_EF_CONSTRUCTION, 256) + .field(METHOD_PARAMETER_M, 16) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); createKnnIndex(testIndex, getKNNDefaultIndexSettings(), mapping); - addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, 100); - // Flush to ensure that index is not re-indexed when node comes back up - flush(testIndex, true); - validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, 100, 5); + Float[] vector1 = { -10.6f, 25.48f }; + Float[] vector2 = { -10.8f, 25.48f }; + Float[] vector3 = { -11.0f, 25.48f }; + Float[] vector4 = { -11.2f, 25.48f }; + addKnnDoc(testIndex, "1", TEST_FIELD, vector1); + addKnnDoc(testIndex, "2", TEST_FIELD, vector2); + addKnnDoc(testIndex, "3", TEST_FIELD, vector3); + addKnnDoc(testIndex, "4", TEST_FIELD, vector4); + + float[] queryVector = { -10.5f, 25.48f }; + Response searchResponse = searchKNNIndex(testIndex, new KNNQueryBuilder(TEST_FIELD, queryVector, k), k); + List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), TEST_FIELD); + assertEquals(k, results.size()); + for (int i = 0; i < k; i++) { + assertEquals(k - i, Integer.parseInt(results.get(i).getDocId())); + } } else { - validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, 100, 5); - addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, 100, 50); - validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, 150, 5); + float[] queryVector = { -10.5f, 25.48f }; + Response searchResponse = searchKNNIndex(testIndex, new KNNQueryBuilder(TEST_FIELD, queryVector, k), k); + List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), TEST_FIELD); + assertEquals(k, results.size()); + for (int i = 0; i < k; i++) { + assertEquals(k - i, Integer.parseInt(results.get(i).getDocId())); + } deleteKNNIndex(testIndex); } - } // Ensure bwc works for binary force merge diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index 3cfa00c72..bd46886ab 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -101,17 +101,17 @@ public void testKNNIndexCreation_withMethodMapper() throws Exception { if (isFirstMixedRound()) { docIdOld = 0; createKnnIndex( - firstMixRoundIndex, - getKNNDefaultIndexSettings(), - createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, FAISS_NAME) + firstMixRoundIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, FAISS_NAME) ); addKNNDocs(firstMixRoundIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); } else { docIdOld = 0; createKnnIndex( - otherMixRoundIndex, - getKNNDefaultIndexSettings(), - createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, FAISS_NAME) + otherMixRoundIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, FAISS_NAME) ); addKNNDocs(otherMixRoundIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); } @@ -178,4 +178,4 @@ public void validateKNNIndexingOnUpgrade(int totalDocsCount, int docId) throws E totalDocsCount = totalDocsCount + NUM_DOCS; validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, totalDocsCount, K); } -} +} \ No newline at end of file From d96725bc9a53ec31d259b6e0f36c4889b28c16b0 Mon Sep 17 00:00:00 2001 From: Sahil Buddharaju Date: Thu, 12 Dec 2024 13:48:12 -0800 Subject: [PATCH 8/8] spotlessApply Signed-off-by: Sahil Buddharaju --- .../org/opensearch/knn/bwc/IndexingIT.java | 60 ++++++++----------- .../org/opensearch/knn/bwc/IndexingIT.java | 39 +++++++----- 2 files changed, 47 insertions(+), 52 deletions(-) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index 9f84e4f32..08597ac04 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -37,10 +37,7 @@ import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; -import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; -import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_BITS; -import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_CONFIDENCE_INTERVAL; import static org.opensearch.knn.common.KNNConstants.NAME; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; @@ -140,17 +137,9 @@ public void testKNNIndexLuceneByteVector() throws Exception { if (isRunningAgainstOldCluster()) { createKnnIndex( - testIndex, - getKNNDefaultIndexSettings(), - createKnnIndexMapping( - TEST_FIELD, - DIMENSIONS, - METHOD_HNSW, - LUCENE_NAME, - SpaceType.L2.getValue(), - true, - VectorDataType.BYTE - ) + testIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, METHOD_HNSW, LUCENE_NAME, SpaceType.L2.getValue(), true, VectorDataType.BYTE) ); addKNNByteDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, 50); // Flush to ensure that index is not re-indexed when node comes back up @@ -164,7 +153,6 @@ public void testKNNIndexLuceneByteVector() throws Exception { } } - public void testKNNIndexLuceneQuantization() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); int k = 4; @@ -172,27 +160,27 @@ public void testKNNIndexLuceneQuantization() throws Exception { if (isRunningAgainstOldCluster()) { String mapping = XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(TEST_FIELD) - .field(VECTOR_TYPE, KNN_VECTOR) - .field(DIMENSION, dimension) - .startObject(KNN_METHOD) - .field(NAME, METHOD_HNSW) - .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) - .field(KNN_ENGINE, LUCENE_NAME) - .startObject(PARAMETERS) - .startObject(METHOD_ENCODER_PARAMETER) - .field(NAME, ENCODER_SQ) - .endObject() - .field(METHOD_PARAMETER_EF_CONSTRUCTION, 256) - .field(METHOD_PARAMETER_M, 16) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - .toString(); + .startObject() + .startObject("properties") + .startObject(TEST_FIELD) + .field(VECTOR_TYPE, KNN_VECTOR) + .field(DIMENSION, dimension) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) + .field(KNN_ENGINE, LUCENE_NAME) + .startObject(PARAMETERS) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .endObject() + .field(METHOD_PARAMETER_EF_CONSTRUCTION, 256) + .field(METHOD_PARAMETER_M, 16) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); createKnnIndex(testIndex, getKNNDefaultIndexSettings(), mapping); Float[] vector1 = { -10.6f, 25.48f }; diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index bd46886ab..d0ee16c99 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -18,7 +18,6 @@ public class IndexingIT extends AbstractRollingUpgradeTestCase { private static final String FAISS_NAME = "faiss"; private static final String LUCENE_NAME = "lucene"; - public void testKNNDefaultIndexSettings() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); switch (getClusterType()) { @@ -101,24 +100,28 @@ public void testKNNIndexCreation_withMethodMapper() throws Exception { if (isFirstMixedRound()) { docIdOld = 0; createKnnIndex( - firstMixRoundIndex, - getKNNDefaultIndexSettings(), - createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, FAISS_NAME) + firstMixRoundIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, FAISS_NAME) ); addKNNDocs(firstMixRoundIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); } else { docIdOld = 0; createKnnIndex( - otherMixRoundIndex, - getKNNDefaultIndexSettings(), - createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, FAISS_NAME) + otherMixRoundIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, FAISS_NAME) ); addKNNDocs(otherMixRoundIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); } break; case UPGRADED: docIdOld = 0; - createKnnIndex(upgradedIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, FAISS_NAME)); + createKnnIndex( + upgradedIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, FAISS_NAME) + ); addKNNDocs(upgradedIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); deleteKNNIndex(testIndex); @@ -143,24 +146,28 @@ public void testKNNLuceneIndexCreation_withMethodMapper() throws Exception { if (isFirstMixedRound()) { docIdOld = 0; createKnnIndex( - firstMixRoundIndex, - getKNNDefaultIndexSettings(), - createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, LUCENE_NAME) + firstMixRoundIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, LUCENE_NAME) ); addKNNDocs(firstMixRoundIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); } else { docIdOld = 0; createKnnIndex( - otherMixRoundIndex, - getKNNDefaultIndexSettings(), - createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, LUCENE_NAME) + otherMixRoundIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, LUCENE_NAME) ); addKNNDocs(otherMixRoundIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); } break; case UPGRADED: docIdOld = 0; - createKnnIndex(upgradedIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, LUCENE_NAME)); + createKnnIndex( + upgradedIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, LUCENE_NAME) + ); addKNNDocs(upgradedIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); deleteKNNIndex(testIndex); @@ -178,4 +185,4 @@ public void validateKNNIndexingOnUpgrade(int totalDocsCount, int docId) throws E totalDocsCount = totalDocsCount + NUM_DOCS; validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, totalDocsCount, K); } -} \ No newline at end of file +}