diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index 628b62352..a4a3db40f 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -86,9 +86,18 @@ testClusters { knn_bwc_version.startsWith("2.14.") || knn_bwc_version.startsWith("2.15.")) { filter { + excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneQuantization" excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexBinaryForceMerge" + excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneByteVector" } } + if (knn_bwc_version.startsWith("2.16.") || + knn_bwc_version.startsWith("2.17.")) { + filter { + excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneQuantization" + } + } + if (versionsBelow2_3.any {knn_bwc_version.startsWith(it) }) { filter { @@ -152,14 +161,22 @@ testClusters { knn_bwc_version.startsWith("2.7.") || knn_bwc_version.startsWith("2.8.") || knn_bwc_version.startsWith("2.9.") || - knn_bwc_version.startsWith("2.10.") || + knn_bwc_version.startsWith("2ls.10.") || knn_bwc_version.startsWith("2.11.") || knn_bwc_version.startsWith("2.12.") || knn_bwc_version.startsWith("2.13.") || knn_bwc_version.startsWith("2.14.") || knn_bwc_version.startsWith("2.15.")) { filter { + excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneQuantization" excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexBinaryForceMerge" + excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneByteVector" + } + } + if (knn_bwc_version.startsWith("2.16.") || + knn_bwc_version.startsWith("2.17.")) { + filter { + excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneQuantization" } } diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index b212d844f..1f3707f79 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -5,14 +5,18 @@ package org.opensearch.knn.bwc; +import com.google.common.collect.ImmutableMap; import org.junit.Assert; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import java.util.Arrays; import java.util.List; import java.util.Map; @@ -22,18 +26,7 @@ import static org.opensearch.knn.TestUtils.NODES_BWC_CLUSTER; import static org.opensearch.knn.TestUtils.PROPERTIES; import static org.opensearch.knn.TestUtils.VECTOR_TYPE; -import static org.opensearch.knn.common.KNNConstants.DIMENSION; -import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; -import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; -import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; -import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; -import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; -import static org.opensearch.knn.common.KNNConstants.NAME; -import static org.opensearch.knn.common.KNNConstants.PARAMETERS; +import static org.opensearch.knn.common.KNNConstants.*; public class IndexingIT extends AbstractRestartUpgradeTestCase { private static final String TEST_FIELD = "test-field"; @@ -126,6 +119,86 @@ public void testKNNIndexLuceneForceMerge() throws Exception { } } + // 2.17 and up + public void testKNNIndexLuceneByteVector() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + + if (isRunningAgainstOldCluster()) { + createKnnIndex( + testIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping( + TEST_FIELD, + DIMENSIONS, + METHOD_HNSW, + LUCENE_NAME, + SpaceType.L2.getValue(), + true, + VectorDataType.BYTE + ) + ); + addKNNByteDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, 50); + // Flush to ensure that index is not re-indexed when node comes back up + flush(testIndex, true); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, 50, 5); + } else { + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, 50, 5); + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, 50, 25); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, 75, 5); + deleteKNNIndex(testIndex); + } + } + + // 2.16 and up + public void testKNNIndexLuceneQuantization() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + + if (isRunningAgainstOldCluster()) { + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(TEST_FIELD) + .field(KNNConstants.TYPE, KNNConstants.TYPE_KNN_VECTOR) + .field(KNNConstants.DIMENSION, DIMENSIONS) + .field(VECTOR_DATA_TYPE_FIELD, VectorDataType.FLOAT) + .field("doc_values", true) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, METHOD_HNSW) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) + .field(KNNConstants.KNN_ENGINE, LUCENE_NAME) + + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, M) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, EF_CONSTRUCTION) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(LUCENE_SQ_BITS, 7) + .field(LUCENE_SQ_CONFIDENCE_INTERVAL, 1.0) + .endObject() + .endObject() + .endObject() + + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), mapping); + + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, 100); + // Flush to ensure that index is not re-indexed when node comes back up + flush(testIndex, true); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, 100, 5); + } else { + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, 100, 5); + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, 100, 50); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, 150, 5); + deleteKNNIndex(testIndex); + } + + } + // Ensure bwc works for binary force merge public void testKNNIndexBinaryForceMerge() throws Exception { int dimension = 40; diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index 10df1a79b..3cfa00c72 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -15,7 +15,9 @@ public class IndexingIT extends AbstractRollingUpgradeTestCase { private static final String ALGO = "hnsw"; - private static final String ENGINE = "faiss"; + private static final String FAISS_NAME = "faiss"; + private static final String LUCENE_NAME = "lucene"; + public void testKNNDefaultIndexSettings() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); @@ -91,7 +93,7 @@ public void testKNNIndexCreation_withMethodMapper() throws Exception { final String upgradedIndex = testIndex + "upgraded"; switch (getClusterType()) { case OLD: - createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, ENGINE)); + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, FAISS_NAME)); int docIdOld = 0; addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); break; @@ -101,7 +103,7 @@ public void testKNNIndexCreation_withMethodMapper() throws Exception { createKnnIndex( firstMixRoundIndex, getKNNDefaultIndexSettings(), - createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, ENGINE) + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, FAISS_NAME) ); addKNNDocs(firstMixRoundIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); } else { @@ -109,14 +111,56 @@ public void testKNNIndexCreation_withMethodMapper() throws Exception { createKnnIndex( otherMixRoundIndex, getKNNDefaultIndexSettings(), - createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, ENGINE) + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, FAISS_NAME) + ); + addKNNDocs(otherMixRoundIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); + } + break; + case UPGRADED: + docIdOld = 0; + createKnnIndex(upgradedIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, FAISS_NAME)); + addKNNDocs(upgradedIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); + + deleteKNNIndex(testIndex); + deleteKNNIndex(firstMixRoundIndex); + deleteKNNIndex(otherMixRoundIndex); + deleteKNNIndex(upgradedIndex); + } + } + + public void testKNNLuceneIndexCreation_withMethodMapper() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + final String firstMixRoundIndex = testIndex + "first-mix-round"; + final String otherMixRoundIndex = testIndex + "other-mix-round"; + final String upgradedIndex = testIndex + "upgraded"; + switch (getClusterType()) { + case OLD: + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, LUCENE_NAME)); + int docIdOld = 0; + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); + break; + case MIXED: + if (isFirstMixedRound()) { + docIdOld = 0; + createKnnIndex( + firstMixRoundIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, LUCENE_NAME) + ); + addKNNDocs(firstMixRoundIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); + } else { + docIdOld = 0; + createKnnIndex( + otherMixRoundIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, LUCENE_NAME) ); addKNNDocs(otherMixRoundIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); } break; case UPGRADED: docIdOld = 0; - createKnnIndex(upgradedIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, ENGINE)); + createKnnIndex(upgradedIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, LUCENE_NAME)); addKNNDocs(upgradedIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); deleteKNNIndex(testIndex);