From 82c02de9141bad3bfc3ebd44a9c3abaa925e649d Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Mon, 18 Nov 2024 12:27:05 -0500 Subject: [PATCH] Fixing MultiDenseVectorScriptDocValuesTests tests (#116940) This fixes two test issues: - 1. Now the tests skip if the multi_dense_vector feature isn't enabled - 2. fixes silly bwc testing where we were testing for big-endian floats, which aren't possible. closes: https://github.com/elastic/elasticsearch/issues/116862 closes: https://github.com/elastic/elasticsearch/issues/116863 --- muted-tests.yml | 3 - .../MultiDenseVectorScriptDocValuesTests.java | 72 +++++++++---------- 2 files changed, 36 insertions(+), 39 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 4382b133522c6..8696a1bddd3d4 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -234,9 +234,6 @@ tests: - class: org.elasticsearch.search.basic.SearchWithRandomIOExceptionsIT method: testRandomDirectoryIOExceptions issue: https://github.com/elastic/elasticsearch/issues/114824 -- class: org.elasticsearch.index.mapper.vectors.MultiDenseVectorScriptDocValuesTests - method: testFloatGetVectorValueAndGetMagnitude - issue: https://github.com/elastic/elasticsearch/issues/116863 - class: org.elasticsearch.xpack.inference.InferenceRestIT method: test {p0=inference/30_semantic_text_inference/Calculates embeddings using the default ELSER 2 endpoint} issue: https://github.com/elastic/elasticsearch/issues/116542 diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorScriptDocValuesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorScriptDocValuesTests.java index ef316c5addefa..435baa477e740 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorScriptDocValuesTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorScriptDocValuesTests.java @@ -18,46 +18,48 @@ import org.elasticsearch.script.field.vectors.MultiDenseVector; import org.elasticsearch.script.field.vectors.MultiDenseVectorDocValuesField; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.index.IndexVersionUtils; +import org.junit.BeforeClass; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Iterator; -import java.util.List; import static org.hamcrest.Matchers.containsString; public class MultiDenseVectorScriptDocValuesTests extends ESTestCase { + @BeforeClass + public static void setup() { + assumeTrue("Requires multi-dense vector support", MultiDenseVectorFieldMapper.FEATURE_FLAG.isEnabled()); + } + public void testFloatGetVectorValueAndGetMagnitude() throws IOException { int dims = 3; float[][][] vectors = { { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } }, { { 1, 0, 2 } } }; float[][] expectedMagnitudes = { { 1.7320f, 2.4495f, 3.3166f }, { 2.2361f } }; - for (IndexVersion indexVersion : List.of(IndexVersionUtils.randomCompatibleVersion(random()), IndexVersion.current())) { - BinaryDocValues docValues = wrap(vectors, ElementType.FLOAT, indexVersion); - BinaryDocValues magnitudeValues = wrap(expectedMagnitudes); - MultiDenseVectorDocValuesField field = new FloatMultiDenseVectorDocValuesField( - docValues, - magnitudeValues, - "test", - ElementType.FLOAT, - dims - ); - MultiDenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues(); - for (int i = 0; i < vectors.length; i++) { - field.setNextDocId(i); - assertEquals(vectors[i].length, field.size()); - assertEquals(dims, scriptDocValues.dims()); - Iterator iterator = scriptDocValues.getVectorValues(); - float[] magnitudes = scriptDocValues.getMagnitudes(); - assertEquals(expectedMagnitudes[i].length, magnitudes.length); - for (int j = 0; j < vectors[i].length; j++) { - assertTrue(iterator.hasNext()); - assertArrayEquals(vectors[i][j], iterator.next(), 0.0001f); - assertEquals(expectedMagnitudes[i][j], magnitudes[j], 0.0001f); - } + BinaryDocValues docValues = wrap(vectors, ElementType.FLOAT); + BinaryDocValues magnitudeValues = wrap(expectedMagnitudes); + MultiDenseVectorDocValuesField field = new FloatMultiDenseVectorDocValuesField( + docValues, + magnitudeValues, + "test", + ElementType.FLOAT, + dims + ); + MultiDenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues(); + for (int i = 0; i < vectors.length; i++) { + field.setNextDocId(i); + assertEquals(vectors[i].length, field.size()); + assertEquals(dims, scriptDocValues.dims()); + Iterator iterator = scriptDocValues.getVectorValues(); + float[] magnitudes = scriptDocValues.getMagnitudes(); + assertEquals(expectedMagnitudes[i].length, magnitudes.length); + for (int j = 0; j < vectors[i].length; j++) { + assertTrue(iterator.hasNext()); + assertArrayEquals(vectors[i][j], iterator.next(), 0.0001f); + assertEquals(expectedMagnitudes[i][j], magnitudes[j], 0.0001f); } } } @@ -67,7 +69,7 @@ public void testByteGetVectorValueAndGetMagnitude() throws IOException { float[][][] vectors = { { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } }, { { 1, 0, 2 } } }; float[][] expectedMagnitudes = { { 1.7320f, 2.4495f, 3.3166f }, { 2.2361f } }; - BinaryDocValues docValues = wrap(vectors, ElementType.BYTE, IndexVersion.current()); + BinaryDocValues docValues = wrap(vectors, ElementType.BYTE); BinaryDocValues magnitudeValues = wrap(expectedMagnitudes); MultiDenseVectorDocValuesField field = new ByteMultiDenseVectorDocValuesField( docValues, @@ -94,10 +96,9 @@ public void testByteGetVectorValueAndGetMagnitude() throws IOException { public void testFloatMetadataAndIterator() throws IOException { int dims = 3; - IndexVersion indexVersion = IndexVersion.current(); float[][][] vectors = new float[][][] { fill(new float[3][dims], ElementType.FLOAT), fill(new float[2][dims], ElementType.FLOAT) }; float[][] magnitudes = new float[][] { new float[3], new float[2] }; - BinaryDocValues docValues = wrap(vectors, ElementType.FLOAT, indexVersion); + BinaryDocValues docValues = wrap(vectors, ElementType.FLOAT); BinaryDocValues magnitudeValues = wrap(magnitudes); MultiDenseVectorDocValuesField field = new FloatMultiDenseVectorDocValuesField( @@ -123,10 +124,9 @@ public void testFloatMetadataAndIterator() throws IOException { public void testByteMetadataAndIterator() throws IOException { int dims = 3; - IndexVersion indexVersion = IndexVersion.current(); float[][][] vectors = new float[][][] { fill(new float[3][dims], ElementType.BYTE), fill(new float[2][dims], ElementType.BYTE) }; float[][] magnitudes = new float[][] { new float[3], new float[2] }; - BinaryDocValues docValues = wrap(vectors, ElementType.BYTE, indexVersion); + BinaryDocValues docValues = wrap(vectors, ElementType.BYTE); BinaryDocValues magnitudeValues = wrap(magnitudes); MultiDenseVectorDocValuesField field = new ByteMultiDenseVectorDocValuesField( docValues, @@ -162,7 +162,7 @@ public void testFloatMissingValues() throws IOException { int dims = 3; float[][][] vectors = { { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } }, { { 1, 0, 2 } } }; float[][] magnitudes = { { 1.7320f, 2.4495f, 3.3166f }, { 2.2361f } }; - BinaryDocValues docValues = wrap(vectors, ElementType.FLOAT, IndexVersion.current()); + BinaryDocValues docValues = wrap(vectors, ElementType.FLOAT); BinaryDocValues magnitudeValues = wrap(magnitudes); MultiDenseVectorDocValuesField field = new FloatMultiDenseVectorDocValuesField( docValues, @@ -186,7 +186,7 @@ public void testByteMissingValues() throws IOException { int dims = 3; float[][][] vectors = { { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } }, { { 1, 0, 2 } } }; float[][] magnitudes = { { 1.7320f, 2.4495f, 3.3166f }, { 2.2361f } }; - BinaryDocValues docValues = wrap(vectors, ElementType.BYTE, IndexVersion.current()); + BinaryDocValues docValues = wrap(vectors, ElementType.BYTE); BinaryDocValues magnitudeValues = wrap(magnitudes); MultiDenseVectorDocValuesField field = new ByteMultiDenseVectorDocValuesField( docValues, @@ -210,7 +210,7 @@ public void testFloatGetFunctionIsNotAccessible() throws IOException { int dims = 3; float[][][] vectors = { { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } }, { { 1, 0, 2 } } }; float[][] magnitudes = { { 1.7320f, 2.4495f, 3.3166f }, { 2.2361f } }; - BinaryDocValues docValues = wrap(vectors, ElementType.FLOAT, IndexVersion.current()); + BinaryDocValues docValues = wrap(vectors, ElementType.FLOAT); BinaryDocValues magnitudeValues = wrap(magnitudes); MultiDenseVectorDocValuesField field = new FloatMultiDenseVectorDocValuesField( docValues, @@ -236,7 +236,7 @@ public void testByteGetFunctionIsNotAccessible() throws IOException { int dims = 3; float[][][] vectors = { { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } }, { { 1, 0, 2 } } }; float[][] magnitudes = { { 1.7320f, 2.4495f, 3.3166f }, { 2.2361f } }; - BinaryDocValues docValues = wrap(vectors, ElementType.BYTE, IndexVersion.current()); + BinaryDocValues docValues = wrap(vectors, ElementType.BYTE); BinaryDocValues magnitudeValues = wrap(magnitudes); MultiDenseVectorDocValuesField field = new ByteMultiDenseVectorDocValuesField( docValues, @@ -306,7 +306,7 @@ public long cost() { }; } - public static BinaryDocValues wrap(float[][][] vectors, ElementType elementType, IndexVersion indexVersion) { + public static BinaryDocValues wrap(float[][][] vectors, ElementType elementType) { return new BinaryDocValues() { int idx = -1; int maxIdx = vectors.length; @@ -316,7 +316,7 @@ public BytesRef binaryValue() { if (idx >= maxIdx) { throw new IllegalStateException("max index exceeded"); } - return mockEncodeDenseVector(vectors[idx], elementType, indexVersion); + return mockEncodeDenseVector(vectors[idx], elementType, IndexVersion.current()); } @Override