Add new experimental rank_vectors mapping for late-interaction second order ranking

docs/changelog/118804.yaml

@@ -0,0 +1,15 @@
+pr: 118804
+summary: Add new experimental `rank_vectors` mapping for late-interaction second order
+  ranking
+area: Vector Search
+type: feature
+issues: []
+highlight:
+  title: Add new experimental `rank_vectors` mapping for late-interaction second order
+    ranking
+  body:
+      Late-interaction models are powerful rerankers. While their size and overall
+      cost doesn't lend itself for HNSW indexing, utilizing them as second order reranking
+      can provide excellent boosts in relevance. The new `rank_vectors` mapping allows for rescoring
+      over new and novel multi-vector late-interaction models like ColBERT or ColPali.
+  notable: true

docs/reference/mapping/types.asciidoc

@@ -180,6 +180,8 @@ include::types/rank-feature.asciidoc[]
 
 include::types/rank-features.asciidoc[]
 
+include::types/rank-vectors.asciidoc[]
+
 include::types/search-as-you-type.asciidoc[]
 
 include::types/semantic-text.asciidoc[]

docs/reference/mapping/types/dense-vector.asciidoc

@@ -1,4 +1,3 @@
-[role="xpack"]
 [[dense-vector]]
 === Dense vector field type
 ++++

docs/reference/mapping/types/rank-vectors.asciidoc

@@ -0,0 +1,183 @@
+[role="xpack"]
+[[rank-vectors]]
+=== Rank Vectors
+++++
+<titleabbrev> Rank Vectors </titleabbrev>
+++++
+experimental::[]
+
+The `rank_vectors` field type enables late-interaction dense vector scoring in Elasticsearch. The number of vectors
+per field can vary, but they must all share the same number of dimensions and element type.
+
+The purpose of vectors stored in this field is second order ranking documents with max-sim similarity.
+
+Here is a simple example of using this field with `float` elements.
+
+[source,console]
+--------------------------------------------------
+PUT my-rank-vectors-float
+{
+  "mappings": {
+    "properties": {
+      "my_vector": {
+        "type": "rank_vectors"
+      }
+    }
+  }
+}
+
+PUT my-rank-vectors-float/_doc/1
+{
+  "my_vector" : [[0.5, 10, 6], [-0.5, 10, 10]]
+}
+
+--------------------------------------------------
+
+In addition to the `float` element type, `byte` and `bit` element types are also supported.
+
+Here is an example of using this field with `byte` elements.
+
+[source,console]
+--------------------------------------------------
+PUT my-rank-vectors-byte
+{
+  "mappings": {
+    "properties": {
+      "my_vector": {
+        "type": "rank_vectors",
+        "element_type": "byte"
+      }
+    }
+  }
+}
+
+PUT my-rank-vectors-byte/_doc/1
+{
+  "my_vector" : [[1, 2, 3], [4, 5, 6]]
+}
+--------------------------------------------------
+
+Here is an example of using this field with `bit` elements.
+
+[source,console]
+--------------------------------------------------
+PUT my-rank-vectors-bit
+{
+  "mappings": {
+    "properties": {
+      "my_vector": {
+        "type": "rank_vectors",
+        "element_type": "bit"
+      }
+    }
+  }
+}
+
+POST /my-bit-vectors/_bulk?refresh
+{"index": {"_id" : "1"}}
+{"my_vector": [127, -127, 0, 1, 42]}
+{"index": {"_id" : "2"}}
+{"my_vector": "8100012a7f"}
+--------------------------------------------------
+
+
+[role="child_attributes"]
+[[rank-vectors-params]]
+==== Parameters for rank vectors fields
+
+The `rank_vectors` field type supports the following parameters:
+
+[[rank-vectors-element-type]]
+`element_type`::
+(Optional, string)
+The data type used to encode vectors. The supported data types are
+`float` (default), `byte`, and bit.
+
+.Valid values for `element_type`
+[%collapsible%open]
+====
+`float`:::
+indexes a 4-byte floating-point
+value per dimension. This is the default value.
+
+`byte`:::
+indexes a 1-byte integer value per dimension.
+
+`bit`:::
+indexes a single bit per dimension. Useful for very high-dimensional vectors or models that specifically support bit vectors.
+NOTE: when using `bit`, the number of dimensions must be a multiple of 8 and must represent the number of bits.
+
+====
+
+`dims`::
+(Optional, integer)
+Number of vector dimensions. Can't exceed `4096`. If `dims` is not specified,
+it will be set to the length of the first vector added to the field.
+
+[[rank-vectors-synthetic-source]]
+==== Synthetic `_source`
+
+IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
+(indices that have `index.mode` set to `time_series`). For other indices
+synthetic `_source` is in technical preview. Features in technical preview may
+be changed or removed in a future release. Elastic will work to fix
+any issues, but features in technical preview are not subject to the support SLA
+of official GA features.
+
+`rank_vectors` fields support <<synthetic-source,synthetic `_source`>> .
+
+[[rank-vectors-scoring]]
+==== Scoring with rank vectors
+
+Rank vectors can be accessed and used in <<query-dsl-script-score-query,`script_score` queries>>.
+
+For example, the following query scores documents based on the maxSim similarity between the query vector and the vectors stored in the `my_vector` field:
+
+[source,console]
+--------------------------------------------------
+GET my-index/_search
+{
+  "query": {
+    "script_score": {
+      "query": {
+        "match_all": {}
+      },
+      "script": {
+        "source": "maxSimDotProduct(params.query_vector, 'my_vector')",
+        "params": {
+          "query_vector": [[0.5, 10, 6], [-0.5, 10, 10]]
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+
+Additionally, asymmetric similarity functions can be used to score against `bit` vectors. For example, the following query scores documents based on the maxSimDotProduct similarity between a floating point query vector and bit vectors stored in the `my_vector` field:
+
+[source,console]
+--------------------------------------------------
+GET my-index/_search
+{
+  "query": {
+    "script_score": {
+      "query": {
+        "match_all": {}
+      },
+      "script": {
+        "source": "maxSimDotProduct(params.query_vector, 'my_vector')",
+        "params": {
+          "query_vector": [
+            [0.35, 0.77, 0.95, 0.15, 0.11, 0.08, 0.58, 0.06, 0.44, 0.52, 0.21,
+       0.62, 0.65, 0.16, 0.64, 0.39, 0.93, 0.06, 0.93, 0.31, 0.92, 0.0,
+       0.66, 0.86, 0.92, 0.03, 0.81, 0.31, 0.2 , 0.92, 0.95, 0.64, 0.19,
+       0.26, 0.77, 0.64, 0.78, 0.32, 0.97, 0.84]
+           ] <1>
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+<1> Note that the query vector has 40 elements, matching the number of bits in the bit vectors.
+

modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt

@@ -50,7 +50,5 @@ static_import {
     double cosineSimilarity(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$CosineSimilarity
     double dotProduct(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$DotProduct
     double hamming(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$Hamming
-    double maxSimDotProduct(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.RankVectorsScoreScriptUtils$MaxSimDotProduct
-    double maxSimInvHamming(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.RankVectorsScoreScriptUtils$MaxSimInvHamming
 }
 

server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java

@@ -102,7 +102,7 @@
 public class DenseVectorFieldMapper extends FieldMapper {
     public static final String COSINE_MAGNITUDE_FIELD_SUFFIX = "._magnitude";
     private static final float EPS = 1e-3f;
-    static final int BBQ_MIN_DIMS = 64;
+    public static final int BBQ_MIN_DIMS = 64;
 
     public static boolean isNotUnitVector(float magnitude) {
         return Math.abs(magnitude - 1.0f) > EPS;
@@ -486,8 +486,12 @@ private VectorData parseHexEncodedVector(
             }
 
             @Override
-            VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity)
-                throws IOException {
+            public VectorData parseKnnVector(
+                DocumentParserContext context,
+                int dims,
+                IntBooleanConsumer dimChecker,
+                VectorSimilarity similarity
+            ) throws IOException {
                 XContentParser.Token token = context.parser().currentToken();
                 return switch (token) {
                     case START_ARRAY -> parseVectorArray(context, dims, dimChecker, similarity);
@@ -517,17 +521,17 @@ public void parseKnnVectorAndIndex(DocumentParserContext context, DenseVectorFie
             }
 
             @Override
-            int getNumBytes(int dimensions) {
+            public int getNumBytes(int dimensions) {
                 return dimensions;
             }
 
             @Override
-            ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) {
+            public ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) {
                 return ByteBuffer.wrap(new byte[numBytes]);
             }
 
             @Override
-            int parseDimensionCount(DocumentParserContext context) throws IOException {
+            public int parseDimensionCount(DocumentParserContext context) throws IOException {
                 XContentParser.Token currentToken = context.parser().currentToken();
                 return switch (currentToken) {
                     case START_ARRAY -> {
@@ -691,8 +695,12 @@ && isNotUnitVector(squaredMagnitude)) {
             }
 
             @Override
-            VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity)
-                throws IOException {
+            public VectorData parseKnnVector(
+                DocumentParserContext context,
+                int dims,
+                IntBooleanConsumer dimChecker,
+                VectorSimilarity similarity
+            ) throws IOException {
                 int index = 0;
                 float squaredMagnitude = 0;
                 float[] vector = new float[dims];
@@ -711,12 +719,12 @@ VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanCon
             }
 
             @Override
-            int getNumBytes(int dimensions) {
+            public int getNumBytes(int dimensions) {
                 return dimensions * Float.BYTES;
             }
 
             @Override
-            ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) {
+            public ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) {
                 return indexVersion.onOrAfter(LITTLE_ENDIAN_FLOAT_STORED_INDEX_VERSION)
                     ? ByteBuffer.wrap(new byte[numBytes]).order(ByteOrder.LITTLE_ENDIAN)
                     : ByteBuffer.wrap(new byte[numBytes]);
@@ -889,8 +897,12 @@ private VectorData parseHexEncodedVector(DocumentParserContext context, IntBoole
             }
 
             @Override
-            VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity)
-                throws IOException {
+            public VectorData parseKnnVector(
+                DocumentParserContext context,
+                int dims,
+                IntBooleanConsumer dimChecker,
+                VectorSimilarity similarity
+            ) throws IOException {
                 XContentParser.Token token = context.parser().currentToken();
                 return switch (token) {
                     case START_ARRAY -> parseVectorArray(context, dims, dimChecker, similarity);
@@ -920,18 +932,18 @@ public void parseKnnVectorAndIndex(DocumentParserContext context, DenseVectorFie
             }
 
             @Override
-            int getNumBytes(int dimensions) {
+            public int getNumBytes(int dimensions) {
                 assert dimensions % Byte.SIZE == 0;
                 return dimensions / Byte.SIZE;
             }
 
             @Override
-            ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) {
+            public ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) {
                 return ByteBuffer.wrap(new byte[numBytes]);
             }
 
             @Override
-            int parseDimensionCount(DocumentParserContext context) throws IOException {
+            public int parseDimensionCount(DocumentParserContext context) throws IOException {
                 XContentParser.Token currentToken = context.parser().currentToken();
                 return switch (currentToken) {
                     case START_ARRAY -> {
@@ -974,16 +986,16 @@ public void checkDimensions(Integer dvDims, int qvDims) {
 
         abstract void parseKnnVectorAndIndex(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException;
 
-        abstract VectorData parseKnnVector(
+        public abstract VectorData parseKnnVector(
             DocumentParserContext context,
             int dims,
             IntBooleanConsumer dimChecker,
             VectorSimilarity similarity
         ) throws IOException;
 
-        abstract int getNumBytes(int dimensions);
+        public abstract int getNumBytes(int dimensions);
 
-        abstract ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes);
+        public abstract ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes);
 
         public abstract void checkVectorBounds(float[] vector);
 
@@ -1001,7 +1013,7 @@ public void checkDimensions(Integer dvDims, int qvDims) {
             }
         }
 
-        int parseDimensionCount(DocumentParserContext context) throws IOException {
+        public int parseDimensionCount(DocumentParserContext context) throws IOException {
             int index = 0;
             for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
                 index++;
@@ -1088,7 +1100,7 @@ public static ElementType fromString(String name) {
         }
     }
 
-    static final Map<String, ElementType> namesToElementType = Map.of(
+    public static final Map<String, ElementType> namesToElementType = Map.of(
         ElementType.BYTE.toString(),
         ElementType.BYTE,
         ElementType.FLOAT.toString(),
@@ -2500,9 +2512,10 @@ public String fieldName() {
     }
 
     /**
-     * @FunctionalInterface for a function that takes a int and boolean
+     * Interface for a function that takes a int and boolean
      */
-    interface IntBooleanConsumer {
+    @FunctionalInterface
+    public interface IntBooleanConsumer {
         void accept(int value, boolean isComplete);
     }
 }