Merge branch 'main' into carlosdelest/fix-dense-vector-ci-test

carlosdelest · Dec 4, 2023 · d47cf04 · d47cf04
2 parents be755e1 + 605c76e
commit d47cf04
Show file tree

Hide file tree

Showing 204 changed files with 5,866 additions and 1,124 deletions.
diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties
@@ -1,5 +1,5 @@
 elasticsearch     = 8.12.0
-lucene            = 9.8.0
+lucene            = 9.9.0
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 21.0.1+12@415e3f918a1f4062a0074a2794853d0d

diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc
@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        9.8.0
-:lucene_version_path:   9_8_0
+:lucene_version:        9.9.0
+:lucene_version_path:   9_9_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

diff --git a/docs/changelog/102032.yaml b/docs/changelog/102032.yaml
@@ -0,0 +1,5 @@
+pr: 102032
+summary: Add vector_operation_count in profile output for knn searches
+area: Vector Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/102093.yaml b/docs/changelog/102093.yaml
@@ -0,0 +1,14 @@
+pr: 102093
+summary: Add byte quantization for float vectors in HNSW 
+area: Vector Search
+type: feature
+issues: []
+highlight:
+  title: Add new `int8_hsnw` index type for int8 quantization for HNSW
+  body: |-
+    This commit adds a new index type called `int8_hnsw`. This index will
+    automatically quantized float32 values into int8 byte values. While
+    this increases disk usage by 25%, it reduces memory required for 
+    fast HNSW search by 75%. Dramatically reducing the resource overhead
+    required for dense vector search.
+  notable: true
diff --git a/docs/changelog/102782.yaml b/docs/changelog/102782.yaml
@@ -0,0 +1,5 @@
+pr: 102782
+summary: Upgrade to Lucene 9.9.0
+area: Search
+type: upgrade
+issues: []
diff --git a/docs/changelog/102877.yaml b/docs/changelog/102877.yaml
@@ -0,0 +1,5 @@
+pr: 102877
+summary: Add basic telelemetry for the inference feature
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/102891.yaml b/docs/changelog/102891.yaml
@@ -0,0 +1,7 @@
+pr: 102891
+summary: "[Query Rules] Fix bug where combining the same metadata with text/numeric\
+  \ values leads to error"
+area: Application
+type: bug
+issues:
+ - 102827
diff --git a/docs/reference/esql/functions/signature/to_boolean.svg b/docs/reference/esql/functions/signature/to_boolean.svg
diff --git a/docs/reference/esql/functions/signature/to_datetime.svg b/docs/reference/esql/functions/signature/to_datetime.svg
diff --git a/docs/reference/esql/functions/signature/to_degrees.svg b/docs/reference/esql/functions/signature/to_degrees.svg
diff --git a/docs/reference/esql/functions/signature/to_double.svg b/docs/reference/esql/functions/signature/to_double.svg
diff --git a/docs/reference/esql/functions/signature/to_integer.svg b/docs/reference/esql/functions/signature/to_integer.svg
diff --git a/docs/reference/esql/functions/signature/to_ip.svg b/docs/reference/esql/functions/signature/to_ip.svg
diff --git a/docs/reference/esql/functions/signature/to_long.svg b/docs/reference/esql/functions/signature/to_long.svg
diff --git a/docs/reference/esql/functions/signature/to_radians.svg b/docs/reference/esql/functions/signature/to_radians.svg
diff --git a/docs/reference/esql/functions/signature/to_unsigned_long.svg b/docs/reference/esql/functions/signature/to_unsigned_long.svg
diff --git a/docs/reference/esql/functions/types/mv_count.asciidoc b/docs/reference/esql/functions/types/mv_count.asciidoc
@@ -2,8 +2,10 @@
 |===
 v | result
 boolean | integer
+cartesian_point | integer
 datetime | integer
 double | integer
+geo_point | integer
 integer | integer
 ip | integer
 keyword | integer

diff --git a/docs/reference/esql/functions/types/to_boolean.asciidoc b/docs/reference/esql/functions/types/to_boolean.asciidoc
@@ -0,0 +1,11 @@
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+v | result
+boolean | boolean
+double | boolean
+integer | boolean
+keyword | boolean
+long | boolean
+text | boolean
+unsigned_long | boolean
+|===
diff --git a/docs/reference/esql/functions/types/to_datetime.asciidoc b/docs/reference/esql/functions/types/to_datetime.asciidoc
@@ -0,0 +1,11 @@
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+v | result
+datetime | datetime
+double | datetime
+integer | datetime
+keyword | datetime
+long | datetime
+text | datetime
+unsigned_long | datetime
+|===
diff --git a/docs/reference/esql/functions/types/to_degrees.asciidoc b/docs/reference/esql/functions/types/to_degrees.asciidoc
@@ -0,0 +1,8 @@
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+v | result
+double | double
+integer | double
+long | double
+unsigned_long | double
+|===
diff --git a/docs/reference/esql/functions/types/to_double.asciidoc b/docs/reference/esql/functions/types/to_double.asciidoc
@@ -0,0 +1,12 @@
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+v | result
+boolean | double
+datetime | double
+double | double
+integer | double
+keyword | double
+long | double
+text | double
+unsigned_long | double
+|===
diff --git a/docs/reference/esql/functions/types/to_integer.asciidoc b/docs/reference/esql/functions/types/to_integer.asciidoc
@@ -0,0 +1,12 @@
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+v | result
+boolean | integer
+datetime | integer
+double | integer
+integer | integer
+keyword | integer
+long | integer
+text | integer
+unsigned_long | integer
+|===
diff --git a/docs/reference/esql/functions/types/to_ip.asciidoc b/docs/reference/esql/functions/types/to_ip.asciidoc
@@ -1,6 +1,7 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
-arg1 | result
+v | result
 ip | ip
 keyword | ip
+text | ip
 |===
diff --git a/docs/reference/esql/functions/types/to_long.asciidoc b/docs/reference/esql/functions/types/to_long.asciidoc
@@ -0,0 +1,14 @@
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+v | result
+boolean | long
+cartesian_point | long
+datetime | long
+double | long
+geo_point | long
+integer | long
+keyword | long
+long | long
+text | long
+unsigned_long | long
+|===
diff --git a/docs/reference/esql/functions/types/to_radians.asciidoc b/docs/reference/esql/functions/types/to_radians.asciidoc
@@ -0,0 +1,8 @@
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+v | result
+double | double
+integer | double
+long | double
+unsigned_long | double
+|===
diff --git a/docs/reference/esql/functions/types/to_string.asciidoc b/docs/reference/esql/functions/types/to_string.asciidoc
@@ -2,8 +2,10 @@
 |===
 v | result
 boolean | keyword
+cartesian_point | keyword
 datetime | keyword
 double | keyword
+geo_point | keyword
 integer | keyword
 ip | keyword
 keyword | keyword

diff --git a/docs/reference/esql/functions/types/to_unsigned_long.asciidoc b/docs/reference/esql/functions/types/to_unsigned_long.asciidoc
@@ -0,0 +1,12 @@
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+v | result
+boolean | unsigned_long
+datetime | unsigned_long
+double | unsigned_long
+integer | unsigned_long
+keyword | unsigned_long
+long | unsigned_long
+text | unsigned_long
+unsigned_long | unsigned_long
+|===
diff --git a/docs/reference/how-to/knn-search.asciidoc b/docs/reference/how-to/knn-search.asciidoc
@@ -52,7 +52,12 @@ of datasets and configurations that we use for our nightly benchmarks.
 include::search-speed.asciidoc[tag=warm-fs-cache]
 
 The following file extensions are used for the approximate kNN search:
-"vec" (for vector values), "vex" (for HNSW graph), "vem" (for metadata).
++
+--
+* `vec` and `veq` for vector values
+* `vex` for HNSW graph
+* `vem`, `vemf`, and `vemq` for metadata
+--
 
 [discrete]
 === Reduce vector dimensionality
@@ -66,6 +71,14 @@ reduction techniques like PCA. When experimenting with different approaches,
 it's important to measure the impact on relevance to ensure the search
 quality is still acceptable.
 
+[discrete]
+=== Reduce vector memory foot-print
+
+The default <<dense-vector-element-type,`element_type`>> is `float`. But this can be
+automatically quantized during index time through <<dense-vector-quantization,`quantization`>>. Quantization will
+reduce the required memory by 4x, but it will also reduce the precision of the vectors. For `float` vectors with
+`dim` greater than or equal to `384`, using a <<dense-vector-quantization,`quantized`>> index is highly recommended.
+
 [discrete]
 === Exclude vector fields from `_source`
 

diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc
@@ -111,6 +111,36 @@ PUT my-index-2
 efficient kNN search. Like most kNN algorithms, HNSW is an approximate method
 that sacrifices result accuracy for improved speed.
 
+[[dense-vector-quantization]]
+==== Automatically quantize vectors for kNN search
+
+The `dense_vector` type supports quantization to reduce the memory footprint required when <<approximate-knn, searching>> `float` vectors.
+Currently the only quantization method supported is `int8` and provided vectors `element_type` must be `float`. To use
+a quantized index, you can set your index type to `int8_hnsw`.
+
+When using the `int8_hnsw` index, each of the `float` vectors' dimensions are quantized to 1-byte integers. This can
+reduce the memory footprint by as much as 75% at the cost of some accuracy. However, the disk usage can increase by
+25% due to the overhead of storing the quantized and raw vectors.
+
+[source,console]
+--------------------------------------------------
+PUT my-byte-quantized-index
+{
+  "mappings": {
+    "properties": {
+      "my_vector": {
+        "type": "dense_vector",
+        "dims": 3,
+        "index": true,
+        "index_options": {
+          "type": "int8_hnsw"
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+
 [role="child_attributes"]
 [[dense-vector-params]]
 ==== Parameters for dense vector fields
@@ -198,8 +228,7 @@ a distinct set of options.
 An optional section that configures the kNN indexing algorithm. The HNSW
 algorithm has two internal parameters that influence how the data structure is
 built. These can be adjusted to improve the accuracy of results, at the
-expense of slower indexing speed. When `index_options` is provided, all of its
-properties must be defined.
+expense of slower indexing speed.
 +
 ^*^ This parameter can only be specified when `index` is `true`.
 +
@@ -209,17 +238,25 @@ properties must be defined.
 ====
 `type`:::
 (Required, string)
-The type of kNN algorithm to use. Currently only `hnsw` is supported.
+The type of kNN algorithm to use. Can be either `hnsw` or `int8_hnsw`.
 
 `m`:::
-(Required, integer)
+(Optional, integer)
 The number of neighbors each node will be connected to in the HNSW graph.
 Defaults to `16`.
 
 `ef_construction`:::
-(Required, integer)
+(Optional, integer)
 The number of candidates to track while assembling the list of nearest
 neighbors for each new node. Defaults to `100`.
+
+`confidence_interval`:::
+(Optional, float)
+Only applicable to `int8_hnsw` index types. The confidence interval to use when quantizing the vectors,
+can be any value between and including `0.90` and `1.0`. This value restricts the values used when calculating
+the quantization thresholds. For example, a value of `0.95` will only use the middle 95% of the values when
+calculating the quantization thresholds (e.g. the highest and lowest 2.5% of values will be ignored).
+Defaults to `1/(dims + 1)`.
 ====
 
 [[dense-vector-synthetic-source]]

diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc
@@ -197,6 +197,11 @@ GET /_xpack/usage
     },
     "node_count" : 1
   },
+  "inference": {
+    "available" : true,
+    "enabled" : true,
+    "models" : []
+  },
   "logstash" : {
     "available" : true,
     "enabled" : true

diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc
@@ -1272,6 +1272,7 @@ One of the `dfs.knn` sections for a shard looks like the following:
 "dfs" : {
     "knn" : [
         {
+        "vector_operations_count" : 4,
         "query" : [
             {
                 "type" : "DocAndScoreQuery",
@@ -1321,7 +1322,7 @@ In the `dfs.knn` portion of the response we can see the output
 the of timings for <<query-section, query>>, <<rewrite-section, rewrite>>,
 and <<collectors-section, collector>>. Unlike many other queries, kNN
 search does the bulk of the work during the query rewrite. This means
-`rewrite_time` represents the time spent on kNN search.
+`rewrite_time` represents the time spent on kNN search. The attribute `vector_operations_count` represents the overall count of vector operations performed during the kNN search.
 
 [[profiling-considerations]]
 ===== Profiling Considerations