Merge branch 'main' into fix-indices-security-resolver-with-ff

docs/reference/query-dsl/intervals-query.asciidoc

@@ -303,12 +303,22 @@ matches. Defaults to `-1`.
 
 If unspecified or set to `-1`, there is no width restriction on the match. If
 set to `0`, the terms must appear next to each other.
+
+Internal intervals can have their own `max_gaps` values. In this case
+we first find internal intervals with their `max_gaps` values, and then
+combine them to see if a gap between internal intervals match
+the value of `max_gaps` of the `all_of` rule.
+
+For examples, how `max_gaps` works, see <<interval-max_gaps-all-rule>>.
 --
 
 `ordered`::
 (Optional, Boolean) If `true`, intervals produced by the rules should appear in
 the order in which they are specified. Defaults to `false`.
 
+If `ordered` is `false`, intervals can appear in any order,
+including overlapping with each other.
+
 `filter`::
 (Optional, <<interval_filter,interval filter>> rule object) Rule used to filter
 returned intervals.
@@ -468,3 +478,94 @@ This query does *not* match a document containing the phrase `hot porridge is
 salty porridge`, because the intervals returned by the match query for `hot
 porridge` only cover the initial two terms in this document, and these do not
 overlap the intervals covering `salty`.
+
+[[interval-max_gaps-all-rule]]
+===== max_gaps in `all_of` ordered and unordered rule
+
+The following `intervals` search returns documents containing `my
+favorite food` without any gap, followed by `cold porridge` that
+can have at most 4 tokens between "cold" and "porridge". These
+two inner intervals when combined in the outer `all_of` interval,
+must have at most 1 gap between each other.
+
+Because the `all_of` rule has `ordered` set to `true`, the inner
+intervals are expected to be in the provided order. Thus,
+this search would match a `my_text` value of `my favorite food is cold
+porridge` but not `when it's cold my favorite food is porridge`.
+
+[source,console]
+--------------------------------------------------
+POST _search
+{
+  "query": {
+    "intervals" : {
+      "my_text" : {
+        "all_of" : {
+          "ordered" : true,     <1>
+          "max_gaps": 1,
+          "intervals" : [
+            {
+              "match" : {
+                "query" : "my favorite food",
+                "max_gaps" : 0,
+                "ordered" : true
+              }
+            },
+            {
+              "match" : {
+                "query" : "cold porridge",
+                "max_gaps" : 4,
+                "ordered" : true
+              }
+            }
+          ]
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+<1> The `ordered` parameter is set to `true`, so intervals must appear in the order specified.
+
+
+Below is the same query, but with `ordered` set to `false`. This means that
+intervals can appear in any order, even overlap with each other.
+Thus, this search would match a `my_text` value of `my favorite food is cold
+porridge`, as well as `when it's cold my favorite food is porridge`.
+In `when it's cold my favorite food is porridge`, `cold .... porridge` interval
+overlaps with `my favorite food` interval.
+
+[source,console]
+--------------------------------------------------
+POST _search
+{
+  "query": {
+    "intervals" : {
+      "my_text" : {
+        "all_of" : {
+          "ordered" : false, <1>
+          "max_gaps": 1,
+          "intervals" : [
+            {
+              "match" : {
+                "query" : "my favorite food",
+                "max_gaps" : 0,
+                "ordered" : true
+              }
+            },
+            {
+              "match" : {
+                "query" : "cold porridge",
+                "max_gaps" : 4,
+                "ordered" : true
+              }
+            }
+          ]
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+<1> The `ordered` parameter is set to `true`, so intervals can appear in any order,
+even overlap with each other.

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/20_knn_retriever.yml

@@ -82,7 +82,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -100,7 +100,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Get rescoring scores - hit ordering may change depending on how things are distributed
   - match: { hits.total: 3 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/210_knn_search_profile.yml

@@ -106,9 +106,9 @@ setup:
             k: 3
             num_candidates: 3
             "rescore_vector":
-              "num_candidates_factor": 2.0
+              "oversample": 2.0
 
-  # We expect the knn search ops + rescoring num_cnaidates (for rescoring) per shard
+  # We expect the knn search ops + rescoring k * oversample (for rescoring) per shard
   - match: { profile.shards.0.dfs.knn.0.vector_operations_count: 6 }
 
   # Search with similarity to check number of operations are propagated correctly
@@ -131,7 +131,7 @@ setup:
             num_candidates: 3
             similarity: 100000
             "rescore_vector":
-              "num_candidates_factor": 2.0
+              "oversample": 2.0
 
-  # We expect the knn search ops + rescoring num_cnaidates (for rescoring) per shard
+  # We expect the knn search ops + rescoring k * oversample (for rescoring) per shard
   - match: { profile.shards.0.dfs.knn.0.vector_operations_count: 6 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml

@@ -558,7 +558,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -598,7 +598,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Compare scores as hit IDs may change depending on how things are distributed
   - match: { hits.total: 3 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml

@@ -115,7 +115,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -140,7 +140,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Get rescoring scores - hit ordering may change depending on how things are distributed
   - match: { hits.total: 3 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml

@@ -378,7 +378,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -398,7 +398,7 @@ setup:
               field: vector
               query_vector: [0.5, 111.3, -13.0, 14.8, -156.0]
               rescore_vector:
-                num_candidates_factor: 1.5
+                oversample: 1.5
 
   # Get rescoring scores - hit ordering may change depending on how things are distributed
   - match: { hits.total: 3 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml

@@ -556,7 +556,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -575,7 +575,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Get rescoring scores - hit ordering may change depending on how things are distributed
   - match: { hits.total: 3 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml

@@ -114,7 +114,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -139,7 +139,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Get rescoring scores - hit ordering may change depending on how things are distributed
   - match: { hits.total: 3 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_flat.yml

@@ -264,7 +264,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -304,7 +304,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Compare scores as hit IDs may change depending on how things are distributed
   - match: { hits.total: 3 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml

@@ -352,7 +352,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -371,7 +371,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Get rescoring scores - hit ordering may change depending on how things are distributed
   - match: { hits.total: 3 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml

@@ -269,7 +269,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -288,7 +288,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Get rescoring scores - hit ordering may change depending on how things are distributed
   - match: { hits.total: 3 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml

@@ -414,7 +414,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -454,7 +454,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Compare scores as hit IDs may change depending on how things are distributed
   - match: { hits.total: 3 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit_flat.yml

@@ -230,7 +230,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -270,7 +270,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Compare scores as hit IDs may change depending on how things are distributed
   - match: { hits.total: 3 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_byte.yml

@@ -263,7 +263,7 @@ setup:
       capabilities:
         - method: GET
           path: /_search
-          capabilities: [knn_quantized_vector_rescore]
+          capabilities: [knn_quantized_vector_rescore_oversample]
   - skip:
       features: "headers"
 
@@ -303,7 +303,7 @@ setup:
             k: 3
             num_candidates: 3
             rescore_vector:
-              num_candidates_factor: 1.5
+              oversample: 1.5
 
   # Compare scores as hit IDs may change depending on how things are distributed
   - match: { hits.total: 3 }