diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 98d3ad1eff10b..9763cef8aefeb 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 8.12.0 -lucene = 9.8.0 +lucene = 9.9.0 bundled_jdk_vendor = openjdk bundled_jdk = 21.0.1+12@415e3f918a1f4062a0074a2794853d0d diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc index 47e9071679cc4..3f44db9928434 100644 --- a/docs/Versions.asciidoc +++ b/docs/Versions.asciidoc @@ -1,8 +1,8 @@ include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[] -:lucene_version: 9.8.0 -:lucene_version_path: 9_8_0 +:lucene_version: 9.9.0 +:lucene_version_path: 9_9_0 :jdk: 11.0.2 :jdk_major: 11 :build_type: tar diff --git a/docs/changelog/102032.yaml b/docs/changelog/102032.yaml new file mode 100644 index 0000000000000..40463b9f252b9 --- /dev/null +++ b/docs/changelog/102032.yaml @@ -0,0 +1,5 @@ +pr: 102032 +summary: Add vector_operation_count in profile output for knn searches +area: Vector Search +type: enhancement +issues: [] diff --git a/docs/changelog/102093.yaml b/docs/changelog/102093.yaml new file mode 100644 index 0000000000000..f6922c0d36be6 --- /dev/null +++ b/docs/changelog/102093.yaml @@ -0,0 +1,14 @@ +pr: 102093 +summary: Add byte quantization for float vectors in HNSW +area: Vector Search +type: feature +issues: [] +highlight: + title: Add new `int8_hsnw` index type for int8 quantization for HNSW + body: |- + This commit adds a new index type called `int8_hnsw`. This index will + automatically quantized float32 values into int8 byte values. While + this increases disk usage by 25%, it reduces memory required for + fast HNSW search by 75%. Dramatically reducing the resource overhead + required for dense vector search. + notable: true diff --git a/docs/changelog/102782.yaml b/docs/changelog/102782.yaml new file mode 100644 index 0000000000000..ed0a004765859 --- /dev/null +++ b/docs/changelog/102782.yaml @@ -0,0 +1,5 @@ +pr: 102782 +summary: Upgrade to Lucene 9.9.0 +area: Search +type: upgrade +issues: [] diff --git a/docs/changelog/102877.yaml b/docs/changelog/102877.yaml new file mode 100644 index 0000000000000..da2de19b19a90 --- /dev/null +++ b/docs/changelog/102877.yaml @@ -0,0 +1,5 @@ +pr: 102877 +summary: Add basic telelemetry for the inference feature +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/102891.yaml b/docs/changelog/102891.yaml new file mode 100644 index 0000000000000..c5d5ed8c6758e --- /dev/null +++ b/docs/changelog/102891.yaml @@ -0,0 +1,7 @@ +pr: 102891 +summary: "[Query Rules] Fix bug where combining the same metadata with text/numeric\ + \ values leads to error" +area: Application +type: bug +issues: + - 102827 diff --git a/docs/reference/esql/functions/signature/to_boolean.svg b/docs/reference/esql/functions/signature/to_boolean.svg new file mode 100644 index 0000000000000..43c2aac2bca53 --- /dev/null +++ b/docs/reference/esql/functions/signature/to_boolean.svg @@ -0,0 +1 @@ +TO_BOOLEAN(v) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/to_datetime.svg b/docs/reference/esql/functions/signature/to_datetime.svg new file mode 100644 index 0000000000000..eb9e74248471a --- /dev/null +++ b/docs/reference/esql/functions/signature/to_datetime.svg @@ -0,0 +1 @@ +TO_DATETIME(v) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/to_degrees.svg b/docs/reference/esql/functions/signature/to_degrees.svg new file mode 100644 index 0000000000000..01fe0a4770156 --- /dev/null +++ b/docs/reference/esql/functions/signature/to_degrees.svg @@ -0,0 +1 @@ +TO_DEGREES(v) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/to_double.svg b/docs/reference/esql/functions/signature/to_double.svg new file mode 100644 index 0000000000000..e785e30ce5f81 --- /dev/null +++ b/docs/reference/esql/functions/signature/to_double.svg @@ -0,0 +1 @@ +TO_DOUBLE(v) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/to_integer.svg b/docs/reference/esql/functions/signature/to_integer.svg new file mode 100644 index 0000000000000..beb2e94039e53 --- /dev/null +++ b/docs/reference/esql/functions/signature/to_integer.svg @@ -0,0 +1 @@ +TO_INTEGER(v) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/to_ip.svg b/docs/reference/esql/functions/signature/to_ip.svg index c049964b254f3..c1669c9376c8b 100644 --- a/docs/reference/esql/functions/signature/to_ip.svg +++ b/docs/reference/esql/functions/signature/to_ip.svg @@ -1 +1 @@ -TO_IP(arg1) \ No newline at end of file +TO_IP(v) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/to_long.svg b/docs/reference/esql/functions/signature/to_long.svg new file mode 100644 index 0000000000000..464d4a001cb35 --- /dev/null +++ b/docs/reference/esql/functions/signature/to_long.svg @@ -0,0 +1 @@ +TO_LONG(v) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/to_radians.svg b/docs/reference/esql/functions/signature/to_radians.svg new file mode 100644 index 0000000000000..712431fb32497 --- /dev/null +++ b/docs/reference/esql/functions/signature/to_radians.svg @@ -0,0 +1 @@ +TO_RADIANS(v) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/to_unsigned_long.svg b/docs/reference/esql/functions/signature/to_unsigned_long.svg new file mode 100644 index 0000000000000..da07b3a4c7349 --- /dev/null +++ b/docs/reference/esql/functions/signature/to_unsigned_long.svg @@ -0,0 +1 @@ +TO_UNSIGNED_LONG(v) \ No newline at end of file diff --git a/docs/reference/esql/functions/types/mv_count.asciidoc b/docs/reference/esql/functions/types/mv_count.asciidoc index 21794bcb1b959..440e66d11096e 100644 --- a/docs/reference/esql/functions/types/mv_count.asciidoc +++ b/docs/reference/esql/functions/types/mv_count.asciidoc @@ -2,8 +2,10 @@ |=== v | result boolean | integer +cartesian_point | integer datetime | integer double | integer +geo_point | integer integer | integer ip | integer keyword | integer diff --git a/docs/reference/esql/functions/types/to_boolean.asciidoc b/docs/reference/esql/functions/types/to_boolean.asciidoc new file mode 100644 index 0000000000000..7f543963eb090 --- /dev/null +++ b/docs/reference/esql/functions/types/to_boolean.asciidoc @@ -0,0 +1,11 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +v | result +boolean | boolean +double | boolean +integer | boolean +keyword | boolean +long | boolean +text | boolean +unsigned_long | boolean +|=== diff --git a/docs/reference/esql/functions/types/to_datetime.asciidoc b/docs/reference/esql/functions/types/to_datetime.asciidoc new file mode 100644 index 0000000000000..bbd755f81f4da --- /dev/null +++ b/docs/reference/esql/functions/types/to_datetime.asciidoc @@ -0,0 +1,11 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +v | result +datetime | datetime +double | datetime +integer | datetime +keyword | datetime +long | datetime +text | datetime +unsigned_long | datetime +|=== diff --git a/docs/reference/esql/functions/types/to_degrees.asciidoc b/docs/reference/esql/functions/types/to_degrees.asciidoc new file mode 100644 index 0000000000000..7cb7ca46022c2 --- /dev/null +++ b/docs/reference/esql/functions/types/to_degrees.asciidoc @@ -0,0 +1,8 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +v | result +double | double +integer | double +long | double +unsigned_long | double +|=== diff --git a/docs/reference/esql/functions/types/to_double.asciidoc b/docs/reference/esql/functions/types/to_double.asciidoc new file mode 100644 index 0000000000000..38e8482b77544 --- /dev/null +++ b/docs/reference/esql/functions/types/to_double.asciidoc @@ -0,0 +1,12 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +v | result +boolean | double +datetime | double +double | double +integer | double +keyword | double +long | double +text | double +unsigned_long | double +|=== diff --git a/docs/reference/esql/functions/types/to_integer.asciidoc b/docs/reference/esql/functions/types/to_integer.asciidoc new file mode 100644 index 0000000000000..bcea15b9ec80b --- /dev/null +++ b/docs/reference/esql/functions/types/to_integer.asciidoc @@ -0,0 +1,12 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +v | result +boolean | integer +datetime | integer +double | integer +integer | integer +keyword | integer +long | integer +text | integer +unsigned_long | integer +|=== diff --git a/docs/reference/esql/functions/types/to_ip.asciidoc b/docs/reference/esql/functions/types/to_ip.asciidoc index a21bbf14d87ca..6d7f9338a9aeb 100644 --- a/docs/reference/esql/functions/types/to_ip.asciidoc +++ b/docs/reference/esql/functions/types/to_ip.asciidoc @@ -1,6 +1,7 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== -arg1 | result +v | result ip | ip keyword | ip +text | ip |=== diff --git a/docs/reference/esql/functions/types/to_long.asciidoc b/docs/reference/esql/functions/types/to_long.asciidoc new file mode 100644 index 0000000000000..5c063739fc5b1 --- /dev/null +++ b/docs/reference/esql/functions/types/to_long.asciidoc @@ -0,0 +1,14 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +v | result +boolean | long +cartesian_point | long +datetime | long +double | long +geo_point | long +integer | long +keyword | long +long | long +text | long +unsigned_long | long +|=== diff --git a/docs/reference/esql/functions/types/to_radians.asciidoc b/docs/reference/esql/functions/types/to_radians.asciidoc new file mode 100644 index 0000000000000..7cb7ca46022c2 --- /dev/null +++ b/docs/reference/esql/functions/types/to_radians.asciidoc @@ -0,0 +1,8 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +v | result +double | double +integer | double +long | double +unsigned_long | double +|=== diff --git a/docs/reference/esql/functions/types/to_string.asciidoc b/docs/reference/esql/functions/types/to_string.asciidoc index b8fcd4477aa70..4de4af735b07f 100644 --- a/docs/reference/esql/functions/types/to_string.asciidoc +++ b/docs/reference/esql/functions/types/to_string.asciidoc @@ -2,8 +2,10 @@ |=== v | result boolean | keyword +cartesian_point | keyword datetime | keyword double | keyword +geo_point | keyword integer | keyword ip | keyword keyword | keyword diff --git a/docs/reference/esql/functions/types/to_unsigned_long.asciidoc b/docs/reference/esql/functions/types/to_unsigned_long.asciidoc new file mode 100644 index 0000000000000..76d9cf44f4dd2 --- /dev/null +++ b/docs/reference/esql/functions/types/to_unsigned_long.asciidoc @@ -0,0 +1,12 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +v | result +boolean | unsigned_long +datetime | unsigned_long +double | unsigned_long +integer | unsigned_long +keyword | unsigned_long +long | unsigned_long +text | unsigned_long +unsigned_long | unsigned_long +|=== diff --git a/docs/reference/how-to/knn-search.asciidoc b/docs/reference/how-to/knn-search.asciidoc index 330847f5806de..066008ce26110 100644 --- a/docs/reference/how-to/knn-search.asciidoc +++ b/docs/reference/how-to/knn-search.asciidoc @@ -52,7 +52,12 @@ of datasets and configurations that we use for our nightly benchmarks. include::search-speed.asciidoc[tag=warm-fs-cache] The following file extensions are used for the approximate kNN search: -"vec" (for vector values), "vex" (for HNSW graph), "vem" (for metadata). ++ +-- +* `vec` and `veq` for vector values +* `vex` for HNSW graph +* `vem`, `vemf`, and `vemq` for metadata +-- [discrete] === Reduce vector dimensionality @@ -66,6 +71,14 @@ reduction techniques like PCA. When experimenting with different approaches, it's important to measure the impact on relevance to ensure the search quality is still acceptable. +[discrete] +=== Reduce vector memory foot-print + +The default <> is `float`. But this can be +automatically quantized during index time through <>. Quantization will +reduce the required memory by 4x, but it will also reduce the precision of the vectors. For `float` vectors with +`dim` greater than or equal to `384`, using a <> index is highly recommended. + [discrete] === Exclude vector fields from `_source` diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc index 446e6c8ea4c43..a2ab44a173a62 100644 --- a/docs/reference/mapping/types/dense-vector.asciidoc +++ b/docs/reference/mapping/types/dense-vector.asciidoc @@ -111,6 +111,36 @@ PUT my-index-2 efficient kNN search. Like most kNN algorithms, HNSW is an approximate method that sacrifices result accuracy for improved speed. +[[dense-vector-quantization]] +==== Automatically quantize vectors for kNN search + +The `dense_vector` type supports quantization to reduce the memory footprint required when <> `float` vectors. +Currently the only quantization method supported is `int8` and provided vectors `element_type` must be `float`. To use +a quantized index, you can set your index type to `int8_hnsw`. + +When using the `int8_hnsw` index, each of the `float` vectors' dimensions are quantized to 1-byte integers. This can +reduce the memory footprint by as much as 75% at the cost of some accuracy. However, the disk usage can increase by +25% due to the overhead of storing the quantized and raw vectors. + +[source,console] +-------------------------------------------------- +PUT my-byte-quantized-index +{ + "mappings": { + "properties": { + "my_vector": { + "type": "dense_vector", + "dims": 3, + "index": true, + "index_options": { + "type": "int8_hnsw" + } + } + } + } +} +-------------------------------------------------- + [role="child_attributes"] [[dense-vector-params]] ==== Parameters for dense vector fields @@ -198,8 +228,7 @@ a distinct set of options. An optional section that configures the kNN indexing algorithm. The HNSW algorithm has two internal parameters that influence how the data structure is built. These can be adjusted to improve the accuracy of results, at the -expense of slower indexing speed. When `index_options` is provided, all of its -properties must be defined. +expense of slower indexing speed. + ^*^ This parameter can only be specified when `index` is `true`. + @@ -209,17 +238,25 @@ properties must be defined. ==== `type`::: (Required, string) -The type of kNN algorithm to use. Currently only `hnsw` is supported. +The type of kNN algorithm to use. Can be either `hnsw` or `int8_hnsw`. `m`::: -(Required, integer) +(Optional, integer) The number of neighbors each node will be connected to in the HNSW graph. Defaults to `16`. `ef_construction`::: -(Required, integer) +(Optional, integer) The number of candidates to track while assembling the list of nearest neighbors for each new node. Defaults to `100`. + +`confidence_interval`::: +(Optional, float) +Only applicable to `int8_hnsw` index types. The confidence interval to use when quantizing the vectors, +can be any value between and including `0.90` and `1.0`. This value restricts the values used when calculating +the quantization thresholds. For example, a value of `0.95` will only use the middle 95% of the values when +calculating the quantization thresholds (e.g. the highest and lowest 2.5% of values will be ignored). +Defaults to `1/(dims + 1)`. ==== [[dense-vector-synthetic-source]] diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc index 959a798378fc6..e2529de75f0e7 100644 --- a/docs/reference/rest-api/usage.asciidoc +++ b/docs/reference/rest-api/usage.asciidoc @@ -197,6 +197,11 @@ GET /_xpack/usage }, "node_count" : 1 }, + "inference": { + "available" : true, + "enabled" : true, + "models" : [] + }, "logstash" : { "available" : true, "enabled" : true diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc index 52dfb91475c53..5b63929934770 100644 --- a/docs/reference/search/profile.asciidoc +++ b/docs/reference/search/profile.asciidoc @@ -1272,6 +1272,7 @@ One of the `dfs.knn` sections for a shard looks like the following: "dfs" : { "knn" : [ { + "vector_operations_count" : 4, "query" : [ { "type" : "DocAndScoreQuery", @@ -1321,7 +1322,7 @@ In the `dfs.knn` portion of the response we can see the output the of timings for <>, <>, and <>. Unlike many other queries, kNN search does the bulk of the work during the query rewrite. This means -`rewrite_time` represents the time spent on kNN search. +`rewrite_time` represents the time spent on kNN search. The attribute `vector_operations_count` represents the overall count of vector operations performed during the kNN search. [[profiling-considerations]] ===== Profiling Considerations diff --git a/docs/reference/search/search-your-data/knn-search.asciidoc b/docs/reference/search/search-your-data/knn-search.asciidoc index c39719f1a3b61..ff64535c705d9 100644 --- a/docs/reference/search/search-your-data/knn-search.asciidoc +++ b/docs/reference/search/search-your-data/knn-search.asciidoc @@ -242,6 +242,114 @@ POST byte-image-index/_search // TEST[s/"k": 10/"k": 3/] // TEST[s/"num_candidates": 100/"num_candidates": 3/] +[discrete] +[[knn-search-quantized-example]] +==== Byte quantized kNN search + +If you want to provide `float` vectors, but want the memory savings of `byte` vectors, you can use the +<> feature. Quantization allows you to provide `float` vectors, but +internally they are indexed as `byte` vectors. Additionally, the original `float` vectors are still retained +in the index. + +To use quantization, you can use the index type `int8_hnsw` object in the `dense_vector` mapping. + +[source,console] +---- +PUT quantized-image-index +{ + "mappings": { + "properties": { + "image-vector": { + "type": "dense_vector", + "element_type": "float", + "dims": 2, + "index": true, + "index_options": { + "type": "int8_hnsw" + } + }, + "title": { + "type": "text" + } + } + } +} +---- +// TEST[continued] + +. Index your `float` vectors. ++ +[source,console] +---- +POST quantized-image-index/_bulk?refresh=true +{ "index": { "_id": "1" } } +{ "image-vector": [0.1, -2], "title": "moose family" } +{ "index": { "_id": "2" } } +{ "image-vector": [0.75, -1], "title": "alpine lake" } +{ "index": { "_id": "3" } } +{ "image-vector": [1.2, 0.1], "title": "full moon" } +---- +//TEST[continued] + +. Run the search using the <>. When searching, the `float` vector is +automatically quantized to a `byte` vector. ++ +[source,console] +---- +POST quantized-image-index/_search +{ + "knn": { + "field": "image-vector", + "query_vector": [0.1, -2], + "k": 10, + "num_candidates": 100 + }, + "fields": [ "title" ] +} +---- +// TEST[continued] +// TEST[s/"k": 10/"k": 3/] +// TEST[s/"num_candidates": 100/"num_candidates": 3/] + +Since the original `float` vectors are still retained in the index, you can optionally use them for re-scoring. Meaning, +you can search over all the vectors quickly using the `int8_hnsw` index and then rescore only the top `k` results. This +provides the best of both worlds, fast search and accurate scoring. + +[source,console] +---- +POST quantized-image-index/_search +{ + "knn": { + "field": "image-vector", + "query_vector": [0.1, -2], + "k": 15, + "num_candidates": 100 + }, + "fields": [ "title" ], + "rescore": { + "window_size": 10, + "query": { + "rescore_query": { + "script_score": { + "query": { + "match_all": {} + }, + "script": { + "source": "cosineSimilarity(params.query_vector, 'image-vector') + 1.0", + "params": { + "query_vector": [0.1, -2] + } + } + } + } + } + } +} +---- +// TEST[continued] +// TEST[s/"k": 15/"k": 3/] +// TEST[s/"num_candidates": 100/"num_candidates": 3/] + [discrete] [[knn-search-filter-example]] ==== Filtered kNN search @@ -903,7 +1011,7 @@ the global top `k` matches across shards. You cannot set the To run an exact kNN search, use a `script_score` query with a vector function. . Explicitly map one or more `dense_vector` fields. If you don't intend to use -the field for approximate kNN, set the `index` mapping option to `false`. This +the field for approximate kNN, set the `index` mapping option to `false`. This can significantly improve indexing speed. + [source,console] diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 8c5022cea289d..9d383c426cb74 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2659,124 +2659,124 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQuery.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQuery.java index 8cf1da77b545d..dc51afe5d420d 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQuery.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQuery.java @@ -236,7 +236,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo for (Term term : terms) { TermStates ts = termStates.computeIfAbsent(term, t -> { try { - return TermStates.build(searcher.getTopReaderContext(), t, scoreMode.needsScores()); + return TermStates.build(searcher, t, scoreMode.needsScores()); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java index 603b19623a0e7..222f0f05d548d 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java @@ -35,6 +35,8 @@ import java.util.Collections; import java.util.List; +import static org.hamcrest.Matchers.containsString; + public class ScaledFloatFieldTypeTests extends FieldTypeTestCase { public void testTermQuery() { @@ -136,35 +138,35 @@ public void testRangeQuery() throws IOException { public void testRoundsUpperBoundCorrectly() { ScaledFloatFieldMapper.ScaledFloatFieldType ft = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float", 100); Query scaledFloatQ = ft.rangeQuery(null, 0.1, true, false, MOCK_CONTEXT); - assertEquals("scaled_float:[-9223372036854775808 TO 9]", scaledFloatQ.toString()); + assertThat(scaledFloatQ.toString(), containsString("scaled_float:[-9223372036854775808 TO 9]")); scaledFloatQ = ft.rangeQuery(null, 0.1, true, true, MOCK_CONTEXT); - assertEquals("scaled_float:[-9223372036854775808 TO 10]", scaledFloatQ.toString()); + assertThat(scaledFloatQ.toString(), containsString("scaled_float:[-9223372036854775808 TO 10]")); scaledFloatQ = ft.rangeQuery(null, 0.095, true, false, MOCK_CONTEXT); - assertEquals("scaled_float:[-9223372036854775808 TO 9]", scaledFloatQ.toString()); + assertThat(scaledFloatQ.toString(), containsString("scaled_float:[-9223372036854775808 TO 9]")); scaledFloatQ = ft.rangeQuery(null, 0.095, true, true, MOCK_CONTEXT); - assertEquals("scaled_float:[-9223372036854775808 TO 9]", scaledFloatQ.toString()); + assertThat(scaledFloatQ.toString(), containsString("scaled_float:[-9223372036854775808 TO 9]")); scaledFloatQ = ft.rangeQuery(null, 0.105, true, false, MOCK_CONTEXT); - assertEquals("scaled_float:[-9223372036854775808 TO 10]", scaledFloatQ.toString()); + assertThat(scaledFloatQ.toString(), containsString("scaled_float:[-9223372036854775808 TO 10]")); scaledFloatQ = ft.rangeQuery(null, 0.105, true, true, MOCK_CONTEXT); - assertEquals("scaled_float:[-9223372036854775808 TO 10]", scaledFloatQ.toString()); + assertThat(scaledFloatQ.toString(), containsString("scaled_float:[-9223372036854775808 TO 10]")); scaledFloatQ = ft.rangeQuery(null, 79.99, true, true, MOCK_CONTEXT); - assertEquals("scaled_float:[-9223372036854775808 TO 7999]", scaledFloatQ.toString()); + assertThat(scaledFloatQ.toString(), containsString("scaled_float:[-9223372036854775808 TO 7999]")); } public void testRoundsLowerBoundCorrectly() { ScaledFloatFieldMapper.ScaledFloatFieldType ft = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float", 100); Query scaledFloatQ = ft.rangeQuery(-0.1, null, false, true, MOCK_CONTEXT); - assertEquals("scaled_float:[-9 TO 9223372036854775807]", scaledFloatQ.toString()); + assertThat(scaledFloatQ.toString(), containsString("scaled_float:[-9 TO 9223372036854775807]")); scaledFloatQ = ft.rangeQuery(-0.1, null, true, true, MOCK_CONTEXT); - assertEquals("scaled_float:[-10 TO 9223372036854775807]", scaledFloatQ.toString()); + assertThat(scaledFloatQ.toString(), containsString("scaled_float:[-10 TO 9223372036854775807]")); scaledFloatQ = ft.rangeQuery(-0.095, null, false, true, MOCK_CONTEXT); - assertEquals("scaled_float:[-9 TO 9223372036854775807]", scaledFloatQ.toString()); + assertThat(scaledFloatQ.toString(), containsString("scaled_float:[-9 TO 9223372036854775807]")); scaledFloatQ = ft.rangeQuery(-0.095, null, true, true, MOCK_CONTEXT); - assertEquals("scaled_float:[-9 TO 9223372036854775807]", scaledFloatQ.toString()); + assertThat(scaledFloatQ.toString(), containsString("scaled_float:[-9 TO 9223372036854775807]")); scaledFloatQ = ft.rangeQuery(-0.105, null, false, true, MOCK_CONTEXT); - assertEquals("scaled_float:[-10 TO 9223372036854775807]", scaledFloatQ.toString()); + assertThat(scaledFloatQ.toString(), containsString("scaled_float:[-10 TO 9223372036854775807]")); scaledFloatQ = ft.rangeQuery(-0.105, null, true, true, MOCK_CONTEXT); - assertEquals("scaled_float:[-10 TO 9223372036854775807]", scaledFloatQ.toString()); + assertThat(scaledFloatQ.toString(), containsString("scaled_float:[-10 TO 9223372036854775807]")); } public void testValueForSearch() { diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index 5a445a1524da5..c76364f48c081 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -75,7 +75,7 @@ import java.util.stream.StreamSupport; import static org.elasticsearch.repositories.RepositoriesModule.METRIC_REQUESTS_COUNT; -import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose; +import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomNonDataPurpose; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.allOf; @@ -85,8 +85,6 @@ import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.startsWith; @@ -271,8 +269,12 @@ public void testMetrics() throws Exception { final List metrics = Measurement.combine(plugins.get(0).getLongCounterMeasurement(METRIC_REQUESTS_COUNT)); assertThat( - statsCollectors.size(), - equalTo(metrics.stream().map(m -> m.attributes().get("operation")).collect(Collectors.toSet()).size()) + statsCollectors.keySet().stream().map(S3BlobStore.StatsKey::operation).collect(Collectors.toSet()), + equalTo( + metrics.stream() + .map(m -> S3BlobStore.Operation.parse((String) m.attributes().get("operation"))) + .collect(Collectors.toSet()) + ) ); metrics.forEach(metric -> { assertThat( @@ -303,23 +305,24 @@ public void testRequestStatsWithOperationPurposes() throws IOException { final String repoName = createRepository(randomRepositoryName()); final RepositoriesService repositoriesService = internalCluster().getCurrentMasterNodeInstance(RepositoriesService.class); final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(repoName); - final BlobStore blobStore = repository.blobStore(); - assertThat(blobStore, instanceOf(BlobStoreWrapper.class)); - final BlobStore delegateBlobStore = ((BlobStoreWrapper) blobStore).delegate(); - assertThat(delegateBlobStore, instanceOf(S3BlobStore.class)); - final S3BlobStore.StatsCollectors statsCollectors = ((S3BlobStore) delegateBlobStore).getStatsCollectors(); + final BlobStoreWrapper blobStore = asInstanceOf(BlobStoreWrapper.class, repository.blobStore()); + final S3BlobStore delegateBlobStore = asInstanceOf(S3BlobStore.class, blobStore.delegate()); + final S3BlobStore.StatsCollectors statsCollectors = delegateBlobStore.getStatsCollectors(); - // Initial stats are collected with the default operation purpose + // Initial stats are collected for repository verification, which counts as SNAPSHOT_METADATA final Set allOperations = EnumSet.allOf(S3BlobStore.Operation.class) .stream() .map(S3BlobStore.Operation::getKey) .collect(Collectors.toUnmodifiableSet()); - statsCollectors.collectors.keySet().forEach(statsKey -> assertThat(statsKey.purpose(), is(OperationPurpose.SNAPSHOT))); + assertThat( + statsCollectors.collectors.keySet().stream().map(S3BlobStore.StatsKey::purpose).collect(Collectors.toUnmodifiableSet()), + equalTo(Set.of(OperationPurpose.SNAPSHOT_METADATA)) + ); final Map initialStats = blobStore.stats(); assertThat(initialStats.keySet(), equalTo(allOperations)); // Collect more stats with an operation purpose other than the default - final OperationPurpose purpose = randomValueOtherThan(OperationPurpose.SNAPSHOT, BlobStoreTestUtil::randomPurpose); + final OperationPurpose purpose = randomValueOtherThan(OperationPurpose.SNAPSHOT_METADATA, BlobStoreTestUtil::randomPurpose); final BlobPath blobPath = repository.basePath().add(randomAlphaOfLength(10)); final BlobContainer blobContainer = blobStore.blobContainer(blobPath); final BytesArray whatToWrite = new BytesArray(randomByteArrayOfLength(randomIntBetween(100, 1000))); @@ -332,7 +335,7 @@ public void testRequestStatsWithOperationPurposes() throws IOException { // Internal stats collection is fine-grained and records different purposes assertThat( statsCollectors.collectors.keySet().stream().map(S3BlobStore.StatsKey::purpose).collect(Collectors.toUnmodifiableSet()), - equalTo(Set.of(OperationPurpose.SNAPSHOT, purpose)) + equalTo(Set.of(OperationPurpose.SNAPSHOT_METADATA, purpose)) ); // The stats report aggregates over different purposes final Map newStats = blobStore.stats(); @@ -341,7 +344,7 @@ public void testRequestStatsWithOperationPurposes() throws IOException { final Set operationsSeenForTheNewPurpose = statsCollectors.collectors.keySet() .stream() - .filter(sk -> sk.purpose() != OperationPurpose.SNAPSHOT) + .filter(sk -> sk.purpose() != OperationPurpose.SNAPSHOT_METADATA) .map(sk -> sk.operation().getKey()) .collect(Collectors.toUnmodifiableSet()); @@ -396,7 +399,7 @@ public void testEnforcedCooldownPeriod() throws IOException { () -> repository.blobStore() .blobContainer(repository.basePath()) .writeBlobAtomic( - randomPurpose(), + randomNonDataPurpose(), BlobStoreRepository.INDEX_FILE_PREFIX + modifiedRepositoryData.getGenId(), serialized, true diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java index 87b3c17bfd91c..93b8ef7e57389 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java @@ -129,6 +129,7 @@ public long readBlobPreferredLength() { @Override public void writeBlob(OperationPurpose purpose, String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) throws IOException { + assert BlobContainer.assertPurposeConsistency(purpose, blobName); assert inputStream.markSupported() : "No mark support on inputStream breaks the S3 SDK's ability to retry requests"; SocketAccess.doPrivilegedIOException(() -> { if (blobSize <= getLargeBlobThresholdInBytes()) { @@ -148,6 +149,7 @@ public void writeMetadataBlob( boolean atomic, CheckedConsumer writer ) throws IOException { + assert purpose != OperationPurpose.SNAPSHOT_DATA && BlobContainer.assertPurposeConsistency(purpose, blobName) : purpose; final String absoluteBlobKey = buildKey(blobName); try ( AmazonS3Reference clientReference = blobStore.clientReference(); @@ -273,6 +275,7 @@ long getLargeBlobThresholdInBytes() { @Override public void writeBlobAtomic(OperationPurpose purpose, String blobName, BytesReference bytes, boolean failIfAlreadyExists) throws IOException { + assert BlobContainer.assertPurposeConsistency(purpose, blobName); writeBlob(purpose, blobName, bytes, failIfAlreadyExists); } diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java index 9ed68976aac8a..b4b136338923f 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -57,6 +57,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomNonDataPurpose; import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose; import static org.elasticsearch.repositories.s3.S3ClientSettings.DISABLE_CHUNKED_ENCODING; import static org.elasticsearch.repositories.s3.S3ClientSettings.ENDPOINT_SETTING; @@ -446,7 +447,7 @@ public void testWriteLargeBlobStreaming() throws Exception { } }); - blobContainer.writeMetadataBlob(randomPurpose(), "write_large_blob_streaming", false, randomBoolean(), out -> { + blobContainer.writeMetadataBlob(randomNonDataPurpose(), "write_large_blob_streaming", false, randomBoolean(), out -> { final byte[] buffer = new byte[16 * 1024]; long outstanding = blobSize; while (outstanding > 0) { diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java index 038af3c2357f9..d662003530c22 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java @@ -12,7 +12,7 @@ import org.apache.lucene.analysis.ja.JapaneseTokenizer; import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode; import org.apache.lucene.analysis.ja.dict.UserDictionary; -import org.apache.lucene.analysis.ja.util.CSVUtil; +import org.apache.lucene.analysis.util.CSVUtil; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_error.json b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_error.json new file mode 100644 index 0000000000000..5d82a3729b501 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_error.json @@ -0,0 +1,39 @@ +{ + "connector.update_error": { + "documentation": { + "url": "https://www.elastic.co/guide/en/enterprise-search/current/connectors.html", + "description": "Updates the error field in the connector document." + }, + "stability": "experimental", + "visibility": "feature_flag", + "feature_flag": "es.connector_api_feature_flag_enabled", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_connector/{connector_id}/_error", + "methods": [ + "PUT" + ], + "parts": { + "connector_id": { + "type": "string", + "description": "The unique identifier of the connector to be updated." + } + } + } + ] + }, + "body": { + "description": "An object containing the connector's error.", + "required": true + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/connector_sync_job.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/connector_sync_job.get.json new file mode 100644 index 0000000000000..6eb461ad62128 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/connector_sync_job.get.json @@ -0,0 +1,32 @@ +{ + "connector_sync_job.get": { + "documentation": { + "url": "https://www.elastic.co/guide/en/enterprise-search/current/connectors.html", + "description": "Returns the details about a connector sync job." + }, + "stability": "experimental", + "visibility": "feature_flag", + "feature_flag": "es.connector_api_feature_flag_enabled", + "headers": { + "accept": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_connector/_sync_job/{connector_sync_job_id}", + "methods": [ + "GET" + ], + "parts": { + "connector_sync_job_id": { + "type": "string", + "description": "The unique identifier of the connector sync job to be returned." + } + } + } + ] + } + } +} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml new file mode 100644 index 0000000000000..948a6e04a128b --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml @@ -0,0 +1,373 @@ +setup: + - skip: + version: ' - 8.11.99' + reason: 'kNN float to byte quantization added in 8.12' + - do: + indices.create: + index: hnsw_byte_quantized + body: + mappings: + properties: + name: + type: keyword + vector: + type: dense_vector + dims: 5 + index: true + similarity: l2_norm + index_options: + type: int8_hnsw + another_vector: + type: dense_vector + dims: 5 + index: true + similarity: l2_norm + index_options: + type: int8_hnsw + + - do: + index: + index: hnsw_byte_quantized + id: "1" + body: + name: cow.jpg + vector: [230.0, 300.33, -34.8988, 15.555, -200.0] + another_vector: [130.0, 115.0, -1.02, 15.555, -100.0] + + - do: + index: + index: hnsw_byte_quantized + id: "2" + body: + name: moose.jpg + vector: [-0.5, 100.0, -13, 14.8, -156.0] + another_vector: [-0.5, 50.0, -1, 1, 120] + + - do: + index: + index: hnsw_byte_quantized + id: "3" + body: + name: rabbit.jpg + vector: [0.5, 111.3, -13.0, 14.8, -156.0] + another_vector: [-0.5, 11.0, 0, 12, 111.0] + + - do: + indices.refresh: {} + +--- +"kNN search only": + - do: + search: + index: hnsw_byte_quantized + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 2 + num_candidates: 3 + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0.fields.name.0: "moose.jpg"} + + - match: {hits.hits.1._id: "3"} + - match: {hits.hits.1.fields.name.0: "rabbit.jpg"} +--- +"kNN multi-field search only": + - do: + search: + index: hnsw_byte_quantized + body: + fields: [ "name" ] + knn: + - {field: vector, query_vector: [-0.5, 90.0, -10, 14.8, -156.0], k: 2, num_candidates: 3} + - {field: another_vector, query_vector: [-0.5, 11.0, 0, 12, 111.0], k: 2, num_candidates: 3} + + - match: {hits.hits.0._id: "3"} + - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + + - match: {hits.hits.1._id: "2"} + - match: {hits.hits.1.fields.name.0: "moose.jpg"} +--- +"kNN search plus query": + - do: + search: + index: hnsw_byte_quantized + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 2 + num_candidates: 3 + query: + term: + name: cow.jpg + + - match: {hits.hits.0._id: "1"} + - match: {hits.hits.0.fields.name.0: "cow.jpg"} + + - match: {hits.hits.1._id: "2"} + - match: {hits.hits.1.fields.name.0: "moose.jpg"} + + - match: {hits.hits.2._id: "3"} + - match: {hits.hits.2.fields.name.0: "rabbit.jpg"} +--- +"kNN multi-field search with query": + - do: + search: + index: hnsw_byte_quantized + body: + fields: [ "name" ] + knn: + - {field: vector, query_vector: [-0.5, 90.0, -10, 14.8, -156.0], k: 2, num_candidates: 3} + - {field: another_vector, query_vector: [-0.5, 11.0, 0, 12, 111.0], k: 2, num_candidates: 3} + query: + term: + name: cow.jpg + + - match: {hits.hits.0._id: "3"} + - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + + - match: {hits.hits.1._id: "1"} + - match: {hits.hits.1.fields.name.0: "cow.jpg"} + + - match: {hits.hits.2._id: "2"} + - match: {hits.hits.2.fields.name.0: "moose.jpg"} +--- +"kNN search with filter": + - do: + search: + index: hnsw_byte_quantized + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 2 + num_candidates: 3 + filter: + term: + name: "rabbit.jpg" + + - match: {hits.total.value: 1} + - match: {hits.hits.0._id: "3"} + - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + + - do: + search: + index: hnsw_byte_quantized + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 2 + num_candidates: 3 + filter: + - term: + name: "rabbit.jpg" + - term: + _id: 2 + + - match: {hits.total.value: 0} + +--- +"KNN Vector similarity search only": + - do: + search: + index: hnsw_byte_quantized + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + similarity: 10.3 + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + + - length: {hits.hits: 1} + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0.fields.name.0: "moose.jpg"} +--- +"Vector similarity with filter only": + - do: + search: + index: hnsw_byte_quantized + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + similarity: 11 + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + filter: {"term": {"name": "moose.jpg"}} + + - length: {hits.hits: 1} + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0.fields.name.0: "moose.jpg"} + + - do: + search: + index: hnsw_byte_quantized + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + similarity: 110 + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + filter: {"term": {"name": "cow.jpg"}} + + - length: {hits.hits: 0} +--- +"Knn search with mip": + - do: + indices.create: + index: mip + body: + mappings: + properties: + name: + type: keyword + vector: + type: dense_vector + dims: 5 + index: true + similarity: max_inner_product + index_options: + type: int8_hnsw + + - do: + index: + index: mip + id: "1" + body: + name: cow.jpg + vector: [230.0, 300.33, -34.8988, 15.555, -200.0] + + - do: + index: + index: mip + id: "2" + body: + name: moose.jpg + vector: [-0.5, 10.0, -13, 14.8, 15.0] + + - do: + index: + index: mip + id: "3" + body: + name: rabbit.jpg + vector: [0.5, 111.3, -13.0, 14.8, -156.0] + + # We force merge into a single segment to make sure scores are more uniform + # Each segment can have a different quantization error, which can affect scores and mip is especially sensitive to this + - do: + indices.forcemerge: + index: mip + max_num_segments: 1 + + - do: + indices.refresh: {} + + - do: + search: + index: mip + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + + + - length: {hits.hits: 3} + - match: {hits.hits.0._id: "1"} + - match: {hits.hits.1._id: "3"} + - match: {hits.hits.2._id: "2"} + + - do: + search: + index: mip + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + filter: { "term": { "name": "moose.jpg" } } + + + + - length: {hits.hits: 1} + - match: {hits.hits.0._id: "2"} +--- +"Cosine similarity with indexed vector": + - skip: + features: "headers" + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "cosineSimilarity(params.query_vector, 'vector')" + params: + query_vector: [0.5, 111.3, -13.0, 14.8, -156.0] + + - match: {hits.total: 3} + + - match: {hits.hits.0._id: "3"} + - gte: {hits.hits.0._score: 0.999} + - lte: {hits.hits.0._score: 1.001} + + - match: {hits.hits.1._id: "2"} + - gte: {hits.hits.1._score: 0.998} + - lte: {hits.hits.1._score: 1.0} + + - match: {hits.hits.2._id: "1"} + - gte: {hits.hits.2._score: 0.78} + - lte: {hits.hits.2._score: 0.791} +--- +"Test bad quantization parameters": + - do: + catch: bad_request + indices.create: + index: bad_hnsw_quantized + body: + mappings: + properties: + vector: + type: dense_vector + dims: 5 + element_type: byte + index: true + index_options: + type: int8_hnsw + + - do: + catch: bad_request + indices.create: + index: bad_hnsw_quantized + body: + mappings: + properties: + vector: + type: dense_vector + dims: 5 + index: false + index_options: + type: int8_hnsw diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml index 38212ba59a51e..0ead7b87f8acf 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml @@ -229,6 +229,72 @@ dfs knn vector profiling: - match: { profile.shards.0.dfs.knn.0.collector.0.reason: "search_top_hits" } - gt: { profile.shards.0.dfs.knn.0.collector.0.time_in_nanos: 0 } +--- +dfs knn vector profiling with vector_operations_count: + - skip: + version: ' - 8.11.99' + reason: vector_operations_count in dfs profiling added in 8.12.0 + + - do: + indices.create: + index: images + body: + settings: + index.number_of_shards: 1 + mappings: + properties: + image: + type: "dense_vector" + dims: 3 + index: true + similarity: "l2_norm" + + - do: + index: + index: images + id: "1" + refresh: true + body: + image: [1, 5, -20] + + - do: + search: + index: images + body: + profile: true + knn: + field: "image" + query_vector: [-5, 9, -12] + k: 1 + num_candidates: 100 + + - match: { hits.total.value: 1 } + - match: { profile.shards.0.dfs.knn.0.query.0.type: "DocAndScoreQuery" } + - match: { profile.shards.0.dfs.knn.0.query.0.description: "DocAndScore[100]" } + - match: { profile.shards.0.dfs.knn.0.vector_operations_count: 1 } + - gt: { profile.shards.0.dfs.knn.0.query.0.time_in_nanos: 0 } + - match: { profile.shards.0.dfs.knn.0.query.0.breakdown.set_min_competitive_score_count: 0 } + - match: { profile.shards.0.dfs.knn.0.query.0.breakdown.set_min_competitive_score: 0 } + - match: { profile.shards.0.dfs.knn.0.query.0.breakdown.match_count: 0 } + - match: { profile.shards.0.dfs.knn.0.query.0.breakdown.match: 0 } + - match: { profile.shards.0.dfs.knn.0.query.0.breakdown.shallow_advance_count: 0 } + - match: { profile.shards.0.dfs.knn.0.query.0.breakdown.shallow_advance: 0 } + - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.next_doc_count: 0 } + - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.next_doc: 0 } + - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.score_count: 0 } + - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.score: 0 } + - match: { profile.shards.0.dfs.knn.0.query.0.breakdown.compute_max_score_count: 0 } + - match: { profile.shards.0.dfs.knn.0.query.0.breakdown.compute_max_score: 0 } + - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.build_scorer_count: 0 } + - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.build_scorer: 0 } + - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.create_weight: 0 } + - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.create_weight_count: 0 } + - gt: { profile.shards.0.dfs.knn.0.rewrite_time: 0 } + - match: { profile.shards.0.dfs.knn.0.collector.0.name: "SimpleTopScoreDocCollector" } + - match: { profile.shards.0.dfs.knn.0.collector.0.reason: "search_top_hits" } + - gt: { profile.shards.0.dfs.knn.0.collector.0.time_in_nanos: 0 } + + --- dfs profile for search with dfs_query_then_fetch: - skip: diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java index 7886e628b26ad..bf937a9d57f02 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java @@ -23,7 +23,7 @@ import java.io.IOException; import java.util.concurrent.ExecutionException; -import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose; +import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomNonDataPurpose; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFutureThrows; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; @@ -98,7 +98,7 @@ private ActionFuture startBlockedCleanup(String repoN garbageFuture, () -> repository.blobStore() .blobContainer(repository.basePath()) - .writeBlob(randomPurpose(), "snap-foo.dat", new BytesArray(new byte[1]), true) + .writeBlob(randomNonDataPurpose(), "snap-foo.dat", new BytesArray(new byte[1]), true) ) ); garbageFuture.get(); @@ -147,7 +147,7 @@ public void testCleanupOldIndexN() throws ExecutionException, InterruptedExcepti () -> repository.blobStore() .blobContainer(repository.basePath()) .writeBlob( - randomPurpose(), + randomNonDataPurpose(), BlobStoreRepository.INDEX_FILE_PREFIX + generation, new BytesArray(new byte[1]), true diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryOperationPurposeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryOperationPurposeIT.java new file mode 100644 index 0000000000000..91eb1dc6eb01b --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryOperationPurposeIT.java @@ -0,0 +1,243 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.repositories.blobstore; + +import org.elasticsearch.cluster.metadata.RepositoryMetadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.blobstore.BlobStore; +import org.elasticsearch.common.blobstore.OperationPurpose; +import org.elasticsearch.common.blobstore.support.BlobMetadata; +import org.elasticsearch.common.blobstore.support.FilterBlobContainer; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.env.Environment; +import org.elasticsearch.indices.recovery.RecoverySettings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.RepositoryPlugin; +import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.fs.FsRepository; +import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; +import org.elasticsearch.xcontent.NamedXContentRegistry; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Collection; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.startsWith; + +public class BlobStoreRepositoryOperationPurposeIT extends AbstractSnapshotIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return CollectionUtils.appendToCopy(super.nodePlugins(), TestPlugin.class); + } + + public void testSnapshotOperationPurposes() throws Exception { + // Perform some simple operations on the repository in order to exercise the checks that the purpose is set correctly for various + // operations + + final var repoName = randomIdentifier(); + createRepository(repoName, TestPlugin.ASSERTING_REPO_TYPE); + + final var count = between(1, 3); + + for (int i = 0; i < count; i++) { + createIndexWithContent("index-" + i); + createFullSnapshot(repoName, "snap-" + i); + } + + final var timeout = TimeValue.timeValueSeconds(10); + clusterAdmin().prepareCleanupRepository(repoName).get(timeout); + clusterAdmin().prepareCloneSnapshot(repoName, "snap-0", "clone-0").setIndices("index-0").get(timeout); + + // restart to ensure that the reads which happen when starting a node on a nonempty repository use the expected purposes + internalCluster().fullRestart(); + + clusterAdmin().prepareGetSnapshots(repoName).get(timeout); + + clusterAdmin().prepareRestoreSnapshot(repoName, "clone-0") + .setRenamePattern("index-0") + .setRenameReplacement("restored-0") + .setWaitForCompletion(true) + .get(timeout); + + for (int i = 0; i < count; i++) { + assertTrue(startDeleteSnapshot(repoName, "snap-" + i).get(10, TimeUnit.SECONDS).isAcknowledged()); + } + + clusterAdmin().prepareDeleteRepository(repoName).get(timeout); + } + + public static class TestPlugin extends Plugin implements RepositoryPlugin { + static final String ASSERTING_REPO_TYPE = "asserting"; + + @Override + public Map getRepositories( + Environment env, + NamedXContentRegistry namedXContentRegistry, + ClusterService clusterService, + BigArrays bigArrays, + RecoverySettings recoverySettings + ) { + return Map.of( + ASSERTING_REPO_TYPE, + metadata -> new AssertingRepository(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings) + ); + } + } + + private static class AssertingRepository extends FsRepository { + AssertingRepository( + RepositoryMetadata metadata, + Environment environment, + NamedXContentRegistry namedXContentRegistry, + ClusterService clusterService, + BigArrays bigArrays, + RecoverySettings recoverySettings + ) { + super(metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + } + + @Override + protected BlobStore createBlobStore() throws Exception { + return new AssertingBlobStore(super.createBlobStore()); + } + } + + private static class AssertingBlobStore implements BlobStore { + private final BlobStore delegateBlobStore; + + AssertingBlobStore(BlobStore delegateBlobStore) { + this.delegateBlobStore = delegateBlobStore; + } + + @Override + public BlobContainer blobContainer(BlobPath path) { + return new AssertingBlobContainer(delegateBlobStore.blobContainer(path)); + } + + @Override + public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { + delegateBlobStore.deleteBlobsIgnoringIfNotExists(purpose, blobNames); + } + + @Override + public void close() throws IOException { + delegateBlobStore.close(); + } + } + + private static class AssertingBlobContainer extends FilterBlobContainer { + + AssertingBlobContainer(BlobContainer delegate) { + super(delegate); + } + + @Override + protected BlobContainer wrapChild(BlobContainer child) { + return new AssertingBlobContainer(child); + } + + @Override + public void writeBlob(OperationPurpose purpose, String blobName, BytesReference bytes, boolean failIfAlreadyExists) + throws IOException { + assertPurposeConsistency(purpose, blobName); + super.writeBlob(purpose, blobName, bytes, failIfAlreadyExists); + } + + @Override + public void writeBlob( + OperationPurpose purpose, + String blobName, + InputStream inputStream, + long blobSize, + boolean failIfAlreadyExists + ) throws IOException { + assertPurposeConsistency(purpose, blobName); + super.writeBlob(purpose, blobName, inputStream, blobSize, failIfAlreadyExists); + } + + @Override + public void writeMetadataBlob( + OperationPurpose purpose, + String blobName, + boolean failIfAlreadyExists, + boolean atomic, + CheckedConsumer writer + ) throws IOException { + assertEquals(blobName, OperationPurpose.SNAPSHOT_METADATA, purpose); + assertPurposeConsistency(purpose, blobName); + super.writeMetadataBlob(purpose, blobName, failIfAlreadyExists, atomic, writer); + } + + @Override + public void writeBlobAtomic(OperationPurpose purpose, String blobName, BytesReference bytes, boolean failIfAlreadyExists) + throws IOException { + assertEquals(blobName, OperationPurpose.SNAPSHOT_METADATA, purpose); + assertPurposeConsistency(purpose, blobName); + super.writeBlobAtomic(purpose, blobName, bytes, failIfAlreadyExists); + } + + @Override + public boolean blobExists(OperationPurpose purpose, String blobName) throws IOException { + assertEquals(blobName, OperationPurpose.SNAPSHOT_METADATA, purpose); + assertPurposeConsistency(purpose, blobName); + return super.blobExists(purpose, blobName); + } + + @Override + public InputStream readBlob(OperationPurpose purpose, String blobName) throws IOException { + assertPurposeConsistency(purpose, blobName); + return super.readBlob(purpose, blobName); + } + + @Override + public InputStream readBlob(OperationPurpose purpose, String blobName, long position, long length) throws IOException { + assertPurposeConsistency(purpose, blobName); + return super.readBlob(purpose, blobName, position, length); + } + + @Override + public Map listBlobsByPrefix(OperationPurpose purpose, String blobNamePrefix) throws IOException { + assertEquals(OperationPurpose.SNAPSHOT_METADATA, purpose); + return super.listBlobsByPrefix(purpose, blobNamePrefix); + } + } + + private static void assertPurposeConsistency(OperationPurpose purpose, String blobName) { + if (blobName.startsWith(BlobStoreRepository.UPLOADED_DATA_BLOB_PREFIX)) { + assertEquals(blobName, OperationPurpose.SNAPSHOT_DATA, purpose); + } else { + assertThat( + blobName, + anyOf( + startsWith(BlobStoreRepository.INDEX_FILE_PREFIX), + startsWith(BlobStoreRepository.METADATA_PREFIX), + startsWith(BlobStoreRepository.SNAPSHOT_PREFIX), + equalTo(BlobStoreRepository.INDEX_LATEST_BLOB), + // verification + equalTo("master.dat"), + startsWith("data-") + ) + ); + assertEquals(blobName, OperationPurpose.SNAPSHOT_METADATA, purpose); + } + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java index 603bfbeaa3dfe..1e8cc954cd850 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java @@ -198,7 +198,10 @@ public void testExplainDateRangeInQueryString() { long twoMonthsAgo = now.minus(2, ChronoUnit.MONTHS).truncatedTo(ChronoUnit.DAYS).toEpochSecond() * 1000; long rangeEnd = (now.plus(1, ChronoUnit.DAYS).truncatedTo(ChronoUnit.DAYS).toEpochSecond() * 1000) - 1; - assertThat(response.getQueryExplanation().get(0).getExplanation(), equalTo("past:[" + twoMonthsAgo + " TO " + rangeEnd + "]")); + assertThat( + response.getQueryExplanation().get(0).getExplanation(), + containsString("past:[" + twoMonthsAgo + " TO " + rangeEnd + "]") + ); assertThat(response.isValid(), equalTo(true)); } diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index c392d3b6b4e29..ca79be9453cfe 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -184,6 +184,8 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_PROFILE = def(8_551_00_0); public static final TransportVersion CLUSTER_STATS_RESCORER_USAGE_ADDED = def(8_552_00_0); public static final TransportVersion ML_INFERENCE_HF_SERVICE_ADDED = def(8_553_00_0); + public static final TransportVersion INFERENCE_USAGE_ADDED = def(8_554_00_0); + public static final TransportVersion UPGRADE_TO_LUCENE_9_9 = def(8_555_00_0); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java index f232591a05a68..17b28ebbe3b4b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat; import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat; +import org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.codecs.KnnVectorsReader; @@ -18,7 +19,7 @@ import org.apache.lucene.codecs.PointsReader; import org.apache.lucene.codecs.StoredFieldsReader; import org.apache.lucene.codecs.TermVectorsReader; -import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.ByteVectorValues; import org.apache.lucene.index.DirectoryReader; @@ -301,6 +302,9 @@ private static void readProximity(Terms terms, PostingsEnum postings) throws IOE private static BlockTermState getBlockTermState(TermsEnum termsEnum, BytesRef term) throws IOException { if (term != null && termsEnum.seekExact(term)) { final TermState termState = termsEnum.termState(); + if (termState instanceof final Lucene99PostingsFormat.IntBlockTermState blockTermState) { + return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP); + } if (termState instanceof final Lucene90PostingsFormat.IntBlockTermState blockTermState) { return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP); } @@ -310,6 +314,7 @@ private static BlockTermState getBlockTermState(TermsEnum termsEnum, BytesRef te if (termState instanceof final Lucene50PostingsFormat.IntBlockTermState blockTermState) { return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP); } + assert false : "unsupported postings format: " + termState; } return null; } @@ -527,7 +532,6 @@ void analyzeKnnVectors(SegmentReader reader, IndexDiskUsageStats stats) throws I for (FieldInfo field : reader.getFieldInfos()) { cancellationChecker.checkForCancellation(); directory.resetBytesRead(); - final KnnCollector collector = new TopKnnCollector(100, Integer.MAX_VALUE); if (field.getVectorDimension() > 0) { switch (field.getVectorEncoding()) { case BYTE -> { @@ -538,6 +542,10 @@ void analyzeKnnVectors(SegmentReader reader, IndexDiskUsageStats stats) throws I // do a couple of randomized searches to figure out min and max offsets of index file ByteVectorValues vectorValues = vectorReader.getByteVectorValues(field.name); + final KnnCollector collector = new TopKnnCollector( + Math.max(1, Math.min(100, vectorValues.size() - 1)), + Integer.MAX_VALUE + ); int numDocsToVisit = reader.maxDoc() < 10 ? reader.maxDoc() : 10 * (int) Math.log10(reader.maxDoc()); int skipFactor = Math.max(reader.maxDoc() / numDocsToVisit, 1); for (int i = 0; i < reader.maxDoc(); i += skipFactor) { @@ -557,6 +565,10 @@ void analyzeKnnVectors(SegmentReader reader, IndexDiskUsageStats stats) throws I // do a couple of randomized searches to figure out min and max offsets of index file FloatVectorValues vectorValues = vectorReader.getFloatVectorValues(field.name); + final KnnCollector collector = new TopKnnCollector( + Math.max(1, Math.min(100, vectorValues.size() - 1)), + Integer.MAX_VALUE + ); int numDocsToVisit = reader.maxDoc() < 10 ? reader.maxDoc() : 10 * (int) Math.log10(reader.maxDoc()); int skipFactor = Math.max(reader.maxDoc() / numDocsToVisit, 1); for (int i = 0; i < reader.maxDoc(); i += skipFactor) { diff --git a/server/src/main/java/org/elasticsearch/action/search/BottomSortValuesCollector.java b/server/src/main/java/org/elasticsearch/action/search/BottomSortValuesCollector.java index 34566ec48ccad..4461b71be9047 100644 --- a/server/src/main/java/org/elasticsearch/action/search/BottomSortValuesCollector.java +++ b/server/src/main/java/org/elasticsearch/action/search/BottomSortValuesCollector.java @@ -10,6 +10,7 @@ import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.Pruning; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopFieldDocs; import org.elasticsearch.search.DocValueFormat; @@ -35,7 +36,7 @@ class BottomSortValuesCollector { this.reverseMuls = new int[sortFields.length]; this.sortFields = sortFields; for (int i = 0; i < sortFields.length; i++) { - comparators[i] = sortFields[i].getComparator(1, false); + comparators[i] = sortFields[i].getComparator(1, Pruning.NONE); reverseMuls[i] = sortFields[i].getReverse() ? -1 : 1; } } diff --git a/server/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java b/server/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java index c832f222ecc69..77c225f5d94cb 100644 --- a/server/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java +++ b/server/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import java.io.IOException; import java.io.InputStream; @@ -116,6 +117,7 @@ void writeBlob(OperationPurpose purpose, String blobName, InputStream inputStrea */ default void writeBlob(OperationPurpose purpose, String blobName, BytesReference bytes, boolean failIfAlreadyExists) throws IOException { + assert assertPurposeConsistency(purpose, blobName); writeBlob(purpose, blobName, bytes.streamInput(), bytes.length(), failIfAlreadyExists); } @@ -261,4 +263,33 @@ default void getRegister(OperationPurpose purpose, String key, ActionListener + *
  • {@link OperationPurpose#SNAPSHOT_DATA} is not used for blobs that look like metadata blobs.
  • + *
  • {@link OperationPurpose#SNAPSHOT_METADATA} is not used for blobs that look like data blobs.
  • + * + */ + // This is fairly lenient because we use a wide variety of blob names and purposes in tests in order to get good coverage. See + // BlobStoreRepositoryOperationPurposeIT for some stricter checks which apply during genuine snapshot operations. + static boolean assertPurposeConsistency(OperationPurpose purpose, String blobName) { + switch (purpose) { + case SNAPSHOT_DATA -> { + // must not be used for blobs with names that look like metadata blobs + assert (blobName.startsWith(BlobStoreRepository.INDEX_FILE_PREFIX) + || blobName.startsWith(BlobStoreRepository.METADATA_PREFIX) + || blobName.startsWith(BlobStoreRepository.SNAPSHOT_PREFIX) + || blobName.equals(BlobStoreRepository.INDEX_LATEST_BLOB)) == false : blobName + " should not use purpose " + purpose; + } + case SNAPSHOT_METADATA -> { + // must not be used for blobs with names that look like data blobs + assert blobName.startsWith(BlobStoreRepository.UPLOADED_DATA_BLOB_PREFIX) == false + : blobName + " should not use purpose " + purpose; + } + case REPOSITORY_ANALYSIS, CLUSTER_STATE, INDICES, TRANSLOG -> { + // no specific requirements + } + } + return true; + } } diff --git a/server/src/main/java/org/elasticsearch/common/blobstore/OperationPurpose.java b/server/src/main/java/org/elasticsearch/common/blobstore/OperationPurpose.java index 568f2968c9e61..5df17c1948870 100644 --- a/server/src/main/java/org/elasticsearch/common/blobstore/OperationPurpose.java +++ b/server/src/main/java/org/elasticsearch/common/blobstore/OperationPurpose.java @@ -15,7 +15,8 @@ * as well as other things that requires further differentiation for the same blob operation. */ public enum OperationPurpose { - SNAPSHOT("Snapshot"), + SNAPSHOT_DATA("SnapshotData"), + SNAPSHOT_METADATA("SnapshotMetadata"), REPOSITORY_ANALYSIS("RepositoryAnalysis"), CLUSTER_STATE("ClusterState"), INDICES("Indices"), diff --git a/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java b/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java index 9f2971e24cbf3..e40ca70460b13 100644 --- a/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java +++ b/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java @@ -183,6 +183,7 @@ public boolean blobExists(OperationPurpose purpose, String blobName) { @Override public InputStream readBlob(OperationPurpose purpose, String name) throws IOException { + assert BlobContainer.assertPurposeConsistency(purpose, name); final Path resolvedPath = path.resolve(name); try { return Files.newInputStream(resolvedPath); @@ -193,6 +194,7 @@ public InputStream readBlob(OperationPurpose purpose, String name) throws IOExce @Override public InputStream readBlob(OperationPurpose purpose, String blobName, long position, long length) throws IOException { + assert BlobContainer.assertPurposeConsistency(purpose, blobName); final SeekableByteChannel channel = Files.newByteChannel(path.resolve(blobName)); if (position > 0L) { channel.position(position); @@ -210,6 +212,7 @@ public long readBlobPreferredLength() { @Override public void writeBlob(OperationPurpose purpose, String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) throws IOException { + assert BlobContainer.assertPurposeConsistency(purpose, blobName); final Path file = path.resolve(blobName); try { writeToPath(inputStream, file, blobSize); @@ -225,6 +228,7 @@ public void writeBlob(OperationPurpose purpose, String blobName, InputStream inp @Override public void writeBlob(OperationPurpose purpose, String blobName, BytesReference bytes, boolean failIfAlreadyExists) throws IOException { + assert BlobContainer.assertPurposeConsistency(purpose, blobName); final Path file = path.resolve(blobName); try { writeToPath(bytes, file); @@ -246,6 +250,7 @@ public void writeMetadataBlob( boolean atomic, CheckedConsumer writer ) throws IOException { + assert purpose != OperationPurpose.SNAPSHOT_DATA && BlobContainer.assertPurposeConsistency(purpose, blobName) : purpose; if (atomic) { final String tempBlob = tempBlobName(blobName); try { @@ -291,6 +296,7 @@ private void writeToPath( @Override public void writeBlobAtomic(OperationPurpose purpose, final String blobName, BytesReference bytes, boolean failIfAlreadyExists) throws IOException { + assert purpose != OperationPurpose.SNAPSHOT_DATA && BlobContainer.assertPurposeConsistency(purpose, blobName) : purpose; final String tempBlob = tempBlobName(blobName); final Path tempBlobPath = path.resolve(tempBlob); try { diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java index a53df0087b251..31a4ca97aad6a 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -87,7 +87,7 @@ import java.util.Objects; public class Lucene { - public static final String LATEST_CODEC = "Lucene95"; + public static final String LATEST_CODEC = "Lucene99"; public static final String SOFT_DELETES_FIELD = "__soft_deletes"; diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 75ee272e7effe..125f9529c4165 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -91,6 +91,7 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion UPGRADE_LUCENE_9_8 = def(8_500_003, Version.LUCENE_9_8_0); public static final IndexVersion ES_VERSION_8_12 = def(8_500_004, Version.LUCENE_9_8_0); public static final IndexVersion NORMALIZED_VECTOR_COSINE = def(8_500_005, Version.LUCENE_9_8_0); + public static final IndexVersion UPGRADE_LUCENE_9_9 = def(8_500_006, Version.LUCENE_9_9_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java index 990d44f5baefc..d4771ba74e0fb 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java +++ b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java @@ -9,7 +9,7 @@ package org.elasticsearch.index.codec; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.MapperService; @@ -35,11 +35,11 @@ public class CodecService { public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays) { final var codecs = new HashMap(); if (mapperService == null) { - codecs.put(DEFAULT_CODEC, new Lucene95Codec()); - codecs.put(BEST_COMPRESSION_CODEC, new Lucene95Codec(Lucene95Codec.Mode.BEST_COMPRESSION)); + codecs.put(DEFAULT_CODEC, new Lucene99Codec()); + codecs.put(BEST_COMPRESSION_CODEC, new Lucene99Codec(Lucene99Codec.Mode.BEST_COMPRESSION)); } else { - codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Lucene95Codec.Mode.BEST_SPEED, mapperService, bigArrays)); - codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMapperCodec(Lucene95Codec.Mode.BEST_COMPRESSION, mapperService, bigArrays)); + codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, bigArrays)); + codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMapperCodec(Lucene99Codec.Mode.BEST_COMPRESSION, mapperService, bigArrays)); } codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault()); for (String codec : Codec.availableCodecs()) { diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java index ee2cb06cb9559..d2ca31fe6a197 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java +++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java @@ -13,7 +13,7 @@ import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.IndexMode; @@ -37,7 +37,7 @@ * per index in real time via the mapping API. If no specific postings format or vector format is * configured for a specific field the default postings or vector format is used. */ -public final class PerFieldMapperCodec extends Lucene95Codec { +public final class PerFieldMapperCodec extends Lucene99Codec { private final MapperService mapperService; private final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat(); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java index a18ea0f90ec08..bc0a6b1ca3373 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java @@ -14,6 +14,7 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.Pruning; import org.apache.lucene.search.Scorable; import org.apache.lucene.search.SortField; import org.apache.lucene.search.comparators.TermOrdValComparator; @@ -68,13 +69,13 @@ protected SortedBinaryDocValues getValues(LeafReaderContext context) throws IOEx protected void setScorer(Scorable scorer) {} @Override - public FieldComparator newComparator(String fieldname, int numHits, boolean enableSkipping, boolean reversed) { + public FieldComparator newComparator(String fieldname, int numHits, Pruning enableSkipping, boolean reversed) { assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName()); final boolean sortMissingLast = sortMissingLast(missingValue) ^ reversed; final BytesRef missingBytes = (BytesRef) missingObject(missingValue, reversed); if (indexFieldData instanceof IndexOrdinalsFieldData) { - return new TermOrdValComparator(numHits, null, sortMissingLast, reversed, false) { + return new TermOrdValComparator(numHits, null, sortMissingLast, reversed, Pruning.NONE) { @Override protected SortedDocValues getSortedDocValues(LeafReaderContext context, String field) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java index dbc3aadde2e9f..ad70f995ef8ed 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java @@ -13,6 +13,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; import org.apache.lucene.search.Scorable; import org.apache.lucene.search.SortField; import org.apache.lucene.search.comparators.DoubleComparator; @@ -72,13 +73,13 @@ private NumericDoubleValues getNumericDocValues(LeafReaderContext context, doubl protected void setScorer(Scorable scorer) {} @Override - public FieldComparator newComparator(String fieldname, int numHits, boolean enableSkipping, boolean reversed) { + public FieldComparator newComparator(String fieldname, int numHits, Pruning enableSkipping, boolean reversed) { assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName()); final double dMissingValue = (Double) missingObject(missingValue, reversed); // NOTE: it's important to pass null as a missing value in the constructor so that // the comparator doesn't check docsWithField since we replace missing values in select() - return new DoubleComparator(numHits, null, null, reversed, false) { + return new DoubleComparator(numHits, null, null, reversed, Pruning.NONE) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new DoubleLeafComparator(context) { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java index 5dbcafcbdb5b8..46897652ee7cc 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java @@ -12,6 +12,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; import org.apache.lucene.search.Scorable; import org.apache.lucene.search.SortField; import org.apache.lucene.search.comparators.FloatComparator; @@ -65,13 +66,13 @@ private NumericDoubleValues getNumericDocValues(LeafReaderContext context, float } @Override - public FieldComparator newComparator(String fieldname, int numHits, boolean enableSkipping, boolean reversed) { + public FieldComparator newComparator(String fieldname, int numHits, Pruning enableSkipping, boolean reversed) { assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName()); final float fMissingValue = (Float) missingObject(missingValue, reversed); // NOTE: it's important to pass null as a missing value in the constructor so that // the comparator doesn't check docsWithField since we replace missing values in select() - return new FloatComparator(numHits, null, null, reversed, false) { + return new FloatComparator(numHits, null, null, reversed, Pruning.NONE) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new FloatLeafComparator(context) { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java index e8d4363ca9932..cf3c0939cd5ae 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java @@ -13,6 +13,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; import org.apache.lucene.search.SortField; import org.apache.lucene.search.comparators.LongComparator; import org.apache.lucene.util.BitSet; @@ -94,13 +95,13 @@ private NumericDocValues getNumericDocValues(LeafReaderContext context, long mis } @Override - public FieldComparator newComparator(String fieldname, int numHits, boolean enableSkipping, boolean reversed) { + public FieldComparator newComparator(String fieldname, int numHits, Pruning enableSkipping, boolean reversed) { assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldName()); final long lMissingValue = (Long) missingObject(missingValue, reversed); // NOTE: it's important to pass null as a missing value in the constructor so that // the comparator doesn't check docsWithField since we replace missing values in select() - return new LongComparator(numHits, null, null, reversed, false) { + return new LongComparator(numHits, null, null, reversed, Pruning.NONE) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new LongLeafComparator(context) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 2859d8bb29917..94b937c534491 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -370,7 +370,7 @@ public CompletionFieldType fieldType() { } static PostingsFormat postingsFormat() { - return PostingsFormat.forName("Completion90"); + return PostingsFormat.forName("Completion99"); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 423f5d81ebbd3..482f10b39fc9c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -11,7 +11,8 @@ import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.KnnVectorsWriter; -import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; @@ -30,11 +31,9 @@ import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.KnnByteVectorQuery; -import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.search.join.DiversifyingChildrenByteKnnVectorQuery; -import org.apache.lucene.search.join.DiversifyingChildrenFloatKnnVectorQuery; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.VectorUtil; import org.elasticsearch.common.xcontent.support.XContentMapValues; @@ -56,6 +55,10 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.vectors.ProfilingDiversifyingChildrenByteKnnVectorQuery; +import org.elasticsearch.search.vectors.ProfilingDiversifyingChildrenFloatKnnVectorQuery; +import org.elasticsearch.search.vectors.ProfilingKnnByteVectorQuery; +import org.elasticsearch.search.vectors.ProfilingKnnFloatVectorQuery; import org.elasticsearch.search.vectors.VectorSimilarityQuery; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -70,6 +73,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Stream; @@ -182,6 +186,13 @@ public Builder(String name, IndexVersion indexVersionCreated) { } } }); + this.indexOptions.addValidator(v -> { + if (v instanceof Int8HnswIndexOptions && elementType.getValue() == ElementType.BYTE) { + throw new IllegalArgumentException( + "[element_type] cannot be [byte] when using index type [" + VectorIndexType.INT8_HNSW.name + "]" + ); + } + }); } @Override @@ -776,26 +787,124 @@ private abstract static class IndexOptions implements ToXContent { IndexOptions(String type) { this.type = type; } + + abstract KnnVectorsFormat getVectorsFormat(); } - private static class HnswIndexOptions extends IndexOptions { + private enum VectorIndexType { + HNSW("hnsw") { + @Override + public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap) { + Object mNode = indexOptionsMap.remove("m"); + Object efConstructionNode = indexOptionsMap.remove("ef_construction"); + if (mNode == null) { + mNode = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; + } + if (efConstructionNode == null) { + efConstructionNode = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; + } + int m = XContentMapValues.nodeIntegerValue(mNode); + int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode); + MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); + return new HnswIndexOptions(m, efConstruction); + } + }, + INT8_HNSW("int8_hnsw") { + @Override + public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap) { + Object mNode = indexOptionsMap.remove("m"); + Object efConstructionNode = indexOptionsMap.remove("ef_construction"); + Object confidenceIntervalNode = indexOptionsMap.remove("confidence_interval"); + if (mNode == null) { + mNode = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; + } + if (efConstructionNode == null) { + efConstructionNode = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; + } + int m = XContentMapValues.nodeIntegerValue(mNode); + int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode); + Float confidenceInterval = null; + if (confidenceIntervalNode != null) { + confidenceInterval = (float) XContentMapValues.nodeDoubleValue(confidenceIntervalNode); + } + MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); + return new Int8HnswIndexOptions(m, efConstruction, confidenceInterval); + } + }; + + static Optional fromString(String type) { + return Stream.of(VectorIndexType.values()).filter(vectorIndexType -> vectorIndexType.name.equals(type)).findFirst(); + } + + private final String name; + + VectorIndexType(String name) { + this.name = name; + } + + abstract IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap); + } + + private static class Int8HnswIndexOptions extends IndexOptions { private final int m; private final int efConstruction; + private final Float confidenceInterval; - static IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap) { - Object mNode = indexOptionsMap.remove("m"); - Object efConstructionNode = indexOptionsMap.remove("ef_construction"); - if (mNode == null) { - throw new MapperParsingException("[index_options] of type [hnsw] requires field [m] to be configured"); - } - if (efConstructionNode == null) { - throw new MapperParsingException("[index_options] of type [hnsw] requires field [ef_construction] to be configured"); + private Int8HnswIndexOptions(int m, int efConstruction, Float confidenceInterval) { + super("int8_hnsw"); + this.m = m; + this.efConstruction = efConstruction; + this.confidenceInterval = confidenceInterval; + } + + @Override + public KnnVectorsFormat getVectorsFormat() { + return new Lucene99HnswScalarQuantizedVectorsFormat(m, efConstruction, 1, confidenceInterval, null); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("type", type); + builder.field("m", m); + builder.field("ef_construction", efConstruction); + if (confidenceInterval != null) { + builder.field("confidence_interval", confidenceInterval); } - int m = XContentMapValues.nodeIntegerValue(mNode); - int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode); - MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); - return new HnswIndexOptions(m, efConstruction); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Int8HnswIndexOptions that = (Int8HnswIndexOptions) o; + return m == that.m && efConstruction == that.efConstruction && Objects.equals(confidenceInterval, that.confidenceInterval); + } + + @Override + public int hashCode() { + return Objects.hash(m, efConstruction, confidenceInterval); + } + + @Override + public String toString() { + return "{type=" + + type + + ", m=" + + m + + ", ef_construction=" + + efConstruction + + ", confidence_interval=" + + confidenceInterval + + "}"; } + } + + private static class HnswIndexOptions extends IndexOptions { + private final int m; + private final int efConstruction; private HnswIndexOptions(int m, int efConstruction) { super("hnsw"); @@ -803,6 +912,11 @@ private HnswIndexOptions(int m, int efConstruction) { this.efConstruction = efConstruction; } + @Override + public KnnVectorsFormat getVectorsFormat() { + return new Lucene99HnswVectorsFormat(m, efConstruction, 1, null); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -828,7 +942,7 @@ public int hashCode() { @Override public String toString() { - return "{type=" + type + ", m=" + m + ", ef_construction=" + efConstruction + " }"; + return "{type=" + type + ", m=" + m + ", ef_construction=" + efConstruction + "}"; } } @@ -989,12 +1103,12 @@ && isNotUnitVector(squaredMagnitude)) { bytes[i] = (byte) queryVector[i]; } yield parentFilter != null - ? new DiversifyingChildrenByteKnnVectorQuery(name(), bytes, filter, numCands, parentFilter) - : new KnnByteVectorQuery(name(), bytes, numCands, filter); + ? new ProfilingDiversifyingChildrenByteKnnVectorQuery(name(), bytes, filter, numCands, parentFilter) + : new ProfilingKnnByteVectorQuery(name(), bytes, numCands, filter); } case FLOAT -> parentFilter != null - ? new DiversifyingChildrenFloatKnnVectorQuery(name(), queryVector, filter, numCands, parentFilter) - : new KnnFloatVectorQuery(name(), queryVector, numCands, filter); + ? new ProfilingDiversifyingChildrenFloatKnnVectorQuery(name(), queryVector, filter, numCands, parentFilter) + : new ProfilingKnnFloatVectorQuery(name(), queryVector, numCands, filter); }; if (similarityThreshold != null) { @@ -1157,11 +1271,9 @@ private static IndexOptions parseIndexOptions(String fieldName, Object propNode) throw new MapperParsingException("[index_options] requires field [type] to be configured"); } String type = XContentMapValues.nodeStringValue(typeNode); - if (type.equals("hnsw")) { - return HnswIndexOptions.parseIndexOptions(fieldName, indexOptionsMap); - } else { - throw new MapperParsingException("Unknown vector index options type [" + type + "] for field [" + fieldName + "]"); - } + return VectorIndexType.fromString(type) + .orElseThrow(() -> new MapperParsingException("Unknown vector index options type [" + type + "] for field [" + fieldName + "]")) + .parseIndexOptions(fieldName, indexOptionsMap); } /** @@ -1169,12 +1281,11 @@ private static IndexOptions parseIndexOptions(String fieldName, Object propNode) * {@code null} if the default format should be used. */ public KnnVectorsFormat getKnnVectorsFormatForField(KnnVectorsFormat defaultFormat) { - KnnVectorsFormat format; + final KnnVectorsFormat format; if (indexOptions == null) { format = defaultFormat; } else { - HnswIndexOptions hnswIndexOptions = (HnswIndexOptions) indexOptions; - format = new Lucene95HnswVectorsFormat(hnswIndexOptions.m, hnswIndexOptions.efConstruction); + format = indexOptions.getVectorsFormat(); } // It's legal to reuse the same format name as this is the same on-disk format. return new KnnVectorsFormat(format.getName()) { diff --git a/server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java b/server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java index 7504f8983b87e..463ff90b47870 100644 --- a/server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java +++ b/server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java @@ -76,7 +76,10 @@ public enum LuceneFilesExtensions { // kNN vectors format VEC("vec", "Vector Data", false, true), VEX("vex", "Vector Index", false, true), - VEM("vem", "Vector Metadata", true, false); + VEM("vem", "Vector Metadata", true, false), + VEMF("vemf", "Flat Vector Metadata", true, false), + VEMQ("vemq", "Scalar Quantized Vector Metadata", true, false), + VEQ("veq", "Scalar Quantized Vector Data", false, true); /** * Allow plugin developers of custom codecs to opt out of the assertion in {@link #fromExtension} diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/SnapshotFilesProvider.java b/server/src/main/java/org/elasticsearch/indices/recovery/SnapshotFilesProvider.java index daf9a809dcf07..1424ef160657b 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/SnapshotFilesProvider.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/SnapshotFilesProvider.java @@ -50,7 +50,7 @@ public InputStream getInputStreamForSnapshotFile( inputStream = new SlicedInputStream(fileInfo.numberOfParts()) { @Override protected InputStream openSlice(int slice) throws IOException { - return container.readBlob(OperationPurpose.SNAPSHOT, fileInfo.partName(slice)); + return container.readBlob(OperationPurpose.SNAPSHOT_DATA, fileInfo.partName(slice)); } }; } diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceServiceResults.java b/server/src/main/java/org/elasticsearch/inference/InferenceServiceResults.java index 37990caeec097..ab5b74faa6530 100644 --- a/server/src/main/java/org/elasticsearch/inference/InferenceServiceResults.java +++ b/server/src/main/java/org/elasticsearch/inference/InferenceServiceResults.java @@ -16,6 +16,17 @@ public interface InferenceServiceResults extends NamedWriteable, ToXContentFragment { + /** + * Transform the result to match the format required for the TransportCoordinatedInferenceAction. + * For the inference plugin TextEmbeddingResults, the {@link #transformToLegacyFormat()} transforms the + * results into an intermediate format only used by the plugin's return value. It doesn't align with what the + * TransportCoordinatedInferenceAction expects. TransportCoordinatedInferenceAction expects an ml plugin + * TextEmbeddingResults. + * + * For other results like SparseEmbeddingResults, this method can be a pass through to the transformToLegacyFormat. + */ + List transformToCoordinationFormat(); + /** * Transform the result to match the format required for versions prior to * {@link org.elasticsearch.TransportVersions#INFERENCE_SERVICE_RESULTS_ADDED} diff --git a/server/src/main/java/org/elasticsearch/lucene/grouping/SinglePassGroupingCollector.java b/server/src/main/java/org/elasticsearch/lucene/grouping/SinglePassGroupingCollector.java index eaa49fceb4e63..b11a034ce4e4c 100644 --- a/server/src/main/java/org/elasticsearch/lucene/grouping/SinglePassGroupingCollector.java +++ b/server/src/main/java/org/elasticsearch/lucene/grouping/SinglePassGroupingCollector.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; import org.apache.lucene.search.Scorable; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.ScoreMode; @@ -169,7 +170,7 @@ private SinglePassGroupingCollector( for (int i = 0; i < sortFields.length; i++) { final SortField sortField = sortFields[i]; // use topNGroups + 1 so we have a spare slot to use for comparing (tracked by this.spareSlot): - comparators[i] = sortField.getComparator(topNGroups + 1, false); + comparators[i] = sortField.getComparator(topNGroups + 1, Pruning.NONE); reversed[i] = sortField.getReverse() ? -1 : 1; } if (after != null) { diff --git a/server/src/main/java/org/elasticsearch/lucene/grouping/TopFieldGroups.java b/server/src/main/java/org/elasticsearch/lucene/grouping/TopFieldGroups.java index 39c807119c481..8e5efa8a880b7 100644 --- a/server/src/main/java/org/elasticsearch/lucene/grouping/TopFieldGroups.java +++ b/server/src/main/java/org/elasticsearch/lucene/grouping/TopFieldGroups.java @@ -9,6 +9,7 @@ import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.Pruning; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; @@ -121,7 +122,7 @@ private static class MergeSortQueue extends PriorityQueue { reverseMul = new int[sortFields.length]; for (int compIDX = 0; compIDX < sortFields.length; compIDX++) { final SortField sortField = sortFields[compIDX]; - comparators[compIDX] = sortField.getComparator(1, false); + comparators[compIDX] = sortField.getComparator(1, Pruning.NONE); reverseMul[compIDX] = sortField.getReverse() ? -1 : 1; } } diff --git a/server/src/main/java/org/elasticsearch/lucene/queries/BlendedTermQuery.java b/server/src/main/java/org/elasticsearch/lucene/queries/BlendedTermQuery.java index a49f02acf4c4d..d88e0e0dd9fcf 100644 --- a/server/src/main/java/org/elasticsearch/lucene/queries/BlendedTermQuery.java +++ b/server/src/main/java/org/elasticsearch/lucene/queries/BlendedTermQuery.java @@ -73,15 +73,14 @@ public Query rewrite(IndexSearcher searcher) throws IOException { if (rewritten != this) { return rewritten; } - IndexReader reader = searcher.getIndexReader(); - IndexReaderContext context = reader.getContext(); TermStates[] ctx = new TermStates[terms.length]; int[] docFreqs = new int[ctx.length]; for (int i = 0; i < terms.length; i++) { - ctx[i] = TermStates.build(context, terms[i], true); + ctx[i] = TermStates.build(searcher, terms[i], true); docFreqs[i] = ctx[i].docFreq(); } + final IndexReader reader = searcher.getIndexReader(); final int maxDoc = reader.maxDoc(); blend(ctx, maxDoc, reader); return topLevelQuery(terms, ctx, docFreqs, maxDoc); diff --git a/server/src/main/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQuery.java b/server/src/main/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQuery.java index 1bf6a1cd4f76c..c5802f092c033 100644 --- a/server/src/main/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQuery.java +++ b/server/src/main/java/org/elasticsearch/lucene/queries/SearchAfterSortedDocQuery.java @@ -16,6 +16,7 @@ import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; @@ -52,7 +53,7 @@ public SearchAfterSortedDocQuery(Sort sort, FieldDoc after) { this.reverseMuls = new int[numFields]; for (int i = 0; i < numFields; i++) { SortField sortField = sort.getSort()[i]; - FieldComparator fieldComparator = sortField.getComparator(1, false); + FieldComparator fieldComparator = sortField.getComparator(1, Pruning.NONE); @SuppressWarnings("unchecked") FieldComparator comparator = (FieldComparator) fieldComparator; comparator.setTopValue(after.fields[i]); diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index cd2b8c73fe90b..c45a048480383 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -513,7 +513,7 @@ public void cloneShardSnapshot( final ShardGeneration existingShardGen; if (shardGeneration == null) { Tuple tuple = buildBlobStoreIndexShardSnapshots( - shardContainer.listBlobsByPrefix(OperationPurpose.SNAPSHOT, INDEX_FILE_PREFIX).keySet(), + shardContainer.listBlobsByPrefix(OperationPurpose.SNAPSHOT_METADATA, INDEX_FILE_PREFIX).keySet(), shardContainer ); existingShardGen = new ShardGeneration(tuple.v2()); @@ -883,7 +883,7 @@ private void createSnapshotsDeletion( listener.onFailure(new RepositoryException(metadata.name(), "repository is readonly")); } else { threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.supply(listener, () -> { - final var originalRootBlobs = blobContainer().listBlobs(OperationPurpose.SNAPSHOT); + final var originalRootBlobs = blobContainer().listBlobs(OperationPurpose.SNAPSHOT_METADATA); // One final best-effort check for other clusters concurrently writing to the repository: final var originalRepositoryData = safeRepositoryData(repositoryDataGeneration, originalRootBlobs); @@ -893,7 +893,7 @@ private void createSnapshotsDeletion( repositoryDataGeneration, SnapshotsService.minCompatibleVersion(minimumNodeVersion, originalRepositoryData, snapshotIds), originalRootBlobs, - blobStore().blobContainer(indicesPath()).children(OperationPurpose.SNAPSHOT), + blobStore().blobContainer(indicesPath()).children(OperationPurpose.SNAPSHOT_DATA), originalRepositoryData ); })); @@ -1243,7 +1243,7 @@ private class ShardSnapshotsDeletion extends AbstractRunnable { @Override protected void doRun() throws Exception { shardContainer = shardContainer(indexId, shardId); - originalShardBlobs = shardContainer.listBlobs(OperationPurpose.SNAPSHOT).keySet(); + originalShardBlobs = shardContainer.listBlobs(OperationPurpose.SNAPSHOT_DATA).keySet(); final BlobStoreIndexShardSnapshots blobStoreIndexShardSnapshots; final long newGen; if (useShardGenerations) { @@ -1380,7 +1380,7 @@ private void cleanupUnlinkedShardLevelBlobs( } snapshotExecutor.execute(ActionRunnable.wrap(listener, l -> { try { - deleteFromContainer(blobContainer(), filesToDelete); + deleteFromContainer(OperationPurpose.SNAPSHOT_DATA, blobContainer(), filesToDelete); l.onResponse(null); } catch (Exception e) { logger.warn(() -> format("%s Failed to delete some blobs during snapshot delete", snapshotIds), e); @@ -1425,7 +1425,7 @@ private void cleanupUnlinkedRootAndIndicesBlobs(RepositoryData newRepositoryData staleBlobDeleteRunner.enqueueTask(listeners.acquire(ref -> { try (ref) { logStaleRootLevelBlobs(newRepositoryData.getGenId() - 1, snapshotIds, staleRootBlobs); - deleteFromContainer(blobContainer(), staleRootBlobs.iterator()); + deleteFromContainer(OperationPurpose.SNAPSHOT_METADATA, blobContainer(), staleRootBlobs.iterator()); for (final var staleRootBlob : staleRootBlobs) { bytesDeleted.addAndGet(originalRootBlobs.get(staleRootBlob).length()); } @@ -1456,7 +1456,7 @@ private void cleanupUnlinkedRootAndIndicesBlobs(RepositoryData newRepositoryData staleBlobDeleteRunner.enqueueTask(listeners.acquire(ref -> { try (ref) { logger.debug("[{}] Found stale index [{}]. Cleaning it up", metadata.name(), indexId); - final var deleteResult = indexEntry.getValue().delete(OperationPurpose.SNAPSHOT); + final var deleteResult = indexEntry.getValue().delete(OperationPurpose.SNAPSHOT_DATA); blobsDeleted.addAndGet(deleteResult.blobsDeleted()); bytesDeleted.addAndGet(deleteResult.bytesDeleted()); logger.debug("[{}] Cleaned up stale index [{}]", metadata.name(), indexId); @@ -1757,7 +1757,7 @@ private void cleanupOldMetadata( threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new AbstractRunnable() { @Override protected void doRun() throws Exception { - deleteFromContainer(blobContainer(), toDelete.iterator()); + deleteFromContainer(OperationPurpose.SNAPSHOT_METADATA, blobContainer(), toDelete.iterator()); } @Override @@ -1854,7 +1854,7 @@ public IndexMetadata getSnapshotIndexMetaData(RepositoryData repositoryData, Sna } } - private void deleteFromContainer(BlobContainer container, Iterator blobs) throws IOException { + private void deleteFromContainer(OperationPurpose purpose, BlobContainer container, Iterator blobs) throws IOException { final Iterator wrappedIterator; if (logger.isTraceEnabled()) { wrappedIterator = new Iterator<>() { @@ -1873,7 +1873,7 @@ public String next() { } else { wrappedIterator = blobs; } - container.deleteBlobsIgnoringIfNotExists(OperationPurpose.SNAPSHOT, wrappedIterator); + container.deleteBlobsIgnoringIfNotExists(purpose, wrappedIterator); } private BlobPath indicesPath() { @@ -2001,7 +2001,7 @@ public String startVerification() { String seed = UUIDs.randomBase64UUID(); byte[] testBytes = Strings.toUTF8Bytes(seed); BlobContainer testContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed))); - testContainer.writeBlobAtomic(OperationPurpose.SNAPSHOT, "master.dat", new BytesArray(testBytes), true); + testContainer.writeBlobAtomic(OperationPurpose.SNAPSHOT_METADATA, "master.dat", new BytesArray(testBytes), true); return seed; } } catch (Exception exp) { @@ -2014,7 +2014,7 @@ public void endVerification(String seed) { if (isReadOnly() == false) { try { final String testPrefix = testBlobPrefix(seed); - blobStore().blobContainer(basePath().add(testPrefix)).delete(OperationPurpose.SNAPSHOT); + blobStore().blobContainer(basePath().add(testPrefix)).delete(OperationPurpose.SNAPSHOT_METADATA); } catch (Exception exp) { throw new RepositoryVerificationException(metadata.name(), "cannot delete test data at " + basePath(), exp); } @@ -2434,7 +2434,7 @@ private RepositoryData getRepositoryData(long indexGen) { // EMPTY is safe here because RepositoryData#fromXContent calls namedObject try ( - InputStream blob = blobContainer().readBlob(OperationPurpose.SNAPSHOT, snapshotsIndexBlobName); + InputStream blob = blobContainer().readBlob(OperationPurpose.SNAPSHOT_METADATA, snapshotsIndexBlobName); XContentParser parser = XContentType.JSON.xContent() .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, blob) ) { @@ -2660,7 +2660,7 @@ public void onFailure(Exception e) { } final String indexBlob = INDEX_FILE_PREFIX + newGen; logger.debug("Repository [{}] writing new index generational blob [{}]", metadata.name(), indexBlob); - writeAtomic(blobContainer(), indexBlob, out -> { + writeAtomic(OperationPurpose.SNAPSHOT_METADATA, blobContainer(), indexBlob, out -> { try (XContentBuilder xContentBuilder = XContentFactory.jsonBuilder(org.elasticsearch.core.Streams.noCloseStream(out))) { newRepositoryData.snapshotsToXContent(xContentBuilder, version); } @@ -2750,7 +2750,13 @@ private void maybeWriteIndexLatest(long newGen) { if (supportURLRepo) { logger.debug("Repository [{}] updating index.latest with generation [{}]", metadata.name(), newGen); try { - writeAtomic(blobContainer(), INDEX_LATEST_BLOB, out -> out.write(Numbers.longToBytes(newGen)), false); + writeAtomic( + OperationPurpose.SNAPSHOT_METADATA, + blobContainer(), + INDEX_LATEST_BLOB, + out -> out.write(Numbers.longToBytes(newGen)), + false + ); } catch (Exception e) { logger.warn( () -> format( @@ -2777,7 +2783,7 @@ private void maybeWriteIndexLatest(long newGen) { private boolean ensureSafeGenerationExists(long safeGeneration, Consumer onFailure) throws IOException { logger.debug("Ensure generation [{}] that is the basis for this write exists in [{}]", safeGeneration, metadata.name()); if (safeGeneration != RepositoryData.EMPTY_REPO_GEN - && blobContainer().blobExists(OperationPurpose.SNAPSHOT, INDEX_FILE_PREFIX + safeGeneration) == false) { + && blobContainer().blobExists(OperationPurpose.SNAPSHOT_METADATA, INDEX_FILE_PREFIX + safeGeneration) == false) { Tuple previousWriterInfo = null; Exception readRepoDataEx = null; try { @@ -2907,7 +2913,7 @@ long latestIndexBlobId() throws IOException { // package private for testing long readSnapshotIndexLatestBlob() throws IOException { final BytesReference content = Streams.readFully( - Streams.limitStream(blobContainer().readBlob(OperationPurpose.SNAPSHOT, INDEX_LATEST_BLOB), Long.BYTES + 1) + Streams.limitStream(blobContainer().readBlob(OperationPurpose.SNAPSHOT_METADATA, INDEX_LATEST_BLOB), Long.BYTES + 1) ); if (content.length() != Long.BYTES) { throw new RepositoryException( @@ -2922,7 +2928,7 @@ long readSnapshotIndexLatestBlob() throws IOException { } private long listBlobsToGetLatestIndexId() throws IOException { - return latestGeneration(blobContainer().listBlobsByPrefix(OperationPurpose.SNAPSHOT, INDEX_FILE_PREFIX).keySet()); + return latestGeneration(blobContainer().listBlobsByPrefix(OperationPurpose.SNAPSHOT_METADATA, INDEX_FILE_PREFIX).keySet()); } private long latestGeneration(Collection rootBlobs) { @@ -2944,13 +2950,14 @@ private long latestGeneration(Collection rootBlobs) { } private void writeAtomic( + OperationPurpose purpose, BlobContainer container, final String blobName, CheckedConsumer writer, boolean failIfAlreadyExists ) throws IOException { logger.trace(() -> format("[%s] Writing [%s] to %s atomically", metadata.name(), blobName, container.path())); - container.writeMetadataBlob(OperationPurpose.SNAPSHOT, blobName, failIfAlreadyExists, true, writer); + container.writeMetadataBlob(purpose, blobName, failIfAlreadyExists, true, writer); } @Override @@ -2976,7 +2983,7 @@ private void doSnapshotShard(SnapshotShardContext context) { if (generation == null) { snapshotStatus.ensureNotAborted(); try { - blobs = shardContainer.listBlobsByPrefix(OperationPurpose.SNAPSHOT, INDEX_FILE_PREFIX).keySet(); + blobs = shardContainer.listBlobsByPrefix(OperationPurpose.SNAPSHOT_METADATA, INDEX_FILE_PREFIX).keySet(); } catch (IOException e) { throw new IndexShardSnapshotFailedException(shardId, "failed to list blobs", e); } @@ -3168,7 +3175,7 @@ private void doSnapshotShard(SnapshotShardContext context) { } snapshotStatus.addProcessedFiles(finalFilesInShardMetadataCount, finalFilesInShardMetadataSize); try { - deleteFromContainer(shardContainer, blobsToDelete.iterator()); + deleteFromContainer(OperationPurpose.SNAPSHOT_METADATA, shardContainer, blobsToDelete.iterator()); } catch (IOException e) { logger.warn( () -> format("[%s][%s] failed to delete old index-N blobs during finalization", snapshotId, shardId), @@ -3223,7 +3230,7 @@ private void doSnapshotShard(SnapshotShardContext context) { }, e -> { try { shardContainer.deleteBlobsIgnoringIfNotExists( - OperationPurpose.SNAPSHOT, + OperationPurpose.SNAPSHOT_DATA, Iterators.flatMap(fileToCleanUp.get().iterator(), f -> Iterators.forRange(0, f.numberOfParts(), f::partName)) ); } catch (Exception innerException) { @@ -3388,7 +3395,7 @@ private void restoreFile(BlobStoreIndexShardSnapshot.FileInfo fileInfo, Store st @Override protected InputStream openSlice(int slice) throws IOException { ensureNotClosing(store); - return container.readBlob(OperationPurpose.SNAPSHOT, fileInfo.partName(slice)); + return container.readBlob(OperationPurpose.SNAPSHOT_DATA, fileInfo.partName(slice)); } })) { final byte[] buffer = new byte[Math.toIntExact(Math.min(bufferSize, fileInfo.length()))]; @@ -3527,7 +3534,12 @@ public void verify(String seed, DiscoveryNode localNode) { } else { BlobContainer testBlobContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed))); try { - testBlobContainer.writeBlob(OperationPurpose.SNAPSHOT, "data-" + localNode.getId() + ".dat", new BytesArray(seed), true); + testBlobContainer.writeBlob( + OperationPurpose.SNAPSHOT_METADATA, + "data-" + localNode.getId() + ".dat", + new BytesArray(seed), + true + ); } catch (Exception exp) { throw new RepositoryVerificationException( metadata.name(), @@ -3535,7 +3547,7 @@ public void verify(String seed, DiscoveryNode localNode) { exp ); } - try (InputStream masterDat = testBlobContainer.readBlob(OperationPurpose.SNAPSHOT, "master.dat")) { + try (InputStream masterDat = testBlobContainer.readBlob(OperationPurpose.SNAPSHOT_METADATA, "master.dat")) { final String seedRead = Streams.readFully(masterDat).utf8ToString(); if (seedRead.equals(seed) == false) { throw new RepositoryVerificationException( @@ -3582,6 +3594,7 @@ private void writeShardIndexBlobAtomic( logger.trace(() -> format("[%s] Writing shard index [%s] to [%s]", metadata.name(), indexGeneration, shardContainer.path())); final String blobName = INDEX_SHARD_SNAPSHOTS_FORMAT.blobName(String.valueOf(indexGeneration)); writeAtomic( + OperationPurpose.SNAPSHOT_METADATA, shardContainer, blobName, out -> INDEX_SHARD_SNAPSHOTS_FORMAT.serialize(updatedSnapshots, blobName, compress, serializationParams, out), @@ -3617,7 +3630,7 @@ public BlobStoreIndexShardSnapshots getBlobStoreIndexShardSnapshots(IndexId inde Set blobs = Collections.emptySet(); if (shardGen == null) { - blobs = shardContainer.listBlobsByPrefix(OperationPurpose.SNAPSHOT, INDEX_FILE_PREFIX).keySet(); + blobs = shardContainer.listBlobsByPrefix(OperationPurpose.SNAPSHOT_METADATA, INDEX_FILE_PREFIX).keySet(); } return buildBlobStoreIndexShardSnapshots(blobs, shardContainer, shardGen).v1(); @@ -3719,7 +3732,7 @@ private void checkAborted() { final String partName = fileInfo.partName(i); logger.trace("[{}] Writing [{}] to [{}]", metadata.name(), partName, shardContainer.path()); final long startMS = threadPool.relativeTimeInMillis(); - shardContainer.writeBlob(OperationPurpose.SNAPSHOT, partName, inputStream, partBytes, false); + shardContainer.writeBlob(OperationPurpose.SNAPSHOT_DATA, partName, inputStream, partBytes, false); logger.trace( "[{}] Writing [{}] of size [{}b] to [{}] took [{}ms]", metadata.name(), diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java index 54cb6fe7c45d3..ca3ff799436c2 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java @@ -118,7 +118,7 @@ public ChecksumBlobStoreFormat( public T read(String repoName, BlobContainer blobContainer, String name, NamedXContentRegistry namedXContentRegistry) throws IOException { String blobName = blobName(name); - try (InputStream in = blobContainer.readBlob(OperationPurpose.SNAPSHOT, blobName)) { + try (InputStream in = blobContainer.readBlob(OperationPurpose.SNAPSHOT_METADATA, blobName)) { return deserialize(repoName, namedXContentRegistry, in); } } @@ -345,7 +345,7 @@ public void write(T obj, BlobContainer blobContainer, String name, boolean compr throws IOException { final String blobName = blobName(name); blobContainer.writeMetadataBlob( - OperationPurpose.SNAPSHOT, + OperationPurpose.SNAPSHOT_METADATA, blobName, false, false, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java index cee90f55597b2..0d088e9406175 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java @@ -21,6 +21,7 @@ import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Sort; @@ -356,8 +357,8 @@ public int hashCode() { } @Override - public FieldComparator getComparator(int numHits, boolean enableSkipping) { - return new LongComparator(1, delegate.getField(), (Long) missingValue, delegate.getReverse(), false) { + public FieldComparator getComparator(int numHits, Pruning enableSkipping) { + return new LongComparator(1, delegate.getField(), (Long) missingValue, delegate.getReverse(), Pruning.NONE) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new LongLeafComparator(context) { diff --git a/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java b/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java index 66ccae1746197..5d3288408c99b 100644 --- a/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java @@ -33,6 +33,7 @@ import org.elasticsearch.search.rescore.RescoreContext; import org.elasticsearch.search.vectors.KnnSearchBuilder; import org.elasticsearch.search.vectors.KnnVectorQueryBuilder; +import org.elasticsearch.search.vectors.ProfilingQuery; import org.elasticsearch.tasks.TaskCancelledException; import java.io.IOException; @@ -215,6 +216,11 @@ static DfsKnnResults singleKnnSearch(Query knnQuery, int k, Profilers profilers, CollectorResult.REASON_SEARCH_TOP_HITS ); topDocs = searcher.search(knnQuery, ipcm); + + if (knnQuery instanceof ProfilingQuery profilingQuery) { + profilingQuery.profile(knnProfiler); + } + knnProfiler.setCollectorResult(ipcm.getCollectorTree()); } // Set profiler back after running KNN searches diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index b7c77e4968854..c47f53ec503b9 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -8,8 +8,6 @@ package org.elasticsearch.search.internal; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; @@ -36,9 +34,7 @@ import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.Bits; import org.apache.lucene.util.SparseFixedBitSet; -import org.apache.lucene.util.ThreadInterruptedException; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; -import org.elasticsearch.common.util.concurrent.FutureUtils; import org.elasticsearch.core.Releasable; import org.elasticsearch.lucene.util.CombinedBitSet; import org.elasticsearch.search.dfs.AggregatedDfs; @@ -58,21 +54,14 @@ import java.util.Objects; import java.util.PriorityQueue; import java.util.Set; -import java.util.concurrent.CancellationException; -import java.util.concurrent.ExecutionException; +import java.util.concurrent.Callable; import java.util.concurrent.Executor; -import java.util.concurrent.Future; -import java.util.concurrent.FutureTask; -import java.util.concurrent.RunnableFuture; -import java.util.concurrent.atomic.AtomicInteger; /** * Context-aware extension of {@link IndexSearcher}. */ public class ContextIndexSearcher extends IndexSearcher implements Releasable { - private static final Logger logger = LogManager.getLogger(ContextIndexSearcher.class); - /** * The interval at which we check for search cancellation when we cannot use * a {@link CancellableBulkScorer}. See {@link #intersectScorerAndBitSet}. @@ -143,7 +132,6 @@ public ContextIndexSearcher( int maximumNumberOfSlices, int minimumDocsPerSlice ) throws IOException { - // we need to pass the executor up so it can potentially be used by query rewrite, which does not rely on slicing super(wrapWithExitableDirectoryReader ? new ExitableDirectoryReader((DirectoryReader) reader, cancellable) : reader, executor); setSimilarity(similarity); setQueryCache(queryCache); @@ -324,22 +312,12 @@ public T search(Query query, CollectorManager col /** * Similar to the lucene implementation, with the following changes made: - * 1) it will wait for all threads to finish before returning when an exception is thrown. In that case, subsequent exceptions will be - * ignored and the first exception is re-thrown after all tasks are completed. - * 2) Tasks are cancelled on exception, as well as on timeout, to prevent needless computation - * 3) collection is unconditionally offloaded to the executor when set, even when there is a single slice or the request does not - * support concurrent collection. The executor is not set only when concurrent search has been explicitly disabled at the cluster level. - * 4) postCollection is performed after each segment is collected. This is needed for aggregations, performed by search worker threads + * 1) postCollection is performed after each segment is collected. This is needed for aggregations, performed by search worker threads * so it can be parallelized. Also, it needs to happen in the same thread where doc_values are read, as it consumes them and Lucene * does not allow consuming them from a different thread. - * 5) handles the ES TimeExceededException + * 2) handles the ES TimeExceededException * */ private T search(Weight weight, CollectorManager collectorManager, C firstCollector) throws IOException { - // the executor will be null only when concurrency is disabled at the cluster level - if (getExecutor() == null) { - search(leafContexts, weight, firstCollector); - return collectorManager.reduce(Collections.singletonList(firstCollector)); - } LeafSlice[] leafSlices = getSlices(); if (leafSlices.length == 0) { assert leafContexts.isEmpty(); @@ -356,92 +334,16 @@ private T search(Weight weight, CollectorManager throw new IllegalStateException("CollectorManager does not always produce collectors with the same score mode"); } } - final List> listTasks = new ArrayList<>(); + final List> listTasks = new ArrayList<>(); for (int i = 0; i < leafSlices.length; ++i) { final LeafReaderContext[] leaves = leafSlices[i].leaves; final C collector = collectors.get(i); - AtomicInteger state = new AtomicInteger(0); - RunnableFuture task = new FutureTask<>(() -> { - if (state.compareAndSet(0, 1)) { - // A slice throws exception or times out: cancel all the tasks, to prevent slices that haven't started yet from - // starting and performing needless computation. - // TODO we will also want to cancel tasks that have already started, reusing the timeout mechanism - try { - search(Arrays.asList(leaves), weight, collector); - if (timeExceeded) { - for (Future future : listTasks) { - FutureUtils.cancel(future); - } - } - } catch (Exception e) { - for (Future future : listTasks) { - FutureUtils.cancel(future); - } - throw e; - } - return collector; - } - throw new CancellationException(); - }) { - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - /* - Future#get (called down below after submitting all tasks) throws CancellationException for a cancelled task while - it is still running. It's important to make sure that search does not leave any tasks behind when it returns. - Overriding cancel ensures that tasks that are already started are left alone once cancelled, so Future#get will - wait for them to finish instead of throwing CancellationException. - Tasks that are cancelled before they are started won't start (same behaviour as the original implementation). - */ - return state.compareAndSet(0, -1); - } - - @Override - public boolean isCancelled() { - return state.get() == -1; - } - }; - listTasks.add(task); - } - logger.trace("Collecting using " + listTasks.size() + " tasks."); - - for (Runnable task : listTasks) { - getExecutor().execute(task); - } - RuntimeException exception = null; - final List collectedCollectors = new ArrayList<>(); - boolean cancellation = false; - for (Future future : listTasks) { - try { - collectedCollectors.add(future.get()); - } catch (InterruptedException e) { - if (exception == null) { - exception = new ThreadInterruptedException(e); - } else { - // we ignore further exceptions - } - } catch (ExecutionException e) { - if (exception == null) { - if (e.getCause() instanceof CancellationException) { - // thrown by the manual cancellation implemented above - we cancel on exception and we will throw the root cause - cancellation = true; - } else { - if (e.getCause() instanceof RuntimeException runtimeException) { - exception = runtimeException; - } else if (e.getCause() instanceof IOException ioException) { - throw ioException; - } else { - exception = new RuntimeException(e.getCause()); - } - } - } else { - // we ignore further exceptions - } - } - } - assert cancellation == false || exception != null || timeExceeded : "cancellation without an exception or timeout?"; - if (exception != null) { - throw exception; + listTasks.add(() -> { + search(Arrays.asList(leaves), weight, collector); + return collector; + }); } + List collectedCollectors = getTaskExecutor().invokeAll(listTasks); return collectorManager.reduce(collectedCollectors); } } diff --git a/server/src/main/java/org/elasticsearch/search/profile/Profilers.java b/server/src/main/java/org/elasticsearch/search/profile/Profilers.java index 2cc29d654ec86..44ad9be7e1e94 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/Profilers.java +++ b/server/src/main/java/org/elasticsearch/search/profile/Profilers.java @@ -65,7 +65,8 @@ public SearchProfileQueryPhaseResult buildQueryPhaseResults() { QueryProfileShardResult result = new QueryProfileShardResult( queryProfiler.getTree(), queryProfiler.getRewriteTime(), - queryProfiler.getCollectorResult() + queryProfiler.getCollectorResult(), + null ); AggregationProfileShardResult aggResults = new AggregationProfileShardResult(aggProfiler.getTree()); return new SearchProfileQueryPhaseResult(Collections.singletonList(result), aggResults); diff --git a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileDfsPhaseResult.java b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileDfsPhaseResult.java index 4e301d5a3300d..5f8e6a893c1b5 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileDfsPhaseResult.java +++ b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileDfsPhaseResult.java @@ -148,7 +148,8 @@ QueryProfileShardResult combineQueryProfileShardResults() { return new QueryProfileShardResult( profileResults, totalRewriteTime, - new CollectorResult("KnnQueryCollector", CollectorResult.REASON_SEARCH_MULTI, totalCollectionTime, subCollectorResults) + new CollectorResult("KnnQueryCollector", CollectorResult.REASON_SEARCH_MULTI, totalCollectionTime, subCollectorResults), + null ); } } diff --git a/server/src/main/java/org/elasticsearch/search/profile/dfs/DfsProfiler.java b/server/src/main/java/org/elasticsearch/search/profile/dfs/DfsProfiler.java index 72104aea8a9b8..0ef4704fa1894 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/dfs/DfsProfiler.java +++ b/server/src/main/java/org/elasticsearch/search/profile/dfs/DfsProfiler.java @@ -68,7 +68,12 @@ public SearchProfileDfsPhaseResult buildDfsPhaseResults() { final List queryProfileShardResult = new ArrayList<>(knnQueryProfilers.size()); for (QueryProfiler queryProfiler : knnQueryProfilers) { queryProfileShardResult.add( - new QueryProfileShardResult(queryProfiler.getTree(), queryProfiler.getRewriteTime(), queryProfiler.getCollectorResult()) + new QueryProfileShardResult( + queryProfiler.getTree(), + queryProfiler.getRewriteTime(), + queryProfiler.getCollectorResult(), + queryProfiler.getVectorOpsCount() + ) ); } return new SearchProfileDfsPhaseResult(dfsProfileResult, queryProfileShardResult); diff --git a/server/src/main/java/org/elasticsearch/search/profile/query/QueryProfileShardResult.java b/server/src/main/java/org/elasticsearch/search/profile/query/QueryProfileShardResult.java index 6c9f1edd6c583..1b799983dd0a4 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/query/QueryProfileShardResult.java +++ b/server/src/main/java/org/elasticsearch/search/profile/query/QueryProfileShardResult.java @@ -8,10 +8,12 @@ package org.elasticsearch.search.profile.query; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Nullable; import org.elasticsearch.search.profile.ProfileResult; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -35,17 +37,27 @@ public final class QueryProfileShardResult implements Writeable, ToXContentObjec public static final String REWRITE_TIME = "rewrite_time"; public static final String QUERY_ARRAY = "query"; + public static final String VECTOR_OPERATIONS_COUNT = "vector_operations_count"; + private final List queryProfileResults; private final CollectorResult profileCollector; private final long rewriteTime; - public QueryProfileShardResult(List queryProfileResults, long rewriteTime, CollectorResult profileCollector) { + private final Long vectorOperationsCount; + + public QueryProfileShardResult( + List queryProfileResults, + long rewriteTime, + CollectorResult profileCollector, + @Nullable Long vectorOperationsCount + ) { assert (profileCollector != null); this.queryProfileResults = queryProfileResults; this.profileCollector = profileCollector; this.rewriteTime = rewriteTime; + this.vectorOperationsCount = vectorOperationsCount; } /** @@ -60,6 +72,9 @@ public QueryProfileShardResult(StreamInput in) throws IOException { profileCollector = new CollectorResult(in); rewriteTime = in.readLong(); + vectorOperationsCount = (in.getTransportVersion().onOrAfter(TransportVersions.UPGRADE_TO_LUCENE_9_9)) + ? in.readOptionalLong() + : null; } @Override @@ -70,6 +85,9 @@ public void writeTo(StreamOutput out) throws IOException { } profileCollector.writeTo(out); out.writeLong(rewriteTime); + if (out.getTransportVersion().onOrAfter(TransportVersions.UPGRADE_TO_LUCENE_9_9)) { + out.writeOptionalLong(vectorOperationsCount); + } } public List getQueryResults() { @@ -87,6 +105,9 @@ public CollectorResult getCollectorResult() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); + if (vectorOperationsCount != null) { + builder.field(VECTOR_OPERATIONS_COUNT, vectorOperationsCount); + } builder.startArray(QUERY_ARRAY); for (ProfileResult p : queryProfileResults) { p.toXContent(builder, params); @@ -127,6 +148,7 @@ public static QueryProfileShardResult fromXContent(XContentParser parser) throws String currentFieldName = null; List queryProfileResults = new ArrayList<>(); long rewriteTime = 0; + Long vectorOperationsCount = null; CollectorResult collector = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -134,6 +156,8 @@ public static QueryProfileShardResult fromXContent(XContentParser parser) throws } else if (token.isValue()) { if (REWRITE_TIME.equals(currentFieldName)) { rewriteTime = parser.longValue(); + } else if (VECTOR_OPERATIONS_COUNT.equals(currentFieldName)) { + vectorOperationsCount = parser.longValue(); } else { parser.skipChildren(); } @@ -153,6 +177,6 @@ public static QueryProfileShardResult fromXContent(XContentParser parser) throws parser.skipChildren(); } } - return new QueryProfileShardResult(queryProfileResults, rewriteTime, collector); + return new QueryProfileShardResult(queryProfileResults, rewriteTime, collector, vectorOperationsCount); } } diff --git a/server/src/main/java/org/elasticsearch/search/profile/query/QueryProfiler.java b/server/src/main/java/org/elasticsearch/search/profile/query/QueryProfiler.java index 8cfbecc14ecf5..a40b1284238b2 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/query/QueryProfiler.java +++ b/server/src/main/java/org/elasticsearch/search/profile/query/QueryProfiler.java @@ -31,10 +31,20 @@ public final class QueryProfiler extends AbstractProfiler newComparator(String fieldname, int numHits, boolean enableSkipping, boolean reversed) { - return new DoubleComparator(numHits, null, null, reversed, false) { + public FieldComparator newComparator(String fieldname, int numHits, Pruning enableSkipping, boolean reversed) { + return new DoubleComparator(numHits, null, null, reversed, Pruning.NONE) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new DoubleLeafComparator(context) { diff --git a/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java b/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java index 58fd3029c0105..9cb554f560d84 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java +++ b/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java @@ -11,6 +11,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; import org.apache.lucene.search.SortField; import org.apache.lucene.search.comparators.DocComparator; @@ -34,8 +35,8 @@ int getShardRequestIndex() { } @Override - public FieldComparator getComparator(int numHits, boolean enableSkipping) { - final DocComparator delegate = new DocComparator(numHits, getReverse(), false); + public FieldComparator getComparator(int numHits, Pruning enableSkipping) { + final DocComparator delegate = new DocComparator(numHits, getReverse(), Pruning.NONE); return new FieldComparator() { @Override diff --git a/server/src/main/java/org/elasticsearch/search/vectors/ProfilingDiversifyingChildrenByteKnnVectorQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/ProfilingDiversifyingChildrenByteKnnVectorQuery.java new file mode 100644 index 0000000000000..568516f494df4 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/vectors/ProfilingDiversifyingChildrenByteKnnVectorQuery.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.vectors; + +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.join.BitSetProducer; +import org.apache.lucene.search.join.DiversifyingChildrenByteKnnVectorQuery; +import org.elasticsearch.search.profile.query.QueryProfiler; + +public class ProfilingDiversifyingChildrenByteKnnVectorQuery extends DiversifyingChildrenByteKnnVectorQuery implements ProfilingQuery { + private long vectorOpsCount; + + public ProfilingDiversifyingChildrenByteKnnVectorQuery( + String field, + byte[] query, + Query childFilter, + int k, + BitSetProducer parentsFilter + ) { + super(field, query, childFilter, k, parentsFilter); + } + + @Override + protected TopDocs mergeLeafResults(TopDocs[] perLeafResults) { + TopDocs topK = super.mergeLeafResults(perLeafResults); + vectorOpsCount = topK.totalHits.value; + return topK; + } + + @Override + public void profile(QueryProfiler queryProfiler) { + queryProfiler.setVectorOpsCount(vectorOpsCount); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/vectors/ProfilingDiversifyingChildrenFloatKnnVectorQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/ProfilingDiversifyingChildrenFloatKnnVectorQuery.java new file mode 100644 index 0000000000000..3d0d2c512dc46 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/vectors/ProfilingDiversifyingChildrenFloatKnnVectorQuery.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.vectors; + +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.join.BitSetProducer; +import org.apache.lucene.search.join.DiversifyingChildrenFloatKnnVectorQuery; +import org.elasticsearch.search.profile.query.QueryProfiler; + +public class ProfilingDiversifyingChildrenFloatKnnVectorQuery extends DiversifyingChildrenFloatKnnVectorQuery implements ProfilingQuery { + private long vectorOpsCount; + + public ProfilingDiversifyingChildrenFloatKnnVectorQuery( + String field, + float[] query, + Query childFilter, + int k, + BitSetProducer parentsFilter + ) { + super(field, query, childFilter, k, parentsFilter); + } + + @Override + protected TopDocs mergeLeafResults(TopDocs[] perLeafResults) { + TopDocs topK = super.mergeLeafResults(perLeafResults); + vectorOpsCount = topK.totalHits.value; + return topK; + } + + @Override + public void profile(QueryProfiler queryProfiler) { + queryProfiler.setVectorOpsCount(vectorOpsCount); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/vectors/ProfilingKnnByteVectorQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/ProfilingKnnByteVectorQuery.java new file mode 100644 index 0000000000000..212cd51c3c22c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/vectors/ProfilingKnnByteVectorQuery.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.vectors; + +import org.apache.lucene.search.KnnByteVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TopDocs; +import org.elasticsearch.search.profile.query.QueryProfiler; + +public class ProfilingKnnByteVectorQuery extends KnnByteVectorQuery implements ProfilingQuery { + private long vectorOpsCount; + + public ProfilingKnnByteVectorQuery(String field, byte[] target, int k, Query filter) { + super(field, target, k, filter); + } + + @Override + protected TopDocs mergeLeafResults(TopDocs[] perLeafResults) { + TopDocs topK = super.mergeLeafResults(perLeafResults); + vectorOpsCount = topK.totalHits.value; + return topK; + } + + @Override + public void profile(QueryProfiler queryProfiler) { + queryProfiler.setVectorOpsCount(vectorOpsCount); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/vectors/ProfilingKnnFloatVectorQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/ProfilingKnnFloatVectorQuery.java new file mode 100644 index 0000000000000..f0818d71ebeab --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/vectors/ProfilingKnnFloatVectorQuery.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.vectors; + +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TopDocs; +import org.elasticsearch.search.profile.query.QueryProfiler; + +public class ProfilingKnnFloatVectorQuery extends KnnFloatVectorQuery implements ProfilingQuery { + private long vectorOpsCount; + + public ProfilingKnnFloatVectorQuery(String field, float[] target, int k, Query filter) { + super(field, target, k, filter); + } + + @Override + protected TopDocs mergeLeafResults(TopDocs[] perLeafResults) { + TopDocs topK = super.mergeLeafResults(perLeafResults); + vectorOpsCount = topK.totalHits.value; + return topK; + } + + @Override + public void profile(QueryProfiler queryProfiler) { + queryProfiler.setVectorOpsCount(vectorOpsCount); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/vectors/ProfilingQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/ProfilingQuery.java new file mode 100644 index 0000000000000..0c12bf38b1d14 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/vectors/ProfilingQuery.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.vectors; + +import org.apache.lucene.document.KnnFloatVectorField; +import org.elasticsearch.search.profile.query.QueryProfiler; + +/** + * + *

    This interface includes the declaration of an abstract method, profile(). Classes implementing this interface + * must provide an implementation for profile() to store profiling information in the {@link QueryProfiler}. + */ + +public interface ProfilingQuery { + + /** + * Store the profiling information in the {@link QueryProfiler} + * @param queryProfiler an instance of {@link KnnFloatVectorField}. + */ + void profile(QueryProfiler queryProfiler); +} diff --git a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java index 2263bfe78f218..f7362c7001c36 100644 --- a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java @@ -39,7 +39,6 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NotSerializableExceptionWrapper; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.CancellableThreadsTests; @@ -129,9 +128,8 @@ public class ExceptionSerializationTests extends ESTestCase { - public void testExceptionRegistration() throws ClassNotFoundException, IOException, URISyntaxException { + public void testExceptionRegistration() throws IOException, URISyntaxException { final Set> notRegistered = new HashSet<>(); - final Set> hasDedicatedWrite = new HashSet<>(); final Set> registered = new HashSet<>(); final String path = "/org/elasticsearch"; final Path startPath = PathUtils.get(ElasticsearchException.class.getProtectionDomain().getCodeSource().getLocation().toURI()) @@ -146,13 +144,13 @@ public void testExceptionRegistration() throws ClassNotFoundException, IOExcepti private Path pkgPrefix = PathUtils.get(path).getParent(); @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { pkgPrefix = pkgPrefix.resolve(dir.getFileName()); return FileVisitResult.CONTINUE; } @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { checkFile(file.getFileName().toString()); return FileVisitResult.CONTINUE; } @@ -180,13 +178,6 @@ private void checkClass(Class clazz) { notRegistered.add(clazz); } else if (ElasticsearchException.isRegistered(clazz.asSubclass(Throwable.class), TransportVersion.current())) { registered.add(clazz); - try { - if (clazz.getMethod("writeTo", StreamOutput.class) != null) { - hasDedicatedWrite.add(clazz); - } - } catch (Exception e) { - // fair enough - } } } @@ -199,7 +190,7 @@ private Class loadClass(String filename) throws ClassNotFoundException { for (Path p : pkgPrefix) { pkg.append(p.getFileName().toString()).append("."); } - pkg.append(filename.substring(0, filename.length() - 6)); + pkg.append(filename, 0, filename.length() - 6); return getClass().getClassLoader().loadClass(pkg.toString()); } @@ -209,7 +200,7 @@ public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOExce } @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + public FileVisitResult postVisitDirectory(Path dir, IOException exc) { pkgPrefix = pkgPrefix.getParent(); return FileVisitResult.CONTINUE; } @@ -220,7 +211,7 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOEx Files.walkFileTree(testStartPath, visitor); assertTrue(notRegistered.remove(TestException.class)); assertTrue(notRegistered.remove(UnknownHeaderException.class)); - assertTrue("Classes subclassing ElasticsearchException must be registered \n" + notRegistered.toString(), notRegistered.isEmpty()); + assertTrue("Classes subclassing ElasticsearchException must be registered \n" + notRegistered, notRegistered.isEmpty()); assertTrue(registered.removeAll(ElasticsearchException.getRegisteredKeys())); // check assertEquals(registered.toString(), 0, registered.size()); } @@ -344,7 +335,7 @@ public void testInvalidIndexTemplateException() throws IOException { assertEquals(ex.name(), "foo"); ex = serialize(new InvalidIndexTemplateException(null, "bar")); assertEquals(ex.getMessage(), "index_template [null] invalid, cause [bar]"); - assertEquals(ex.name(), null); + assertNull(ex.name()); } public void testActionTransportException() throws IOException { @@ -353,17 +344,12 @@ public void testActionTransportException() throws IOException { assertEquals("[name?][" + transportAddress + "][ACTION BABY!] message?", ex.getMessage()); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/102868") public void testSearchContextMissingException() throws IOException { ShardSearchContextId contextId = new ShardSearchContextId(UUIDs.randomBase64UUID(), randomLong()); - TransportVersion version = TransportVersionUtils.randomVersion(random()); + TransportVersion version = TransportVersionUtils.randomCompatibleVersion(random()); SearchContextMissingException ex = serialize(new SearchContextMissingException(contextId), version); assertThat(ex.contextId().getId(), equalTo(contextId.getId())); - if (version.onOrAfter(TransportVersions.V_7_7_0)) { - assertThat(ex.contextId().getSessionId(), equalTo(contextId.getSessionId())); - } else { - assertThat(ex.contextId().getSessionId(), equalTo("")); - } + assertThat(ex.contextId().getSessionId(), equalTo(contextId.getSessionId())); } public void testCircuitBreakingException() throws IOException { @@ -422,7 +408,7 @@ public void testConnectTransportException() throws IOException { } public void testSearchPhaseExecutionException() throws IOException { - ShardSearchFailure[] empty = new ShardSearchFailure[0]; + ShardSearchFailure[] empty = ShardSearchFailure.EMPTY_ARRAY; SearchPhaseExecutionException ex = serialize(new SearchPhaseExecutionException("boom", "baam", new NullPointerException(), empty)); assertEquals("boom", ex.getPhaseName()); assertEquals("baam", ex.getMessage()); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStatsTests.java index 10419719a5ed1..cc4509500f9c1 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStatsTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.admin.cluster.stats; +import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.TransportVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable.Reader; @@ -19,6 +20,7 @@ import java.util.List; import java.util.Map; +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/102920") // failing test is final, mute whole suite public class SearchUsageStatsTests extends AbstractWireSerializingTestCase { private static final List QUERY_TYPES = List.of( diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java index fec7a86bd3e59..6c79946cce15f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java @@ -12,9 +12,9 @@ import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; -import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; -import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; @@ -54,7 +54,7 @@ import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; -import org.apache.lucene.search.suggest.document.Completion90PostingsFormat; +import org.apache.lucene.search.suggest.document.Completion99PostingsFormat; import org.apache.lucene.search.suggest.document.CompletionPostingsFormat; import org.apache.lucene.search.suggest.document.SuggestField; import org.apache.lucene.store.Directory; @@ -263,7 +263,7 @@ public void testKnnVectors() throws Exception { logger.info("--> stats {}", stats); long dataBytes = (long) numDocs * dimension * Float.BYTES; // size of flat vector data - long indexBytesEstimate = (long) numDocs * (Lucene95HnswVectorsFormat.DEFAULT_MAX_CONN / 4); // rough size of HNSW graph + long indexBytesEstimate = (long) numDocs * (Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN / 4); // rough size of HNSW graph assertThat("numDocs=" + numDocs + ";dimension=" + dimension, stats.total().getKnnVectorsBytes(), greaterThan(dataBytes)); long connectionOverhead = stats.total().getKnnVectorsBytes() - dataBytes; assertThat("numDocs=" + numDocs, connectionOverhead, greaterThan(indexBytesEstimate)); @@ -326,11 +326,11 @@ public void testTriangle() throws Exception { public void testCompletionField() throws Exception { IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true) .setUseCompoundFile(false) - .setCodec(new Lucene95Codec(Lucene95Codec.Mode.BEST_SPEED) { + .setCodec(new Lucene99Codec(Lucene99Codec.Mode.BEST_SPEED) { @Override public PostingsFormat getPostingsFormatForField(String field) { if (field.startsWith("suggest_")) { - return new Completion90PostingsFormat(randomFrom(CompletionPostingsFormat.FSTLoadMode.values())); + return new Completion99PostingsFormat(randomFrom(CompletionPostingsFormat.FSTLoadMode.values())); } else { return super.postingsFormat(); } @@ -413,25 +413,25 @@ private static void addFieldsToDoc(Document doc, IndexableField[] fields) { enum CodecMode { BEST_SPEED { @Override - Lucene95Codec.Mode mode() { - return Lucene95Codec.Mode.BEST_SPEED; + Lucene99Codec.Mode mode() { + return Lucene99Codec.Mode.BEST_SPEED; } }, BEST_COMPRESSION { @Override - Lucene95Codec.Mode mode() { - return Lucene95Codec.Mode.BEST_COMPRESSION; + Lucene99Codec.Mode mode() { + return Lucene99Codec.Mode.BEST_COMPRESSION; } }; - abstract Lucene95Codec.Mode mode(); + abstract Lucene99Codec.Mode mode(); } static void indexRandomly(Directory directory, CodecMode codecMode, int numDocs, Consumer addFields) throws IOException { IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true) .setUseCompoundFile(randomBoolean()) - .setCodec(new Lucene95Codec(codecMode.mode())); + .setCodec(new Lucene99Codec(codecMode.mode())); try (IndexWriter writer = new IndexWriter(directory, config)) { for (int i = 0; i < numDocs; i++) { final Document doc = new Document(); @@ -639,10 +639,10 @@ static void rewriteIndexWithPerFieldCodec(Directory source, CodecMode mode, Dire try (DirectoryReader reader = DirectoryReader.open(source)) { IndexWriterConfig config = new IndexWriterConfig().setSoftDeletesField(Lucene.SOFT_DELETES_FIELD) .setUseCompoundFile(randomBoolean()) - .setCodec(new Lucene95Codec(mode.mode()) { + .setCodec(new Lucene99Codec(mode.mode()) { @Override public PostingsFormat getPostingsFormatForField(String field) { - return new Lucene90PostingsFormat(); + return new Lucene99PostingsFormat(); } @Override @@ -652,7 +652,7 @@ public DocValuesFormat getDocValuesFormatForField(String field) { @Override public KnnVectorsFormat getKnnVectorsFormatForField(String field) { - return new Lucene95HnswVectorsFormat(); + return new Lucene99HnswVectorsFormat(); } @Override @@ -709,7 +709,7 @@ static void collectPerFieldStats(SegmentReader reader, IndexDiskUsageStats stats stats.addStoredField("_all_stored_fields", bytes); case TVX, TVD -> stats.addTermVectors("_all_vectors_fields", bytes); case NVD, NVM -> stats.addNorms("_all_norms_fields", bytes); - case VEM, VEC, VEX -> stats.addKnnVectors(fieldLookup.getVectorsField(file), bytes); + case VEM, VEMF, VEC, VEX, VEQ, VEMQ -> stats.addKnnVectors(fieldLookup.getVectorsField(file), bytes); } } } finally { diff --git a/server/src/test/java/org/elasticsearch/action/search/BottomSortValuesCollectorTests.java b/server/src/test/java/org/elasticsearch/action/search/BottomSortValuesCollectorTests.java index 31f3fe7066bed..4305d0af9a7c1 100644 --- a/server/src/test/java/org/elasticsearch/action/search/BottomSortValuesCollectorTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/BottomSortValuesCollectorTests.java @@ -10,6 +10,7 @@ import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.Pruning; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.search.TotalHits; @@ -234,7 +235,7 @@ private Object[] newDateNanoArray(String... values) { private TopFieldDocs createTopDocs(SortField sortField, int totalHits, Object[] values) { FieldDoc[] fieldDocs = new FieldDoc[values.length]; @SuppressWarnings("unchecked") - FieldComparator cmp = (FieldComparator) sortField.getComparator(1, false); + FieldComparator cmp = (FieldComparator) sortField.getComparator(1, Pruning.NONE); for (int i = 0; i < values.length; i++) { fieldDocs[i] = new FieldDoc(i, Float.NaN, new Object[] { values[i] }); } diff --git a/server/src/test/java/org/elasticsearch/common/blobstore/fs/FsBlobContainerTests.java b/server/src/test/java/org/elasticsearch/common/blobstore/fs/FsBlobContainerTests.java index 1f54046630cf8..67712af9ef57b 100644 --- a/server/src/test/java/org/elasticsearch/common/blobstore/fs/FsBlobContainerTests.java +++ b/server/src/test/java/org/elasticsearch/common/blobstore/fs/FsBlobContainerTests.java @@ -46,6 +46,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomNonDataPurpose; import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -228,14 +229,19 @@ private static void checkAtomicWrite() throws IOException { BlobPath.EMPTY, path ); - container.writeBlobAtomic(randomPurpose(), blobName, new BytesArray(randomByteArrayOfLength(randomIntBetween(1, 512))), true); + container.writeBlobAtomic( + randomNonDataPurpose(), + blobName, + new BytesArray(randomByteArrayOfLength(randomIntBetween(1, 512))), + true + ); final var blobData = new BytesArray(randomByteArrayOfLength(randomIntBetween(1, 512))); - container.writeBlobAtomic(randomPurpose(), blobName, blobData, false); + container.writeBlobAtomic(randomNonDataPurpose(), blobName, blobData, false); assertEquals(blobData, Streams.readFully(container.readBlob(randomPurpose(), blobName))); expectThrows( FileAlreadyExistsException.class, () -> container.writeBlobAtomic( - randomPurpose(), + randomNonDataPurpose(), blobName, new BytesArray(randomByteArrayOfLength(randomIntBetween(1, 512))), true diff --git a/server/src/test/java/org/elasticsearch/index/IndexVersionTests.java b/server/src/test/java/org/elasticsearch/index/IndexVersionTests.java index e13ce9702cdfd..dcf73ec617e60 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexVersionTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexVersionTests.java @@ -106,8 +106,10 @@ public void testDefinedConstants() throws IllegalAccessException { field.getModifiers() ); - Matcher matcher = historicalVersion.matcher(field.getName()); - if (matcher.matches()) { + Matcher matcher; + if ("UPGRADE_TO_LUCENE_9_9".equals(field.getName())) { + // OK + } else if ((matcher = historicalVersion.matcher(field.getName())).matches()) { // old-style version constant String idString = matcher.group(1) + padNumber(matcher.group(2)) + padNumber(matcher.group(3)) + "99"; assertEquals( diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index b7a5b665ce58f..625c536a1c0d5 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -10,7 +10,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -44,21 +44,21 @@ public class CodecTests extends ESTestCase { public void testResolveDefaultCodecs() throws Exception { CodecService codecService = createCodecService(); assertThat(codecService.codec("default"), instanceOf(PerFieldMapperCodec.class)); - assertThat(codecService.codec("default"), instanceOf(Lucene95Codec.class)); + assertThat(codecService.codec("default"), instanceOf(Lucene99Codec.class)); } public void testDefault() throws Exception { Codec codec = createCodecService().codec("default"); - assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_SPEED, codec); + assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec); } public void testBestCompression() throws Exception { Codec codec = createCodecService().codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); } // write some docs with it, inspect .si to see this was the used compression - private void assertStoredFieldsCompressionEquals(Lucene95Codec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene99Codec.Mode expected, Codec actual) throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(null); iwc.setCodec(actual); @@ -70,7 +70,7 @@ private void assertStoredFieldsCompressionEquals(Lucene95Codec.Mode expected, Co SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader(); String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene95Codec.Mode.valueOf(v)); + assertEquals(expected, Lucene99Codec.Mode.valueOf(v)); ir.close(); dir.close(); } diff --git a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java index adb6ef77f2873..e2a2c72d3eae3 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java @@ -8,7 +8,7 @@ package org.elasticsearch.index.codec; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; @@ -168,7 +168,7 @@ private PerFieldMapperCodec createCodec(boolean timestampField, boolean timeSeri """; mapperService.merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); } - return new PerFieldMapperCodec(Lucene95Codec.Mode.BEST_SPEED, mapperService, BigArrays.NON_RECYCLING_INSTANCE); + return new PerFieldMapperCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, BigArrays.NON_RECYCLING_INSTANCE); } public void testUseES87TSDBEncodingSettingDisabled() throws IOException { @@ -207,7 +207,7 @@ private PerFieldMapperCodec createCodec(boolean enableES87TSDBCodec, boolean tim settings.put(IndexSettings.TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING.getKey(), enableES87TSDBCodec); MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), settings.build(), "test"); mapperService.merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); - return new PerFieldMapperCodec(Lucene95Codec.Mode.BEST_SPEED, mapperService, BigArrays.NON_RECYCLING_INSTANCE); + return new PerFieldMapperCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, BigArrays.NON_RECYCLING_INSTANCE); } } diff --git a/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java b/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java index 2a72b1fe40ec6..7c2c40e078cb4 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java @@ -8,12 +8,12 @@ package org.elasticsearch.index.engine; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.search.suggest.document.Completion90PostingsFormat; +import org.apache.lucene.search.suggest.document.Completion99PostingsFormat; import org.apache.lucene.search.suggest.document.SuggestField; import org.apache.lucene.store.Directory; import org.elasticsearch.ElasticsearchException; @@ -43,8 +43,8 @@ public void testExceptionsAreNotCached() { public void testCompletionStatsCache() throws IOException, InterruptedException { final IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); - final PostingsFormat postingsFormat = new Completion90PostingsFormat(); - indexWriterConfig.setCodec(new Lucene95Codec() { + final PostingsFormat postingsFormat = new Completion99PostingsFormat(); + indexWriterConfig.setCodec(new Lucene99Codec() { @Override public PostingsFormat getPostingsFormatForField(String field) { return postingsFormat; // all fields are suggest fields diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index 99302e377b61f..1f473d0ade35b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -15,7 +15,7 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; -import org.apache.lucene.search.suggest.document.Completion90PostingsFormat; +import org.apache.lucene.search.suggest.document.Completion99PostingsFormat; import org.apache.lucene.search.suggest.document.CompletionAnalyzer; import org.apache.lucene.search.suggest.document.ContextSuggestField; import org.apache.lucene.search.suggest.document.FuzzyCompletionQuery; @@ -149,7 +149,7 @@ public void testPostingsFormat() throws IOException { Codec codec = codecService.codec("default"); assertThat(codec, instanceOf(PerFieldMapperCodec.class)); PerFieldMapperCodec perFieldCodec = (PerFieldMapperCodec) codec; - assertThat(perFieldCodec.getPostingsFormatForField("field"), instanceOf(Completion90PostingsFormat.class)); + assertThat(perFieldCodec.getPostingsFormatForField("field"), instanceOf(Completion99PostingsFormat.class)); } public void testDefaultConfiguration() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index 5fe3711b1d034..1602e76c1a5fd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -233,12 +233,12 @@ public void testDateRangeQueryUsingMappingFormat() { RangeFieldType fieldType = new RangeFieldType("field", formatter); final Query query = fieldType.rangeQuery(from, to, true, true, relation, null, fieldType.dateMathParser(), context); - assertEquals("field:", query.toString()); + assertThat(query.toString(), containsString("field:")); // compare lower and upper bounds with what we would get on a `date` field DateFieldType dateFieldType = new DateFieldType("field", DateFieldMapper.Resolution.MILLISECONDS, formatter); final Query queryOnDateField = dateFieldType.rangeQuery(from, to, true, true, relation, null, fieldType.dateMathParser(), context); - assertEquals("field:[1465975790000 TO 1466062190999]", queryOnDateField.toString()); + assertThat(queryOnDateField.toString(), containsString("field:[1465975790000 TO 1466062190999]")); } /** diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 5047d54a98213..c417ec995a20a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -56,8 +56,8 @@ import java.util.List; import java.util.Set; -import static org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat.DEFAULT_BEAM_WIDTH; -import static org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat.DEFAULT_MAX_CONN; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -504,6 +504,11 @@ public void testInvalidParameters() { ); assertThat(e.getMessage(), containsString("[index_options] requires field [type] to be configured")); + e = expectThrows( + MapperParsingException.class, + () -> createDocumentMapper(fieldMapping(b -> b.field("type", "dense_vector").field("dims", 3).field("element_type", "foo"))) + ); + assertThat(e.getMessage(), containsString("invalid element_type [foo]; available types are ")); e = expectThrows( MapperParsingException.class, () -> createDocumentMapper( @@ -514,18 +519,35 @@ public void testInvalidParameters() { .field("index", true) .startObject("index_options") .field("type", "hnsw") - .field("ef_construction", 100) + .startObject("foo") + .endObject() .endObject() ) ) ); - assertThat(e.getMessage(), containsString("[index_options] of type [hnsw] requires field [m] to be configured")); - + assertThat( + e.getMessage(), + containsString("Failed to parse mapping: Mapping definition for [field] has unsupported parameters: [foo : {}]") + ); e = expectThrows( MapperParsingException.class, - () -> createDocumentMapper(fieldMapping(b -> b.field("type", "dense_vector").field("dims", 3).field("element_type", "bytes"))) + () -> createDocumentMapper( + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", 3) + .field("element_type", "byte") + .field("similarity", "l2_norm") + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .endObject() + ) + ) + ); + assertThat( + e.getMessage(), + containsString("Failed to parse mapping: [element_type] cannot be [byte] when using index type [int8_hnsw]") ); - assertThat(e.getMessage(), containsString("invalid element_type [bytes]; available types are ")); } public void testInvalidParametersBeforeIndexedByDefault() { @@ -1055,6 +1077,8 @@ public void testFloatVectorQueryBoundaries() throws IOException { public void testKnnVectorsFormat() throws IOException { final int m = randomIntBetween(1, DEFAULT_MAX_CONN + 10); final int efConstruction = randomIntBetween(1, DEFAULT_BEAM_WIDTH + 10); + boolean setM = randomBoolean(); + boolean setEfConstruction = randomBoolean(); MapperService mapperService = createMapperService(fieldMapping(b -> { b.field("type", "dense_vector"); b.field("dims", 4); @@ -1062,19 +1086,59 @@ public void testKnnVectorsFormat() throws IOException { b.field("similarity", "dot_product"); b.startObject("index_options"); b.field("type", "hnsw"); + if (setM) { + b.field("m", m); + } + if (setEfConstruction) { + b.field("ef_construction", efConstruction); + } + b.endObject(); + })); + CodecService codecService = new CodecService(mapperService, BigArrays.NON_RECYCLING_INSTANCE); + Codec codec = codecService.codec("default"); + assertThat(codec, instanceOf(PerFieldMapperCodec.class)); + KnnVectorsFormat knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); + String expectedString = "Lucene99HnswVectorsFormat(name=Lucene99HnswVectorsFormat, maxConn=" + + (setM ? m : DEFAULT_MAX_CONN) + + ", beamWidth=" + + (setEfConstruction ? efConstruction : DEFAULT_BEAM_WIDTH) + + ", flatVectorFormat=Lucene99FlatVectorsFormat()" + + ")"; + assertEquals(expectedString, knnVectorsFormat.toString()); + } + + public void testKnnQuantizedHNSWVectorsFormat() throws IOException { + final int m = randomIntBetween(1, DEFAULT_MAX_CONN + 10); + final int efConstruction = randomIntBetween(1, DEFAULT_BEAM_WIDTH + 10); + boolean setConfidenceInterval = randomBoolean(); + float confidenceInterval = (float) randomDoubleBetween(0.90f, 1.0f, true); + MapperService mapperService = createMapperService(fieldMapping(b -> { + b.field("type", "dense_vector"); + b.field("dims", 4); + b.field("index", true); + b.field("similarity", "dot_product"); + b.startObject("index_options"); + b.field("type", "int8_hnsw"); b.field("m", m); b.field("ef_construction", efConstruction); + if (setConfidenceInterval) { + b.field("confidence_interval", confidenceInterval); + } b.endObject(); })); CodecService codecService = new CodecService(mapperService, BigArrays.NON_RECYCLING_INSTANCE); Codec codec = codecService.codec("default"); assertThat(codec, instanceOf(PerFieldMapperCodec.class)); KnnVectorsFormat knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); - String expectedString = "Lucene95HnswVectorsFormat(name=Lucene95HnswVectorsFormat, maxConn=" + String expectedString = "Lucene99HnswScalarQuantizedVectorsFormat(name=Lucene99HnswScalarQuantizedVectorsFormat, maxConn=" + m + ", beamWidth=" + efConstruction - + ")"; + + ", flatVectorFormat=Lucene99ScalarQuantizedVectorsFormat(" + + "name=Lucene99ScalarQuantizedVectorsFormat, confidenceInterval=" + + (setConfidenceInterval ? confidenceInterval : null) + + ", rawVectorFormat=Lucene99FlatVectorsFormat()" + + "))"; assertEquals(expectedString, knnVectorsFormat.toString()); } diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java index ef625706ffffe..adfc333e9dc7e 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Numbers; import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; @@ -67,7 +68,6 @@ import java.util.stream.Collectors; import static org.elasticsearch.repositories.RepositoryDataTests.generateRandomRepoData; -import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; @@ -204,7 +204,7 @@ public void testCorruptIndexLatestFile() throws Exception { for (int i = 0; i < 16; i++) { repository.blobContainer() - .writeBlob(randomPurpose(), BlobStoreRepository.INDEX_LATEST_BLOB, new BytesArray(buffer, 0, i), false); + .writeBlob(OperationPurpose.SNAPSHOT_METADATA, BlobStoreRepository.INDEX_LATEST_BLOB, new BytesArray(buffer, 0, i), false); if (i == 8) { assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(generation)); } else { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java index 35fe9c400888c..7d3799b2db35d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java @@ -11,6 +11,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.Pruning; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; @@ -367,7 +368,7 @@ private Comparator sortFieldsComparator(SortField[] sortFields) { FieldComparator[] comparators = new FieldComparator[sortFields.length]; for (int i = 0; i < sortFields.length; i++) { // Values passed to getComparator shouldn't matter - comparators[i] = sortFields[i].getComparator(0, false); + comparators[i] = sortFields[i].getComparator(0, Pruning.NONE); } return (lhs, rhs) -> { FieldDoc l = (FieldDoc) lhs; diff --git a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java index 9e6b6330d2f23..a4e52af5f43c2 100644 --- a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java @@ -45,7 +45,6 @@ import org.apache.lucene.search.Scorable; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.SimpleCollector; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHitCountCollectorManager; @@ -58,13 +57,10 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.SparseFixedBitSet; -import org.apache.lucene.util.ThreadInterruptedException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.IOUtils; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; @@ -81,17 +77,11 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashSet; import java.util.IdentityHashMap; import java.util.List; import java.util.Set; -import java.util.concurrent.CopyOnWriteArraySet; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executor; import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.search.internal.ContextIndexSearcher.intersectScorerAndBitSet; import static org.elasticsearch.search.internal.ExitableDirectoryReader.ExitableLeafReader; @@ -525,330 +515,6 @@ public boolean isCacheable(LeafReaderContext ctx) { } } - /** - * Simulate one or more exceptions being thrown while collecting, through a custom query that throws IOException in its Weight#scorer. - * Verify that the slices that had to wait because there were no available threads in the pool are not started following the exception, - * which triggers a cancellation of all the tasks that are part of the running search. - * Simulate having N threads busy doing other work (e.g. other searches) otherwise all slices can be executed directly, given that - * the number of slices is dependent on the max pool size. - */ - public void testCancelSliceTasksOnException() throws Exception { - try (Directory dir = newDirectory()) { - indexDocs(dir); - int numThreads = randomIntBetween(4, 6); - int numBusyThreads = randomIntBetween(0, 3); - int numAvailableThreads = numThreads - numBusyThreads; - ThreadPoolExecutor executor = EsExecutors.newFixed( - ContextIndexSearcherTests.class.getName(), - numThreads, - -1, - EsExecutors.daemonThreadFactory(""), - new ThreadContext(Settings.EMPTY), - EsExecutors.TaskTrackingConfig.DO_NOT_TRACK - ); - ExecutorTestWrapper executorTestWrapper = new ExecutorTestWrapper(executor, numBusyThreads); - try (DirectoryReader directoryReader = DirectoryReader.open(dir)) { - Set throwingLeaves = new HashSet<>(); - Set scoredLeaves = new CopyOnWriteArraySet<>(); - final int[] newCollectorsCalls; - final boolean[] reduceCalled; - LeafSlice[] leafSlices; - try ( - ContextIndexSearcher contextIndexSearcher = new ContextIndexSearcher( - directoryReader, - IndexSearcher.getDefaultSimilarity(), - IndexSearcher.getDefaultQueryCache(), - IndexSearcher.getDefaultQueryCachingPolicy(), - true, - executorTestWrapper, - executor.getMaximumPoolSize(), - 1 - ) - ) { - leafSlices = contextIndexSearcher.getSlices(); - int numThrowingLeafSlices = randomIntBetween(1, 3); - for (int i = 0; i < numThrowingLeafSlices; i++) { - LeafSlice throwingLeafSlice = leafSlices[randomIntBetween(0, Math.min(leafSlices.length, numAvailableThreads) - 1)]; - throwingLeaves.add(randomFrom(throwingLeafSlice.leaves)); - } - Query query = new TestQuery() { - @Override - public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) { - return new ConstantScoreWeight(this, boost) { - @Override - public Scorer scorer(LeafReaderContext context) throws IOException { - if (throwingLeaves.contains(context)) { - // a random segment of some random slices throws exception. Other slices may or may not have started - throw new IOException(); - } - scoredLeaves.add(context); - return new ConstantScoreScorer( - this, - boost, - ScoreMode.COMPLETE, - DocIdSetIterator.all(context.reader().maxDoc()) - ); - } - - @Override - public boolean isCacheable(LeafReaderContext ctx) { - return false; - } - }; - } - }; - newCollectorsCalls = new int[] { 0 }; - reduceCalled = new boolean[] { false }; - CollectorManager collectorManager = new CollectorManager<>() { - @Override - public Collector newCollector() { - newCollectorsCalls[0]++; - return new SimpleCollector() { - @Override - public void collect(int doc) { - - } - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE; - } - }; - } - - @Override - public Integer reduce(Collection collectors) { - reduceCalled[0] = true; - return null; - } - }; - expectThrows(IOException.class, () -> contextIndexSearcher.search(query, collectorManager)); - assertBusy(() -> { - // active count is approximate, wait until it converges to the expected number - if (executor.getActiveCount() > numBusyThreads) { - throw new AssertionError("no search tasks should be left running"); - } - }); - } - // as many tasks as slices have been created - assertEquals(leafSlices.length, newCollectorsCalls[0]); - // unexpected exception thrown, reduce is not called, there are no results to return - assertFalse(reduceCalled[0]); - Set expectedScoredLeaves = new HashSet<>(); - // the first N slices, where N is the number of available permits, will run straight-away, the others will be cancelled - for (int i = 0; i < leafSlices.length; i++) { - if (i == numAvailableThreads) { - break; - } - LeafSlice leafSlice = leafSlices[i]; - for (LeafReaderContext context : leafSlice.leaves) { - // collect the segments that we expect to score in each slice, and stop at those that throw - if (throwingLeaves.contains(context)) { - break; - } - expectedScoredLeaves.add(context); - } - } - // The slice that threw exception is not counted. The others that could be executed directly are, but they may have been - // cancelled before they could even start, hence we are going to score at most the segments that the slices that can be - // executed straight-away (before reaching the max pool size) are made of. We can't guarantee that we score all of them. - // We do want to guarantee that the remaining slices won't even start and none of their leaves are scored. - assertTrue(expectedScoredLeaves.containsAll(scoredLeaves)); - } finally { - executorTestWrapper.stopBusyThreads(); - terminate(executor); - } - } - } - - /** - * Simulate one or more timeout being thrown while collecting, through a custom query that times out in its Weight#scorer. - * Verify that the slices that had to wait because there were no available threads in the pool are not started following the timeout, - * which triggers a cancellation of all the tasks that are part of the running search. - * Simulate having N threads busy doing other work (e.g. other searches) otherwise all slices can be executed directly, given that - * the number of slices is dependent on the max pool size. - */ - public void testCancelSliceTasksOnTimeout() throws Exception { - try (Directory dir = newDirectory()) { - indexDocs(dir); - int numThreads = randomIntBetween(4, 6); - int numBusyThreads = randomIntBetween(0, 3); - int numAvailableThreads = numThreads - numBusyThreads; - ThreadPoolExecutor executor = EsExecutors.newFixed( - ContextIndexSearcherTests.class.getName(), - numThreads, - -1, - EsExecutors.daemonThreadFactory(""), - new ThreadContext(Settings.EMPTY), - EsExecutors.TaskTrackingConfig.DO_NOT_TRACK - ); - ExecutorTestWrapper executorTestWrapper = new ExecutorTestWrapper(executor, numBusyThreads); - try (DirectoryReader directoryReader = DirectoryReader.open(dir)) { - Set throwingLeaves = new HashSet<>(); - Set scoredLeaves = new CopyOnWriteArraySet<>(); - final int[] newCollectorsCalls; - final boolean[] reduceCalled; - LeafSlice[] leafSlices; - try ( - ContextIndexSearcher contextIndexSearcher = new ContextIndexSearcher( - directoryReader, - IndexSearcher.getDefaultSimilarity(), - IndexSearcher.getDefaultQueryCache(), - IndexSearcher.getDefaultQueryCachingPolicy(), - true, - executorTestWrapper, - executor.getMaximumPoolSize(), - 1 - ) - ) { - leafSlices = contextIndexSearcher.getSlices(); - int numThrowingLeafSlices = randomIntBetween(1, 3); - for (int i = 0; i < numThrowingLeafSlices; i++) { - LeafSlice throwingLeafSlice = leafSlices[randomIntBetween(0, Math.min(leafSlices.length, numAvailableThreads) - 1)]; - throwingLeaves.add(randomFrom(throwingLeafSlice.leaves)); - } - Query query = new TestQuery() { - @Override - public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) { - return new ConstantScoreWeight(this, boost) { - @Override - public Scorer scorer(LeafReaderContext context) { - if (throwingLeaves.contains(context)) { - // a random segment of some random slices throws exception. Other slices may or may not have - // started. - contextIndexSearcher.throwTimeExceededException(); - } - scoredLeaves.add(context); - return new ConstantScoreScorer( - this, - boost, - ScoreMode.COMPLETE, - DocIdSetIterator.all(context.reader().maxDoc()) - ); - } - - @Override - public boolean isCacheable(LeafReaderContext ctx) { - return false; - } - }; - } - }; - newCollectorsCalls = new int[] { 0 }; - reduceCalled = new boolean[] { false }; - CollectorManager collectorManager = new CollectorManager<>() { - @Override - public Collector newCollector() { - newCollectorsCalls[0]++; - return new SimpleCollector() { - @Override - public void collect(int doc) { - - } - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE; - } - }; - } - - @Override - public Integer reduce(Collection collectors) { - reduceCalled[0] = true; - return null; - } - }; - contextIndexSearcher.search(query, collectorManager); - assertBusy(() -> { - // active count is approximate, wait until it converges to the expected number - if (executor.getActiveCount() > numBusyThreads) { - throw new AssertionError("no search tasks should be left running"); - } - }); - assertTrue(contextIndexSearcher.timeExceeded()); - } - // as many tasks as slices have been created - assertEquals(leafSlices.length, newCollectorsCalls[0]); - assertTrue(reduceCalled[0]); - Set expectedScoredLeaves = new HashSet<>(); - // the first N slices, where N is the number of available permits, will run straight-away, the others will be cancelled - for (int i = 0; i < leafSlices.length; i++) { - if (i == numAvailableThreads) { - break; - } - LeafSlice leafSlice = leafSlices[i]; - for (LeafReaderContext context : leafSlice.leaves) { - // collect the segments that we expect to score in each slice, and stop at those that throw - if (throwingLeaves.contains(context)) { - break; - } - expectedScoredLeaves.add(context); - } - } - // The slice that timed out is not counted. The others that could be executed directly are, but they may have been - // cancelled before they could even start, hence we are going to score at most the segments that the slices that can be - // executed straight-away (before reaching the max pool size) are made of. We can't guarantee that we score all of them. - // We do want to guarantee that the remaining slices won't even start and none of their leaves are scored. - assertTrue(expectedScoredLeaves.containsAll(scoredLeaves)); - } finally { - executorTestWrapper.stopBusyThreads(); - terminate(executor); - } - } - } - - private static class ExecutorTestWrapper implements Executor { - private final ThreadPoolExecutor executor; - private final AtomicInteger startedTasks = new AtomicInteger(0); - private final CountDownLatch busyThreadsLatch = new CountDownLatch(1); - - ExecutorTestWrapper(ThreadPoolExecutor executor, int numBusyThreads) { - this.executor = executor; - // keep some of the threads occupied to simulate the situation where the slices tasks get queued up. - // This is a realistic scenario that does not get tested otherwise by executing a single concurrent search, given that the - // number of slices is capped by max pool size. - for (int i = 0; i < numBusyThreads; i++) { - execute(() -> { - try { - busyThreadsLatch.await(); - } catch (InterruptedException e) { - throw new ThreadInterruptedException(e); - } - }); - } - } - - void stopBusyThreads() { - busyThreadsLatch.countDown(); - } - - @Override - public void execute(Runnable command) { - int started = startedTasks.incrementAndGet(); - if (started > executor.getMaximumPoolSize()) { - try { - /* - There could be tasks that complete quickly before the exception is handled, which leaves room for new tasks that are - about to get cancelled to start before their cancellation becomes effective. We can accept that cancellation may or may - not be effective for the slices that belong to the first batch of tasks until all threads are busy and adjust the - test expectations accordingly, but for the subsequent slices, we want to assert that they are cancelled and never - executed. The only way to guarantee that is waiting for cancellation to kick in. - */ - assertBusy(() -> { - Future future = (Future) command; - if (future.isCancelled() == false) { - throw new AssertionError("task should be cancelled"); - } - }); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - executor.execute(command); - } - } - private static class TestQuery extends Query { @Override public String toString(String field) { diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfileShardResultTests.java b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfileShardResultTests.java index f8c8d38e92805..f28425172ead5 100644 --- a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfileShardResultTests.java +++ b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfileShardResultTests.java @@ -33,7 +33,9 @@ public static QueryProfileShardResult createTestItem() { if (randomBoolean()) { rewriteTime = rewriteTime % 1000; // make sure to often test this with small values too } - return new QueryProfileShardResult(queryProfileResults, rewriteTime, profileCollector); + + Long vectorOperationsCount = randomBoolean() ? null : randomNonNegativeLong(); + return new QueryProfileShardResult(queryProfileResults, rewriteTime, profileCollector, vectorOperationsCount); } @Override diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java index fafe66c743ce8..ffa54184d652f 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java @@ -44,6 +44,7 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.Pruning; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.ScoreDoc; @@ -689,7 +690,6 @@ public void testIndexSortScrollOptimization() throws Exception { assertThat(context.queryResult().getTotalHits().value, equalTo((long) numDocs)); int sizeMinus1 = context.queryResult().topDocs().topDocs.scoreDocs.length - 1; FieldDoc lastDoc = (FieldDoc) context.queryResult().topDocs().topDocs.scoreDocs[sizeMinus1]; - context.setSearcher(earlyTerminationContextSearcher(reader, 10)); QueryPhase.addCollectorsAndSearch(context); assertNull(context.queryResult().terminatedEarly()); @@ -701,7 +701,7 @@ public void testIndexSortScrollOptimization() throws Exception { @SuppressWarnings("unchecked") FieldComparator comparator = (FieldComparator) searchSortAndFormat.sort.getSort()[i].getComparator( 1, - i == 0 + i == 0 ? Pruning.GREATER_THAN : Pruning.NONE ); int cmp = comparator.compareValues(firstDoc.fields[i], lastDoc.fields[i]); if (cmp == 0) { diff --git a/server/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java b/server/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java index 74c4b991ff401..ff963835f55f6 100644 --- a/server/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java @@ -11,6 +11,7 @@ import org.apache.lucene.document.LatLonDocValuesField; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.Pruning; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortedNumericSortField; @@ -216,7 +217,7 @@ public SortField.Type reducedType() { } @Override - public FieldComparator newComparator(String fieldname, int numHits, boolean enableSkipping, boolean reversed) { + public FieldComparator newComparator(String fieldname, int numHits, Pruning enableSkipping, boolean reversed) { return null; } diff --git a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java index 0bb170ed04430..474f891767081 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java @@ -10,8 +10,6 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.KnnByteVectorQuery; -import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.Query; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; @@ -101,13 +99,13 @@ protected void doAssertLuceneQuery(KnnVectorQueryBuilder queryBuilder, Query que Query knnQuery = ((VectorSimilarityQuery) query).getInnerKnnQuery(); assertThat(((VectorSimilarityQuery) query).getSimilarity(), equalTo(queryBuilder.getVectorSimilarity())); switch (elementType()) { - case FLOAT -> assertTrue(knnQuery instanceof KnnFloatVectorQuery); - case BYTE -> assertTrue(knnQuery instanceof KnnByteVectorQuery); + case FLOAT -> assertTrue(knnQuery instanceof ProfilingKnnFloatVectorQuery); + case BYTE -> assertTrue(knnQuery instanceof ProfilingKnnByteVectorQuery); } } else { switch (elementType()) { - case FLOAT -> assertTrue(query instanceof KnnFloatVectorQuery); - case BYTE -> assertTrue(query instanceof KnnByteVectorQuery); + case FLOAT -> assertTrue(query instanceof ProfilingKnnFloatVectorQuery); + case BYTE -> assertTrue(query instanceof ProfilingKnnByteVectorQuery); } } @@ -119,13 +117,13 @@ protected void doAssertLuceneQuery(KnnVectorQueryBuilder queryBuilder, Query que Query filterQuery = booleanQuery.clauses().isEmpty() ? null : booleanQuery; // The field should always be resolved to the concrete field Query knnVectorQueryBuilt = switch (elementType()) { - case BYTE -> new KnnByteVectorQuery( + case BYTE -> new ProfilingKnnByteVectorQuery( VECTOR_FIELD, getByteQueryVector(queryBuilder.queryVector()), queryBuilder.numCands(), filterQuery ); - case FLOAT -> new KnnFloatVectorQuery(VECTOR_FIELD, queryBuilder.queryVector(), queryBuilder.numCands(), filterQuery); + case FLOAT -> new ProfilingKnnFloatVectorQuery(VECTOR_FIELD, queryBuilder.queryVector(), queryBuilder.numCands(), filterQuery); }; if (query instanceof VectorSimilarityQuery vectorSimilarityQuery) { query = vectorSimilarityQuery.getInnerKnnQuery(); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 5b59040bbb04d..19f0d1e2e88a0 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -273,7 +273,11 @@ public void verifyReposThenStopServices() { (BlobStoreRepository) testClusterNodes.randomMasterNodeSafe().repositoriesService.repository("repo") ); deterministicTaskQueue.runAllRunnableTasks(); - assertNull(future.result()); + assertTrue(future.isDone()); + final var result = future.result(); + if (result != null) { + fail(result); + } } finally { testClusterNodes.nodes.values().forEach(TestClusterNodes.TestClusterNode::stop); } diff --git a/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/SeekTrackingDirectoryWrapper.java b/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/SeekTrackingDirectoryWrapper.java index 9b1991b52e500..9b3d31022c589 100644 --- a/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/SeekTrackingDirectoryWrapper.java +++ b/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/SeekTrackingDirectoryWrapper.java @@ -143,6 +143,11 @@ public RandomAccessInput randomAccessSlice(long offset, long length) throws IOEx IndexInput slice = wrapIndexInput(directory, name, innerSlice); // return default impl return new RandomAccessInput() { + @Override + public long length() { + return slice.length(); + } + @Override public byte readByte(long pos) throws IOException { slice.seek(pos); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index aecd81882c108..63a726d83f79e 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -10,7 +10,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig; @@ -245,7 +245,7 @@ protected static void withLuceneIndex( CheckedConsumer test ) throws IOException { IndexWriterConfig iwc = new IndexWriterConfig(IndexShard.buildIndexAnalyzer(mapperService)).setCodec( - new PerFieldMapperCodec(Lucene95Codec.Mode.BEST_SPEED, mapperService, BigArrays.NON_RECYCLING_INSTANCE) + new PerFieldMapperCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, BigArrays.NON_RECYCLING_INSTANCE) ); try (Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc)) { builder.accept(iw); diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java index 15f33131fa114..3d4dea430a9b5 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java @@ -36,6 +36,7 @@ import java.util.Set; import java.util.concurrent.Executor; +import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomNonDataPurpose; import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.contains; @@ -275,7 +276,7 @@ private static void createDanglingIndex(final BlobStoreRepository repo, final Ex .writeBlob(randomPurpose(), "bar", new ByteArrayInputStream(new byte[3]), 3, false); for (String prefix : Arrays.asList("snap-", "meta-")) { blobStore.blobContainer(repo.basePath()) - .writeBlob(randomPurpose(), prefix + "foo.dat", new ByteArrayInputStream(new byte[3]), 3, false); + .writeBlob(randomNonDataPurpose(), prefix + "foo.dat", new ByteArrayInputStream(new byte[3]), 3, false); } })); future.get(); @@ -285,8 +286,8 @@ private static void createDanglingIndex(final BlobStoreRepository repo, final Ex final BlobStore blobStore = repo.blobStore(); return blobStore.blobContainer(repo.basePath().add("indices")).children(randomPurpose()).containsKey("foo") && blobStore.blobContainer(repo.basePath().add("indices").add("foo")).blobExists(randomPurpose(), "bar") - && blobStore.blobContainer(repo.basePath()).blobExists(randomPurpose(), "meta-foo.dat") - && blobStore.blobContainer(repo.basePath()).blobExists(randomPurpose(), "snap-foo.dat"); + && blobStore.blobContainer(repo.basePath()).blobExists(randomNonDataPurpose(), "meta-foo.dat") + && blobStore.blobContainer(repo.basePath()).blobExists(randomNonDataPurpose(), "snap-foo.dat"); })); assertTrue(corruptionFuture.get()); } diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java index 383c2b3c2d13b..79e4a8da713c5 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java @@ -67,6 +67,7 @@ import static org.apache.lucene.tests.util.LuceneTestCase.random; import static org.elasticsearch.test.ESTestCase.randomFrom; import static org.elasticsearch.test.ESTestCase.randomIntBetween; +import static org.elasticsearch.test.ESTestCase.randomValueOtherThan; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasKey; @@ -105,7 +106,7 @@ public static PlainActionFuture assertConsistencyAsync(BlobStore try { final BlobContainer blobContainer = repository.blobContainer(); final long latestGen; - try (DataInputStream inputStream = new DataInputStream(blobContainer.readBlob(randomPurpose(), "index.latest"))) { + try (DataInputStream inputStream = new DataInputStream(blobContainer.readBlob(randomNonDataPurpose(), "index.latest"))) { latestGen = inputStream.readLong(); } catch (NoSuchFileException e) { throw new AssertionError("Could not find index.latest blob for repo [" + repository + "]"); @@ -113,7 +114,7 @@ public static PlainActionFuture assertConsistencyAsync(BlobStore assertIndexGenerations(blobContainer, latestGen); final RepositoryData repositoryData; try ( - InputStream blob = blobContainer.readBlob(randomPurpose(), BlobStoreRepository.INDEX_FILE_PREFIX + latestGen); + InputStream blob = blobContainer.readBlob(randomNonDataPurpose(), BlobStoreRepository.INDEX_FILE_PREFIX + latestGen); XContentParser parser = XContentType.JSON.xContent() .createParser(XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE), blob) ) { @@ -462,4 +463,8 @@ private static ClusterService mockClusterService(ClusterState initialState) { public static OperationPurpose randomPurpose() { return randomFrom(OperationPurpose.values()); } + + public static OperationPurpose randomNonDataPurpose() { + return randomValueOtherThan(OperationPurpose.SNAPSHOT_DATA, BlobStoreTestUtil::randomPurpose); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java index 578a7898bcd1e..a2499c06d6ccc 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; +import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.blobstore.support.BlobMetadata; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; @@ -62,6 +63,7 @@ import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.READONLY_SETTING_KEY; import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.SNAPSHOT_INDEX_NAME_FORMAT; import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.SNAPSHOT_NAME_FORMAT; +import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomNonDataPurpose; import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; @@ -228,7 +230,7 @@ public static void writeBlob( if (randomBoolean()) { container.writeBlob(randomPurpose(), blobName, bytesArray, failIfAlreadyExists); } else { - container.writeBlobAtomic(randomPurpose(), blobName, bytesArray, failIfAlreadyExists); + container.writeBlobAtomic(randomNonDataPurpose(), blobName, bytesArray, failIfAlreadyExists); } } @@ -556,7 +558,7 @@ public void testDanglingShardLevelBlobCleanup() throws Exception { // Create an extra dangling blob as if from an earlier snapshot that failed to clean up shardContainer.writeBlob( - randomPurpose(), + OperationPurpose.SNAPSHOT_DATA, BlobStoreRepository.UPLOADED_DATA_BLOB_PREFIX + UUIDs.randomBase64UUID(random()), BytesArray.EMPTY, true diff --git a/test/framework/src/main/java/org/elasticsearch/search/geo/GeoDistanceQueryBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/geo/GeoDistanceQueryBuilderTestCase.java index c9520bcfd051e..3866a57761fef 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/geo/GeoDistanceQueryBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/geo/GeoDistanceQueryBuilderTestCase.java @@ -325,9 +325,9 @@ private void assertGeoDistanceRangeQuery(String query, double lat, double lon, d // so we cannot access its fields directly to check and have to use toString() here instead. double qLat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat)); double qLon = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lon)); - assertEquals( + assertThat( parsedQuery.toString(), - "mapped_geo_point:" + qLat + "," + qLon + " +/- " + distanceUnit.toMeters(distance) + " meters" + containsString("mapped_geo_point:" + qLat + "," + qLon + " +/- " + distanceUnit.toMeters(distance) + " meters") ); } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java index b83cc7bba06e5..ff7195f9f5f37 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java @@ -16,7 +16,7 @@ */ public enum FeatureFlag { TIME_SERIES_MODE("es.index_mode_feature_flag_registered=true", Version.fromString("8.0.0"), null), - INFERENCE_RESCORER("es.inference_rescorer_feature_flag_enabled=true", Version.fromString("8.10.0"), null), + LEARN_TO_RANK("es.learn_to_rank_feature_flag_enabled=true", Version.fromString("8.10.0"), null), FAILURE_STORE_ENABLED("es.failure_store_feature_flag_enabled=true", Version.fromString("8.12.0"), null); public final String systemProperty; diff --git a/x-pack/plugin/core/src/main/java/module-info.java b/x-pack/plugin/core/src/main/java/module-info.java index 4aa2e145228b8..f747d07224454 100644 --- a/x-pack/plugin/core/src/main/java/module-info.java +++ b/x-pack/plugin/core/src/main/java/module-info.java @@ -75,6 +75,7 @@ exports org.elasticsearch.xpack.core.indexing; exports org.elasticsearch.xpack.core.inference.action; exports org.elasticsearch.xpack.core.inference.results; + exports org.elasticsearch.xpack.core.inference; exports org.elasticsearch.xpack.core.logstash; exports org.elasticsearch.xpack.core.ml.action; exports org.elasticsearch.xpack.core.ml.annotations; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java index 50485ecc21d9a..c332694d93975 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshot.java @@ -234,6 +234,7 @@ private SegmentCommitInfo syncSegment( si.name, si.maxDoc(), false, + si.getHasBlocks(), si.getCodec(), si.getDiagnostics(), si.getId(), diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index ac16631bacb73..df19648307a0b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -55,6 +55,7 @@ import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType; import org.elasticsearch.xpack.core.ilm.UnfollowAction; import org.elasticsearch.xpack.core.ilm.WaitForSnapshotAction; +import org.elasticsearch.xpack.core.inference.InferenceFeatureSetUsage; import org.elasticsearch.xpack.core.logstash.LogstashFeatureSetUsage; import org.elasticsearch.xpack.core.ml.MachineLearningFeatureSetUsage; import org.elasticsearch.xpack.core.ml.MlMetadata; @@ -133,6 +134,8 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.LOGSTASH, LogstashFeatureSetUsage::new), // ML new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.MACHINE_LEARNING, MachineLearningFeatureSetUsage::new), + // inference + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.INFERENCE, InferenceFeatureSetUsage::new), // monitoring new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.MONITORING, MonitoringFeatureSetUsage::new), // security diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java index c8a78af429592..801ef2c463e95 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java @@ -18,6 +18,8 @@ public final class XPackField { public static final String GRAPH = "graph"; /** Name constant for the machine learning feature. */ public static final String MACHINE_LEARNING = "ml"; + /** Name constant for the inference feature. */ + public static final String INFERENCE = "inference"; /** Name constant for the Logstash feature. */ public static final String LOGSTASH = "logstash"; /** Name constant for the Beats feature. */ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java index d96fd91ed3f22..c0e6d96c1569a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java @@ -27,6 +27,7 @@ public class XPackUsageFeatureAction extends ActionType modelStats; + + public InferenceFeatureSetUsage(Collection modelStats) { + super(XPackField.INFERENCE, true, true); + this.modelStats = modelStats; + } + + public InferenceFeatureSetUsage(StreamInput in) throws IOException { + super(in); + this.modelStats = in.readCollectionAsList(ModelStats::new); + } + + @Override + protected void innerXContent(XContentBuilder builder, Params params) throws IOException { + super.innerXContent(builder, params); + builder.xContentList("models", modelStats); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeCollection(modelStats); + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.INFERENCE_USAGE_ADDED; + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java index 20279e82d6c09..910ea5cab214d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java @@ -81,6 +81,11 @@ public Map asMap() { return map; } + @Override + public List transformToCoordinationFormat() { + return transformToLegacyFormat(); + } + @Override public List transformToLegacyFormat() { return embeddings.stream() diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingResults.java index 7a7ccab2b4daa..ace5974866038 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingResults.java @@ -78,6 +78,14 @@ public String getWriteableName() { return NAME; } + @Override + public List transformToCoordinationFormat() { + return embeddings.stream() + .map(embedding -> embedding.values.stream().mapToDouble(value -> value).toArray()) + .map(values -> new org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingResults(TEXT_EMBEDDING, values, false)) + .toList(); + } + @Override @SuppressWarnings("deprecation") public List transformToLegacyFormat() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/InferenceFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/InferenceFeatureSetUsageTests.java new file mode 100644 index 0000000000000..8f64b521c64c9 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/InferenceFeatureSetUsageTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.inference; + +import com.carrotsearch.randomizedtesting.generators.RandomStrings; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; + +public class InferenceFeatureSetUsageTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return InferenceFeatureSetUsage.ModelStats::new; + } + + @Override + protected InferenceFeatureSetUsage.ModelStats createTestInstance() { + RandomStrings.randomAsciiLettersOfLength(random(), 10); + return new InferenceFeatureSetUsage.ModelStats( + randomIdentifier(), + TaskType.values()[randomInt(TaskType.values().length - 1)], + randomInt(10) + ); + } + + @Override + protected InferenceFeatureSetUsage.ModelStats mutateInstance(InferenceFeatureSetUsage.ModelStats modelStats) throws IOException { + InferenceFeatureSetUsage.ModelStats newModelStats = new InferenceFeatureSetUsage.ModelStats(modelStats); + newModelStats.add(); + return newModelStats; + } +} diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/260_rule_query_search.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/260_rule_query_search.yml index b41636e624674..c287209da5bed 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/260_rule_query_search.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/260_rule_query_search.yml @@ -194,4 +194,46 @@ setup: - match: { hits.hits.0._id: 'doc2' } - match: { hits.hits.1._id: 'doc3' } +--- +"Perform a rule query over a ruleset with combined numeric and text rule matching": + + - do: + query_ruleset.put: + ruleset_id: combined-ruleset + body: + rules: + - rule_id: rule1 + type: pinned + criteria: + - type: fuzzy + metadata: foo + values: [ bar ] + actions: + ids: + - 'doc1' + - rule_id: rule2 + type: pinned + criteria: + - type: lte + metadata: foo + values: [ 100 ] + actions: + ids: + - 'doc2' + - do: + search: + body: + query: + rule_query: + organic: + query_string: + default_field: text + query: blah blah blah + match_criteria: + foo: baz + ruleset_id: combined-ruleset + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: 'doc1' } + diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/335_connector_update_error.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/335_connector_update_error.yml new file mode 100644 index 0000000000000..70021e3899525 --- /dev/null +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/335_connector_update_error.yml @@ -0,0 +1,60 @@ +setup: + - skip: + version: " - 8.11.99" + reason: Introduced in 8.12.0 + + - do: + connector.put: + connector_id: test-connector + body: + index_name: search-1-test + name: my-connector + language: pl + is_native: false + service_type: super-connector + +--- +"Update Connector Error": + - do: + connector.update_error: + connector_id: test-connector + body: + error: "some error" + + + - match: { result: updated } + + - do: + connector.get: + connector_id: test-connector + + - match: { error: "some error" } + +--- +"Update Connector Error - 404 when connector doesn't exist": + - do: + catch: "missing" + connector.update_error: + connector_id: test-non-existent-connector + body: + error: "some error" + +--- +"Update Connector Error - 400 status code when connector_id is empty": + - do: + catch: "bad_request" + connector.update_error: + connector_id: "" + body: + error: "some error" + +--- +"Update Connector Error - 400 status code when payload is not string": + - do: + catch: "bad_request" + connector.update_error: + connector_id: test-connector + body: + error: + field_1: test + field_2: something diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/440_connector_sync_job_get.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/440_connector_sync_job_get.yml new file mode 100644 index 0000000000000..ade0736436e87 --- /dev/null +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/440_connector_sync_job_get.yml @@ -0,0 +1,36 @@ +setup: + - skip: + version: " - 8.11.99" + reason: Introduced in 8.12.0 + - do: + connector.put: + connector_id: test-connector + body: + index_name: search-test + name: my-connector + language: de + is_native: false + service_type: super-connector + +--- +'Get connector sync job': + - do: + connector_sync_job.post: + body: + id: test-connector + job_type: access_control + trigger_method: scheduled + - set: { id: id } + - match: { id: $id } + - do: + connector_sync_job.get: + connector_sync_job_id: $id + - match: { job_type: access_control } + - match: { trigger_method: scheduled } + +--- +'Get connector sync job - Missing sync job id': + - do: + connector_sync_job.get: + connector_sync_job_id: non-existing-sync-job-id + catch: missing diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java index 2a53a46760868..f93177666f3d8 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java @@ -50,6 +50,7 @@ import org.elasticsearch.xpack.application.connector.action.RestGetConnectorAction; import org.elasticsearch.xpack.application.connector.action.RestListConnectorAction; import org.elasticsearch.xpack.application.connector.action.RestPutConnectorAction; +import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorErrorAction; import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorFilteringAction; import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorLastSeenAction; import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorLastSyncStatsAction; @@ -59,11 +60,13 @@ import org.elasticsearch.xpack.application.connector.action.TransportGetConnectorAction; import org.elasticsearch.xpack.application.connector.action.TransportListConnectorAction; import org.elasticsearch.xpack.application.connector.action.TransportPutConnectorAction; +import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorErrorAction; import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorFilteringAction; import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorLastSeenAction; import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorLastSyncStatsAction; import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorPipelineAction; import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorSchedulingAction; +import org.elasticsearch.xpack.application.connector.action.UpdateConnectorErrorAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorFilteringAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorLastSeenAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorLastSyncStatsAction; @@ -72,14 +75,17 @@ import org.elasticsearch.xpack.application.connector.syncjob.action.CancelConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.CheckInConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.DeleteConnectorSyncJobAction; +import org.elasticsearch.xpack.application.connector.syncjob.action.GetConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.PostConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.RestCancelConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.RestCheckInConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.RestDeleteConnectorSyncJobAction; +import org.elasticsearch.xpack.application.connector.syncjob.action.RestGetConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.RestPostConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.TransportCancelConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.TransportCheckInConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.TransportDeleteConnectorSyncJobAction; +import org.elasticsearch.xpack.application.connector.syncjob.action.TransportGetConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.TransportPostConnectorSyncJobAction; import org.elasticsearch.xpack.application.rules.QueryRulesConfig; import org.elasticsearch.xpack.application.rules.QueryRulesIndexService; @@ -201,6 +207,7 @@ protected XPackLicenseState getLicenseState() { new ActionHandler<>(GetConnectorAction.INSTANCE, TransportGetConnectorAction.class), new ActionHandler<>(ListConnectorAction.INSTANCE, TransportListConnectorAction.class), new ActionHandler<>(PutConnectorAction.INSTANCE, TransportPutConnectorAction.class), + new ActionHandler<>(UpdateConnectorErrorAction.INSTANCE, TransportUpdateConnectorErrorAction.class), new ActionHandler<>(UpdateConnectorFilteringAction.INSTANCE, TransportUpdateConnectorFilteringAction.class), new ActionHandler<>(UpdateConnectorLastSeenAction.INSTANCE, TransportUpdateConnectorLastSeenAction.class), new ActionHandler<>(UpdateConnectorLastSyncStatsAction.INSTANCE, TransportUpdateConnectorLastSyncStatsAction.class), @@ -208,6 +215,7 @@ protected XPackLicenseState getLicenseState() { new ActionHandler<>(UpdateConnectorSchedulingAction.INSTANCE, TransportUpdateConnectorSchedulingAction.class), // SyncJob API + new ActionHandler<>(GetConnectorSyncJobAction.INSTANCE, TransportGetConnectorSyncJobAction.class), new ActionHandler<>(PostConnectorSyncJobAction.INSTANCE, TransportPostConnectorSyncJobAction.class), new ActionHandler<>(DeleteConnectorSyncJobAction.INSTANCE, TransportDeleteConnectorSyncJobAction.class), new ActionHandler<>(CheckInConnectorSyncJobAction.INSTANCE, TransportCheckInConnectorSyncJobAction.class), @@ -267,6 +275,7 @@ public List getRestHandlers( new RestGetConnectorAction(), new RestListConnectorAction(), new RestPutConnectorAction(), + new RestUpdateConnectorErrorAction(), new RestUpdateConnectorFilteringAction(), new RestUpdateConnectorLastSeenAction(), new RestUpdateConnectorLastSyncStatsAction(), @@ -274,6 +283,7 @@ public List getRestHandlers( new RestUpdateConnectorSchedulingAction(), // SyncJob API + new RestGetConnectorSyncJobAction(), new RestPostConnectorSyncJobAction(), new RestDeleteConnectorSyncJobAction(), new RestCancelConnectorSyncJobAction(), diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java index 45b906d815aee..d68cc9f7227bc 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java @@ -200,14 +200,14 @@ public Connector(StreamInput in) throws IOException { public static final ParseField CONFIGURATION_FIELD = new ParseField("configuration"); static final ParseField CUSTOM_SCHEDULING_FIELD = new ParseField("custom_scheduling"); static final ParseField DESCRIPTION_FIELD = new ParseField("description"); - static final ParseField ERROR_FIELD = new ParseField("error"); + public static final ParseField ERROR_FIELD = new ParseField("error"); static final ParseField FEATURES_FIELD = new ParseField("features"); public static final ParseField FILTERING_FIELD = new ParseField("filtering"); public static final ParseField INDEX_NAME_FIELD = new ParseField("index_name"); static final ParseField IS_NATIVE_FIELD = new ParseField("is_native"); public static final ParseField LANGUAGE_FIELD = new ParseField("language"); public static final ParseField LAST_SEEN_FIELD = new ParseField("last_seen"); - static final ParseField NAME_FIELD = new ParseField("name"); + public static final ParseField NAME_FIELD = new ParseField("name"); public static final ParseField PIPELINE_FIELD = new ParseField("pipeline"); public static final ParseField SCHEDULING_FIELD = new ParseField("scheduling"); public static final ParseField SERVICE_TYPE_FIELD = new ParseField("service_type"); @@ -457,8 +457,28 @@ public String getConnectorId() { return connectorId; } - public ConnectorScheduling getScheduling() { - return scheduling; + public String getApiKeyId() { + return apiKeyId; + } + + public Map getConfiguration() { + return configuration; + } + + public Map getCustomScheduling() { + return customScheduling; + } + + public String getDescription() { + return description; + } + + public String getError() { + return error; + } + + public ConnectorFeatures getFeatures() { + return features; } public List getFiltering() { @@ -469,20 +489,40 @@ public String getIndexName() { return indexName; } + public boolean isNative() { + return isNative; + } + public String getLanguage() { return language; } + public String getName() { + return name; + } + public ConnectorIngestPipeline getPipeline() { return pipeline; } + public ConnectorScheduling getScheduling() { + return scheduling; + } + public String getServiceType() { return serviceType; } - public Map getConfiguration() { - return configuration; + public ConnectorStatus getStatus() { + return status; + } + + public Object getSyncCursor() { + return syncCursor; + } + + public boolean isSyncNow() { + return syncNow; } public ConnectorSyncInfo getSyncInfo() { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index d99ad28dc3970..744a4d2028990 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -31,6 +31,7 @@ import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.application.connector.action.UpdateConnectorErrorAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorFilteringAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorLastSeenAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorLastSyncStatsAction; @@ -323,6 +324,36 @@ public void updateConnectorScheduling(UpdateConnectorSchedulingAction.Request re } } + /** + * Updates the error property of a {@link Connector}. + * + * @param request The request for updating the connector's error. + * @param listener The listener for handling responses, including successful updates or errors. + */ + public void updateConnectorError(UpdateConnectorErrorAction.Request request, ActionListener listener) { + try { + String connectorId = request.getConnectorId(); + final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc( + new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) + .id(connectorId) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)) + ); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorId)); + return; + } + l.onResponse(updateResponse); + }) + ); + } catch (Exception e) { + listener.onFailure(e); + } + } + private static ConnectorIndexService.ConnectorResult mapSearchResponseToConnectorList(SearchResponse response) { final List connectorResults = Arrays.stream(response.getHits().getHits()) .map(ConnectorIndexService::hitToConnector) diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorErrorAction.java new file mode 100644 index 0000000000000..ea8bd1b4ee50f --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorErrorAction.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.application.EnterpriseSearch; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.PUT; + +public class RestUpdateConnectorErrorAction extends BaseRestHandler { + + @Override + public String getName() { + return "connector_update_error_action"; + } + + @Override + public List routes() { + return List.of(new Route(PUT, "/" + EnterpriseSearch.CONNECTOR_API_ENDPOINT + "/{connector_id}/_error")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { + UpdateConnectorErrorAction.Request request = UpdateConnectorErrorAction.Request.fromXContentBytes( + restRequest.param("connector_id"), + restRequest.content(), + restRequest.getXContentType() + ); + return channel -> client.execute( + UpdateConnectorErrorAction.INSTANCE, + request, + new RestToXContentListener<>(channel, UpdateConnectorErrorAction.Response::status, r -> null) + ); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorErrorAction.java new file mode 100644 index 0000000000000..629fd14861cf6 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorErrorAction.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.application.connector.ConnectorIndexService; + +public class TransportUpdateConnectorErrorAction extends HandledTransportAction< + UpdateConnectorErrorAction.Request, + UpdateConnectorErrorAction.Response> { + + protected final ConnectorIndexService connectorIndexService; + + @Inject + public TransportUpdateConnectorErrorAction( + TransportService transportService, + ClusterService clusterService, + ActionFilters actionFilters, + Client client + ) { + super( + UpdateConnectorErrorAction.NAME, + transportService, + actionFilters, + UpdateConnectorErrorAction.Request::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.connectorIndexService = new ConnectorIndexService(client); + } + + @Override + protected void doExecute( + Task task, + UpdateConnectorErrorAction.Request request, + ActionListener listener + ) { + connectorIndexService.updateConnectorError(request, listener.map(r -> new UpdateConnectorErrorAction.Response(r.getResult()))); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java new file mode 100644 index 0000000000000..c9e48dac08cd5 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java @@ -0,0 +1,186 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.application.connector.Connector; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class UpdateConnectorErrorAction extends ActionType { + + public static final UpdateConnectorErrorAction INSTANCE = new UpdateConnectorErrorAction(); + public static final String NAME = "cluster:admin/xpack/connector/update_error"; + + public UpdateConnectorErrorAction() { + super(NAME, UpdateConnectorErrorAction.Response::new); + } + + public static class Request extends ActionRequest implements ToXContentObject { + + private final String connectorId; + + @Nullable + private final String error; + + public Request(String connectorId, String error) { + this.connectorId = connectorId; + this.error = error; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.connectorId = in.readString(); + this.error = in.readOptionalString(); + } + + public String getConnectorId() { + return connectorId; + } + + public String getError() { + return error; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + + if (Strings.isNullOrEmpty(connectorId)) { + validationException = addValidationError("[connector_id] cannot be null or empty.", validationException); + } + + return validationException; + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "connector_update_error_request", + false, + ((args, connectorId) -> new UpdateConnectorErrorAction.Request(connectorId, (String) args[0])) + ); + + static { + PARSER.declareStringOrNull(optionalConstructorArg(), Connector.ERROR_FIELD); + } + + public static UpdateConnectorErrorAction.Request fromXContentBytes( + String connectorId, + BytesReference source, + XContentType xContentType + ) { + try (XContentParser parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, source, xContentType)) { + return UpdateConnectorErrorAction.Request.fromXContent(parser, connectorId); + } catch (IOException e) { + throw new ElasticsearchParseException("Failed to parse: " + source.utf8ToString(), e); + } + } + + public static UpdateConnectorErrorAction.Request fromXContent(XContentParser parser, String connectorId) throws IOException { + return PARSER.parse(parser, connectorId); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field(Connector.ERROR_FIELD.getPreferredName(), error); + } + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(connectorId); + out.writeOptionalString(error); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(connectorId, request.connectorId) && Objects.equals(error, request.error); + } + + @Override + public int hashCode() { + return Objects.hash(connectorId, error); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + final DocWriteResponse.Result result; + + public Response(StreamInput in) throws IOException { + super(in); + result = DocWriteResponse.Result.readFrom(in); + } + + public Response(DocWriteResponse.Result result) { + this.result = result; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + this.result.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("result", this.result.getLowercase()); + builder.endObject(); + return builder; + } + + public RestStatus status() { + return switch (result) { + case NOT_FOUND -> RestStatus.NOT_FOUND; + default -> RestStatus.OK; + }; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response that = (Response) o; + return Objects.equals(result, that.result); + } + + @Override + public int hashCode() { + return Objects.hash(result); + } + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java index 6c0e9635d986d..2a302ddb68199 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java @@ -7,22 +7,36 @@ package org.elasticsearch.xpack.application.connector.syncjob; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.application.connector.Connector; +import org.elasticsearch.xpack.application.connector.ConnectorFiltering; +import org.elasticsearch.xpack.application.connector.ConnectorIngestPipeline; import org.elasticsearch.xpack.application.connector.ConnectorSyncStatus; import java.io.IOException; import java.time.Instant; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Objects; +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; + /** * Represents a sync job in the Elasticsearch ecosystem. Sync jobs refer to a unit of work, which syncs data from a 3rd party * data source into an Elasticsearch index using the Connectors service. A ConnectorSyncJob always refers @@ -60,7 +74,7 @@ public class ConnectorSyncJob implements Writeable, ToXContentObject { static final ParseField CREATED_AT_FIELD = new ParseField("created_at"); - static final ParseField DELETED_DOCUMENT_COUNT = new ParseField("deleted_document_count"); + static final ParseField DELETED_DOCUMENT_COUNT_FIELD = new ParseField("deleted_document_count"); static final ParseField ERROR_FIELD = new ParseField("error"); @@ -92,6 +106,7 @@ public class ConnectorSyncJob implements Writeable, ToXContentObject { static final ConnectorSyncJobTriggerMethod DEFAULT_TRIGGER_METHOD = ConnectorSyncJobTriggerMethod.ON_DEMAND; + @Nullable private final Instant cancelationRequestedAt; @Nullable @@ -127,7 +142,6 @@ public class ConnectorSyncJob implements Writeable, ToXContentObject { private final ConnectorSyncStatus status; - @Nullable private final long totalDocumentCount; private final ConnectorSyncJobTriggerMethod triggerMethod; @@ -217,44 +231,269 @@ public ConnectorSyncJob(StreamInput in) throws IOException { this.workerHostname = in.readOptionalString(); } + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "connector_sync_job", + true, + (args) -> { + int i = 0; + return new Builder().setCancellationRequestedAt((Instant) args[i++]) + .setCanceledAt((Instant) args[i++]) + .setCompletedAt((Instant) args[i++]) + .setConnector((Connector) args[i++]) + .setCreatedAt((Instant) args[i++]) + .setDeletedDocumentCount((Long) args[i++]) + .setError((String) args[i++]) + .setId((String) args[i++]) + .setIndexedDocumentCount((Long) args[i++]) + .setIndexedDocumentVolume((Long) args[i++]) + .setJobType((ConnectorSyncJobType) args[i++]) + .setLastSeen((Instant) args[i++]) + .setMetadata((Map) args[i++]) + .setStartedAt((Instant) args[i++]) + .setStatus((ConnectorSyncStatus) args[i++]) + .setTotalDocumentCount((Long) args[i++]) + .setTriggerMethod((ConnectorSyncJobTriggerMethod) args[i++]) + .setWorkerHostname((String) args[i]) + .build(); + } + ); + + static { + PARSER.declareField( + optionalConstructorArg(), + (p, c) -> Instant.parse(p.text()), + CANCELATION_REQUESTED_AT_FIELD, + ObjectParser.ValueType.STRING_OR_NULL + ); + PARSER.declareField(optionalConstructorArg(), (p, c) -> Instant.parse(p.text()), CANCELED_AT_FIELD, ObjectParser.ValueType.STRING); + PARSER.declareField(optionalConstructorArg(), (p, c) -> Instant.parse(p.text()), COMPLETED_AT_FIELD, ObjectParser.ValueType.STRING); + PARSER.declareField( + constructorArg(), + (p, c) -> ConnectorSyncJob.syncJobConnectorFromXContent(p), + CONNECTOR_FIELD, + ObjectParser.ValueType.OBJECT + ); + PARSER.declareField(constructorArg(), (p, c) -> Instant.parse(p.text()), CREATED_AT_FIELD, ObjectParser.ValueType.STRING); + PARSER.declareLong(constructorArg(), DELETED_DOCUMENT_COUNT_FIELD); + PARSER.declareStringOrNull(optionalConstructorArg(), ERROR_FIELD); + PARSER.declareString(constructorArg(), ID_FIELD); + PARSER.declareLong(constructorArg(), INDEXED_DOCUMENT_COUNT_FIELD); + PARSER.declareLong(constructorArg(), INDEXED_DOCUMENT_VOLUME_FIELD); + PARSER.declareField( + constructorArg(), + (p, c) -> ConnectorSyncJobType.fromString(p.text()), + JOB_TYPE_FIELD, + ObjectParser.ValueType.STRING + ); + PARSER.declareField(constructorArg(), (p, c) -> Instant.parse(p.text()), LAST_SEEN_FIELD, ObjectParser.ValueType.STRING); + PARSER.declareField(constructorArg(), (p, c) -> p.map(), METADATA_FIELD, ObjectParser.ValueType.OBJECT); + PARSER.declareField(optionalConstructorArg(), (p, c) -> Instant.parse(p.text()), STARTED_AT_FIELD, ObjectParser.ValueType.STRING); + PARSER.declareField( + constructorArg(), + (p, c) -> ConnectorSyncStatus.fromString(p.text()), + STATUS_FIELD, + ObjectParser.ValueType.STRING + ); + PARSER.declareLong(constructorArg(), TOTAL_DOCUMENT_COUNT_FIELD); + PARSER.declareField( + constructorArg(), + (p, c) -> ConnectorSyncJobTriggerMethod.fromString(p.text()), + TRIGGER_METHOD_FIELD, + ObjectParser.ValueType.STRING + ); + PARSER.declareString(optionalConstructorArg(), WORKER_HOSTNAME_FIELD); + } + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser SYNC_JOB_CONNECTOR_PARSER = new ConstructingObjectParser<>( + "sync_job_connector", + true, + (args) -> { + int i = 0; + return new Connector.Builder().setConnectorId((String) args[i++]) + .setFiltering((List) args[i++]) + .setIndexName((String) args[i++]) + .setLanguage((String) args[i++]) + .setPipeline((ConnectorIngestPipeline) args[i++]) + .setServiceType((String) args[i++]) + .setConfiguration((Map) args[i++]) + .build(); + } + ); + + static { + SYNC_JOB_CONNECTOR_PARSER.declareString(constructorArg(), Connector.ID_FIELD); + SYNC_JOB_CONNECTOR_PARSER.declareObjectArray( + optionalConstructorArg(), + (p, c) -> ConnectorFiltering.fromXContent(p), + Connector.FILTERING_FIELD + ); + SYNC_JOB_CONNECTOR_PARSER.declareString(optionalConstructorArg(), Connector.INDEX_NAME_FIELD); + SYNC_JOB_CONNECTOR_PARSER.declareString(optionalConstructorArg(), Connector.LANGUAGE_FIELD); + SYNC_JOB_CONNECTOR_PARSER.declareField( + optionalConstructorArg(), + (p, c) -> ConnectorIngestPipeline.fromXContent(p), + Connector.PIPELINE_FIELD, + ObjectParser.ValueType.OBJECT + ); + SYNC_JOB_CONNECTOR_PARSER.declareString(optionalConstructorArg(), Connector.SERVICE_TYPE_FIELD); + SYNC_JOB_CONNECTOR_PARSER.declareField( + optionalConstructorArg(), + (parser, context) -> parser.map(), + Connector.CONFIGURATION_FIELD, + ObjectParser.ValueType.OBJECT + ); + } + + public static ConnectorSyncJob fromXContentBytes(BytesReference source, XContentType xContentType) { + try (XContentParser parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, source, xContentType)) { + return ConnectorSyncJob.fromXContent(parser); + } catch (IOException e) { + throw new ElasticsearchParseException("Failed to parse a connector sync job document.", e); + } + } + + public static ConnectorSyncJob fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + public static Connector syncJobConnectorFromXContent(XContentParser parser) throws IOException { + return SYNC_JOB_CONNECTOR_PARSER.parse(parser, null); + } + public String getId() { return id; } + public Instant getCancelationRequestedAt() { + return cancelationRequestedAt; + } + + public Instant getCanceledAt() { + return canceledAt; + } + + public Instant getCompletedAt() { + return completedAt; + } + + public Connector getConnector() { + return connector; + } + + public Instant getCreatedAt() { + return createdAt; + } + + public long getDeletedDocumentCount() { + return deletedDocumentCount; + } + + public String getError() { + return error; + } + + public long getIndexedDocumentCount() { + return indexedDocumentCount; + } + + public long getIndexedDocumentVolume() { + return indexedDocumentVolume; + } + + public ConnectorSyncJobType getJobType() { + return jobType; + } + + public Instant getLastSeen() { + return lastSeen; + } + + public Map getMetadata() { + return metadata; + } + + public Instant getStartedAt() { + return startedAt; + } + + public ConnectorSyncStatus getStatus() { + return status; + } + + public long getTotalDocumentCount() { + return totalDocumentCount; + } + + public ConnectorSyncJobTriggerMethod getTriggerMethod() { + return triggerMethod; + } + + public String getWorkerHostname() { + return workerHostname; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); { - builder.field(CANCELATION_REQUESTED_AT_FIELD.getPreferredName(), cancelationRequestedAt); - builder.field(CANCELED_AT_FIELD.getPreferredName(), canceledAt); - builder.field(COMPLETED_AT_FIELD.getPreferredName(), completedAt); + if (cancelationRequestedAt != null) { + builder.field(CANCELATION_REQUESTED_AT_FIELD.getPreferredName(), cancelationRequestedAt); + } + if (canceledAt != null) { + builder.field(CANCELED_AT_FIELD.getPreferredName(), canceledAt); + } + if (completedAt != null) { + builder.field(COMPLETED_AT_FIELD.getPreferredName(), completedAt); + } builder.startObject(CONNECTOR_FIELD.getPreferredName()); { builder.field(Connector.ID_FIELD.getPreferredName(), connector.getConnectorId()); - builder.field(Connector.FILTERING_FIELD.getPreferredName(), connector.getFiltering()); - builder.field(Connector.INDEX_NAME_FIELD.getPreferredName(), connector.getIndexName()); - builder.field(Connector.LANGUAGE_FIELD.getPreferredName(), connector.getLanguage()); - builder.field(Connector.PIPELINE_FIELD.getPreferredName(), connector.getPipeline()); - builder.field(Connector.SERVICE_TYPE_FIELD.getPreferredName(), connector.getServiceType()); - builder.field(Connector.CONFIGURATION_FIELD.getPreferredName(), connector.getConfiguration()); + if (connector.getFiltering() != null) { + builder.field(Connector.FILTERING_FIELD.getPreferredName(), connector.getFiltering()); + } + if (connector.getIndexName() != null) { + builder.field(Connector.INDEX_NAME_FIELD.getPreferredName(), connector.getIndexName()); + } + if (connector.getLanguage() != null) { + builder.field(Connector.LANGUAGE_FIELD.getPreferredName(), connector.getLanguage()); + } + if (connector.getPipeline() != null) { + builder.field(Connector.PIPELINE_FIELD.getPreferredName(), connector.getPipeline()); + } + if (connector.getServiceType() != null) { + builder.field(Connector.SERVICE_TYPE_FIELD.getPreferredName(), connector.getServiceType()); + } + if (connector.getConfiguration() != null) { + builder.field(Connector.CONFIGURATION_FIELD.getPreferredName(), connector.getConfiguration()); + } } builder.endObject(); builder.field(CREATED_AT_FIELD.getPreferredName(), createdAt); - builder.field(DELETED_DOCUMENT_COUNT.getPreferredName(), deletedDocumentCount); - builder.field(ERROR_FIELD.getPreferredName(), error); + builder.field(DELETED_DOCUMENT_COUNT_FIELD.getPreferredName(), deletedDocumentCount); + if (error != null) { + builder.field(ERROR_FIELD.getPreferredName(), error); + } builder.field(ID_FIELD.getPreferredName(), id); builder.field(INDEXED_DOCUMENT_COUNT_FIELD.getPreferredName(), indexedDocumentCount); builder.field(INDEXED_DOCUMENT_VOLUME_FIELD.getPreferredName(), indexedDocumentVolume); builder.field(JOB_TYPE_FIELD.getPreferredName(), jobType); - builder.field(LAST_SEEN_FIELD.getPreferredName(), lastSeen); + if (lastSeen != null) { + builder.field(LAST_SEEN_FIELD.getPreferredName(), lastSeen); + } builder.field(METADATA_FIELD.getPreferredName(), metadata); - builder.field(STARTED_AT_FIELD.getPreferredName(), startedAt); + if (startedAt != null) { + builder.field(STARTED_AT_FIELD.getPreferredName(), startedAt); + } builder.field(STATUS_FIELD.getPreferredName(), status); builder.field(TOTAL_DOCUMENT_COUNT_FIELD.getPreferredName(), totalDocumentCount); builder.field(TRIGGER_METHOD_FIELD.getPreferredName(), triggerMethod); - builder.field(WORKER_HOSTNAME_FIELD.getPreferredName(), workerHostname); + if (workerHostname != null) { + builder.field(WORKER_HOSTNAME_FIELD.getPreferredName(), workerHostname); + } } builder.endObject(); return builder; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java index ab593fe99fcee..5e1686dde80f2 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java @@ -27,6 +27,7 @@ import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.engine.DocumentMissingException; import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.application.connector.Connector; import org.elasticsearch.xpack.application.connector.ConnectorFiltering; import org.elasticsearch.xpack.application.connector.ConnectorIndexService; @@ -174,6 +175,40 @@ public void checkInConnectorSyncJob(String connectorSyncJobId, ActionListener listener) { + final GetRequest getRequest = new GetRequest(CONNECTOR_SYNC_JOB_INDEX_NAME).id(connectorSyncJobId).realtime(true); + + try { + clientWithOrigin.get( + getRequest, + new DelegatingIndexNotFoundOrDocumentMissingActionListener<>(connectorSyncJobId, listener, (l, getResponse) -> { + if (getResponse.isExists() == false) { + l.onFailure(new ResourceNotFoundException(connectorSyncJobId)); + return; + } + + try { + final ConnectorSyncJob syncJob = ConnectorSyncJob.fromXContentBytes( + getResponse.getSourceAsBytesRef(), + XContentType.JSON + ); + l.onResponse(syncJob); + } catch (Exception e) { + listener.onFailure(e); + } + }) + ); + } catch (Exception e) { + listener.onFailure(e); + } + } + /** * Cancels the {@link ConnectorSyncJob} in the underlying index. * Canceling means to set the {@link ConnectorSyncStatus} to "canceling" and not "canceled" as this is an async operation. @@ -211,7 +246,6 @@ public void cancelConnectorSyncJob(String connectorSyncJobId, ActionListener { + + public static final GetConnectorSyncJobAction INSTANCE = new GetConnectorSyncJobAction(); + public static final String NAME = "cluster:admin/xpack/connector/sync_job/get"; + + private GetConnectorSyncJobAction() { + super(NAME, GetConnectorSyncJobAction.Response::new); + } + + public static class Request extends ActionRequest implements ToXContentObject { + private final String connectorSyncJobId; + + private static final ParseField CONNECTOR_ID_FIELD = new ParseField("connector_id"); + + public Request(StreamInput in) throws IOException { + super(in); + this.connectorSyncJobId = in.readString(); + } + + public Request(String connectorSyncJobId) { + this.connectorSyncJobId = connectorSyncJobId; + } + + public String getConnectorSyncJobId() { + return connectorSyncJobId; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + + if (Strings.isNullOrEmpty(connectorSyncJobId)) { + validationException = addValidationError( + ConnectorSyncJobConstants.EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE, + validationException + ); + } + + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(connectorSyncJobId); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(connectorSyncJobId, request.connectorSyncJobId); + } + + @Override + public int hashCode() { + return Objects.hash(connectorSyncJobId); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(CONNECTOR_ID_FIELD.getPreferredName(), connectorSyncJobId); + builder.endObject(); + return builder; + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "get_connector_sync_job_request", + false, + (args) -> new Request((String) args[0]) + ); + + static { + PARSER.declareString(constructorArg(), CONNECTOR_ID_FIELD); + } + + public static Request parse(XContentParser parser) { + return PARSER.apply(parser, null); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + private final ConnectorSyncJob connectorSyncJob; + + public Response(ConnectorSyncJob connectorSyncJob) { + this.connectorSyncJob = connectorSyncJob; + } + + public Response(StreamInput in) throws IOException { + super(in); + this.connectorSyncJob = new ConnectorSyncJob(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + connectorSyncJob.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return connectorSyncJob.toXContent(builder, params); + } + + public static GetConnectorSyncJobAction.Response fromXContent(XContentParser parser) throws IOException { + return new GetConnectorSyncJobAction.Response(ConnectorSyncJob.fromXContent(parser)); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response response = (Response) o; + return Objects.equals(connectorSyncJob, response.connectorSyncJob); + } + + @Override + public int hashCode() { + return Objects.hash(connectorSyncJob); + } + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestGetConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestGetConnectorSyncJobAction.java new file mode 100644 index 0000000000000..1f5606810757e --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestGetConnectorSyncJobAction.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.syncjob.action; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.application.EnterpriseSearch; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobConstants.CONNECTOR_SYNC_JOB_ID_PARAM; + +public class RestGetConnectorSyncJobAction extends BaseRestHandler { + @Override + public String getName() { + return "connector_sync_job_get_action"; + } + + @Override + public List routes() { + return List.of( + new Route( + RestRequest.Method.GET, + "/" + EnterpriseSearch.CONNECTOR_SYNC_JOB_API_ENDPOINT + "/{" + CONNECTOR_SYNC_JOB_ID_PARAM + "}" + ) + ); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + GetConnectorSyncJobAction.Request request = new GetConnectorSyncJobAction.Request(restRequest.param(CONNECTOR_SYNC_JOB_ID_PARAM)); + return restChannel -> client.execute(GetConnectorSyncJobAction.INSTANCE, request, new RestToXContentListener<>(restChannel)); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportGetConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportGetConnectorSyncJobAction.java new file mode 100644 index 0000000000000..1024b9953fd09 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportGetConnectorSyncJobAction.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.syncjob.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService; + +public class TransportGetConnectorSyncJobAction extends HandledTransportAction< + GetConnectorSyncJobAction.Request, + GetConnectorSyncJobAction.Response> { + + protected final ConnectorSyncJobIndexService connectorSyncJobIndexService; + + @Inject + public TransportGetConnectorSyncJobAction( + TransportService transportService, + ClusterService clusterService, + ActionFilters actionFilters, + Client client + ) { + super( + GetConnectorSyncJobAction.NAME, + transportService, + actionFilters, + GetConnectorSyncJobAction.Request::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.connectorSyncJobIndexService = new ConnectorSyncJobIndexService(client); + } + + @Override + protected void doExecute( + Task task, + GetConnectorSyncJobAction.Request request, + ActionListener listener + ) { + connectorSyncJobIndexService.getConnectorSyncJob( + request.getConnectorSyncJobId(), + listener.map(GetConnectorSyncJobAction.Response::new) + ); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRule.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRule.java index 9b2ce393e5b04..9cca42b0402bf 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRule.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRule.java @@ -294,7 +294,7 @@ public AppliedQueryRules applyRule(AppliedQueryRules appliedRules, Map resp = new AtomicReference<>(null); @@ -399,4 +417,29 @@ public void onFailure(Exception e) { assertNotNull("Received null response from update scheduling request", resp.get()); return resp.get(); } + + private UpdateResponse awaitUpdateConnectorError(UpdateConnectorErrorAction.Request updatedError) throws Exception { + CountDownLatch latch = new CountDownLatch(1); + final AtomicReference resp = new AtomicReference<>(null); + final AtomicReference exc = new AtomicReference<>(null); + connectorIndexService.updateConnectorError(updatedError, new ActionListener<>() { + @Override + public void onResponse(UpdateResponse indexResponse) { + resp.set(indexResponse); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + exc.set(e); + latch.countDown(); + } + }); + assertTrue("Timeout waiting for update error request", latch.await(REQUEST_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + if (exc.get() != null) { + throw exc.get(); + } + assertNotNull("Received null response from update error request", resp.get()); + return resp.get(); + } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorActionRequestBWCSerializingTests.java new file mode 100644 index 0000000000000..94092cee61b40 --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorActionRequestBWCSerializingTests.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; + +import java.io.IOException; + +public class UpdateConnectorErrorActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase< + UpdateConnectorErrorAction.Request> { + + private String connectorId; + + @Override + protected Writeable.Reader instanceReader() { + return UpdateConnectorErrorAction.Request::new; + } + + @Override + protected UpdateConnectorErrorAction.Request createTestInstance() { + this.connectorId = randomUUID(); + return new UpdateConnectorErrorAction.Request(connectorId, randomAlphaOfLengthBetween(5, 15)); + } + + @Override + protected UpdateConnectorErrorAction.Request mutateInstance(UpdateConnectorErrorAction.Request instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected UpdateConnectorErrorAction.Request doParseInstance(XContentParser parser) throws IOException { + return UpdateConnectorErrorAction.Request.fromXContent(parser, this.connectorId); + } + + @Override + protected UpdateConnectorErrorAction.Request mutateInstanceForVersion( + UpdateConnectorErrorAction.Request instance, + TransportVersion version + ) { + return instance; + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorActionResponseBWCSerializingTests.java new file mode 100644 index 0000000000000..a39fcac3d2f04 --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorActionResponseBWCSerializingTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; + +import java.io.IOException; + +public class UpdateConnectorErrorActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< + UpdateConnectorErrorAction.Response> { + + @Override + protected Writeable.Reader instanceReader() { + return UpdateConnectorErrorAction.Response::new; + } + + @Override + protected UpdateConnectorErrorAction.Response createTestInstance() { + return new UpdateConnectorErrorAction.Response(randomFrom(DocWriteResponse.Result.values())); + } + + @Override + protected UpdateConnectorErrorAction.Response mutateInstance(UpdateConnectorErrorAction.Response instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected UpdateConnectorErrorAction.Response mutateInstanceForVersion( + UpdateConnectorErrorAction.Response instance, + TransportVersion version + ) { + return instance; + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java index cadc8b761cbe3..8613078e3074e 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java @@ -80,46 +80,21 @@ public void testCreateConnectorSyncJob() throws Exception { PostConnectorSyncJobAction.Request syncJobRequest = ConnectorSyncJobTestUtils.getRandomPostConnectorSyncJobActionRequest( connector.getConnectorId() ); - PostConnectorSyncJobAction.Response response = awaitPutConnectorSyncJob(syncJobRequest); - Map connectorSyncJobSource = getConnectorSyncJobSourceById(response.getId()); - - String id = (String) connectorSyncJobSource.get(ConnectorSyncJob.ID_FIELD.getPreferredName()); - ConnectorSyncJobType requestJobType = syncJobRequest.getJobType(); - ConnectorSyncJobType jobType = ConnectorSyncJobType.fromString( - (String) connectorSyncJobSource.get(ConnectorSyncJob.JOB_TYPE_FIELD.getPreferredName()) - ); - ConnectorSyncJobTriggerMethod requestTriggerMethod = syncJobRequest.getTriggerMethod(); - ConnectorSyncJobTriggerMethod triggerMethod = ConnectorSyncJobTriggerMethod.fromString( - (String) connectorSyncJobSource.get(ConnectorSyncJob.TRIGGER_METHOD_FIELD.getPreferredName()) - ); - - ConnectorSyncStatus initialStatus = ConnectorSyncStatus.fromString( - (String) connectorSyncJobSource.get(ConnectorSyncJob.STATUS_FIELD.getPreferredName()) - ); - - Instant createdNow = Instant.parse((String) connectorSyncJobSource.get(ConnectorSyncJob.CREATED_AT_FIELD.getPreferredName())); - Instant lastSeen = Instant.parse((String) connectorSyncJobSource.get(ConnectorSyncJob.LAST_SEEN_FIELD.getPreferredName())); + PostConnectorSyncJobAction.Response response = awaitPutConnectorSyncJob(syncJobRequest); - Integer totalDocumentCount = (Integer) connectorSyncJobSource.get(ConnectorSyncJob.TOTAL_DOCUMENT_COUNT_FIELD.getPreferredName()); - Integer indexedDocumentCount = (Integer) connectorSyncJobSource.get( - ConnectorSyncJob.INDEXED_DOCUMENT_COUNT_FIELD.getPreferredName() - ); - Integer indexedDocumentVolume = (Integer) connectorSyncJobSource.get( - ConnectorSyncJob.INDEXED_DOCUMENT_VOLUME_FIELD.getPreferredName() - ); - Integer deletedDocumentCount = (Integer) connectorSyncJobSource.get(ConnectorSyncJob.DELETED_DOCUMENT_COUNT.getPreferredName()); - - assertThat(id, notNullValue()); - assertThat(jobType, equalTo(requestJobType)); - assertThat(triggerMethod, equalTo(requestTriggerMethod)); - assertThat(initialStatus, equalTo(ConnectorSyncJob.DEFAULT_INITIAL_STATUS)); - assertThat(createdNow, equalTo(lastSeen)); - assertThat(totalDocumentCount, equalTo(0)); - assertThat(indexedDocumentCount, equalTo(0)); - assertThat(indexedDocumentVolume, equalTo(0)); - assertThat(deletedDocumentCount, equalTo(0)); + ConnectorSyncJob connectorSyncJob = awaitGetConnectorSyncJob(response.getId()); + + assertThat(connectorSyncJob.getId(), notNullValue()); + assertThat(connectorSyncJob.getJobType(), equalTo(requestJobType)); + assertThat(connectorSyncJob.getTriggerMethod(), equalTo(requestTriggerMethod)); + assertThat(connectorSyncJob.getStatus(), equalTo(ConnectorSyncJob.DEFAULT_INITIAL_STATUS)); + assertThat(connectorSyncJob.getCreatedAt(), equalTo(connectorSyncJob.getLastSeen())); + assertThat(connectorSyncJob.getTotalDocumentCount(), equalTo(0L)); + assertThat(connectorSyncJob.getIndexedDocumentCount(), equalTo(0L)); + assertThat(connectorSyncJob.getIndexedDocumentVolume(), equalTo(0L)); + assertThat(connectorSyncJob.getDeletedDocumentCount(), equalTo(0L)); } public void testCreateConnectorSyncJob_WithMissingJobType_ExpectDefaultJobTypeToBeSet() throws Exception { @@ -130,12 +105,9 @@ public void testCreateConnectorSyncJob_WithMissingJobType_ExpectDefaultJobTypeTo ); PostConnectorSyncJobAction.Response response = awaitPutConnectorSyncJob(syncJobRequest); - Map connectorSyncJobSource = getConnectorSyncJobSourceById(response.getId()); - ConnectorSyncJobType jobType = ConnectorSyncJobType.fromString( - (String) connectorSyncJobSource.get(ConnectorSyncJob.JOB_TYPE_FIELD.getPreferredName()) - ); + ConnectorSyncJob connectorSyncJob = awaitGetConnectorSyncJob(response.getId()); - assertThat(jobType, equalTo(ConnectorSyncJob.DEFAULT_JOB_TYPE)); + assertThat(connectorSyncJob.getJobType(), equalTo(ConnectorSyncJob.DEFAULT_JOB_TYPE)); } public void testCreateConnectorSyncJob_WithMissingTriggerMethod_ExpectDefaultTriggerMethodToBeSet() throws Exception { @@ -146,12 +118,9 @@ public void testCreateConnectorSyncJob_WithMissingTriggerMethod_ExpectDefaultTri ); PostConnectorSyncJobAction.Response response = awaitPutConnectorSyncJob(syncJobRequest); - Map connectorSyncJobSource = getConnectorSyncJobSourceById(response.getId()); - ConnectorSyncJobTriggerMethod triggerMethod = ConnectorSyncJobTriggerMethod.fromString( - (String) connectorSyncJobSource.get(ConnectorSyncJob.TRIGGER_METHOD_FIELD.getPreferredName()) - ); + ConnectorSyncJob connectorSyncJob = awaitGetConnectorSyncJob(response.getId()); - assertThat(triggerMethod, equalTo(ConnectorSyncJob.DEFAULT_TRIGGER_METHOD)); + assertThat(connectorSyncJob.getTriggerMethod(), equalTo(ConnectorSyncJob.DEFAULT_TRIGGER_METHOD)); } public void testCreateConnectorSyncJob_WithMissingConnectorId_ExpectException() throws Exception { @@ -184,6 +153,28 @@ public void testDeleteConnectorSyncJob_WithMissingSyncJobId_ExpectException() { expectThrows(ResourceNotFoundException.class, () -> awaitDeleteConnectorSyncJob(NON_EXISTING_SYNC_JOB_ID)); } + public void testGetConnectorSyncJob() throws Exception { + PostConnectorSyncJobAction.Request syncJobRequest = ConnectorSyncJobTestUtils.getRandomPostConnectorSyncJobActionRequest( + connector.getConnectorId() + ); + ConnectorSyncJobType jobType = syncJobRequest.getJobType(); + ConnectorSyncJobTriggerMethod triggerMethod = syncJobRequest.getTriggerMethod(); + + PostConnectorSyncJobAction.Response response = awaitPutConnectorSyncJob(syncJobRequest); + String syncJobId = response.getId(); + + ConnectorSyncJob syncJob = awaitGetConnectorSyncJob(syncJobId); + + assertThat(syncJob.getId(), equalTo(syncJobId)); + assertThat(syncJob.getJobType(), equalTo(jobType)); + assertThat(syncJob.getTriggerMethod(), equalTo(triggerMethod)); + assertThat(syncJob.getConnector().getConnectorId(), equalTo(connector.getConnectorId())); + } + + public void testGetConnectorSyncJob_WithMissingSyncJobId_ExpectException() { + expectThrows(ResourceNotFoundException.class, () -> awaitGetConnectorSyncJob(NON_EXISTING_SYNC_JOB_ID)); + } + public void testCheckInConnectorSyncJob() throws Exception { PostConnectorSyncJobAction.Request syncJobRequest = ConnectorSyncJobTestUtils.getRandomPostConnectorSyncJobActionRequest( connector.getConnectorId() @@ -346,6 +337,33 @@ private Map getConnectorSyncJobSourceById(String syncJobId) thro return getResponseActionFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).getSource(); } + private ConnectorSyncJob awaitGetConnectorSyncJob(String connectorSyncJobId) throws Exception { + CountDownLatch latch = new CountDownLatch(1); + final AtomicReference resp = new AtomicReference<>(null); + final AtomicReference exc = new AtomicReference<>(null); + + connectorSyncJobIndexService.getConnectorSyncJob(connectorSyncJobId, new ActionListener() { + @Override + public void onResponse(ConnectorSyncJob connectorSyncJob) { + resp.set(connectorSyncJob); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + exc.set(e); + latch.countDown(); + } + }); + + assertTrue("Timeout waiting for get request", latch.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + if (exc.get() != null) { + throw exc.get(); + } + assertNotNull("Received null response from get request", resp.get()); + return resp.get(); + } + private UpdateResponse awaitCheckInConnectorSyncJob(String connectorSyncJobId) throws Exception { CountDownLatch latch = new CountDownLatch(1); final AtomicReference resp = new AtomicReference<>(null); diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTestUtils.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTestUtils.java index 4fa1b9122284d..9ec404e109496 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTestUtils.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTestUtils.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.application.connector.syncjob.action.CancelConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.CheckInConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.DeleteConnectorSyncJobAction; +import org.elasticsearch.xpack.application.connector.syncjob.action.GetConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.PostConnectorSyncJobAction; import java.time.Instant; @@ -100,4 +101,12 @@ public static CancelConnectorSyncJobAction.Request getRandomCancelConnectorSyncJ public static CheckInConnectorSyncJobAction.Request getRandomCheckInConnectorSyncJobActionRequest() { return new CheckInConnectorSyncJobAction.Request(randomAlphaOfLength(10)); } + + public static GetConnectorSyncJobAction.Request getRandomGetConnectorSyncJobRequest() { + return new GetConnectorSyncJobAction.Request(randomAlphaOfLength(10)); + } + + public static GetConnectorSyncJobAction.Response getRandomGetConnectorSyncJobResponse() { + return new GetConnectorSyncJobAction.Response(getRandomConnectorSyncJob()); + } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java index aeecf582c9ec7..ace1138b8e987 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java @@ -7,15 +7,23 @@ package org.elasticsearch.xpack.application.connector.syncjob; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.application.connector.Connector; +import org.elasticsearch.xpack.application.connector.ConnectorSyncStatus; import org.junit.Before; import java.io.IOException; +import java.time.Instant; import java.util.List; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.notNullValue; public class ConnectorSyncJobTests extends ESTestCase { @@ -35,6 +43,205 @@ public final void testRandomSerialization() throws IOException { } } + public void testFromXContent_WithAllFields_AllSet() throws IOException { + String content = XContentHelper.stripWhitespace(""" + { + "cancelation_requested_at": "2023-12-01T14:19:39.394194Z", + "canceled_at": "2023-12-01T14:19:39.394194Z", + "completed_at": "2023-12-01T14:19:39.394194Z", + "connector": { + "connector_id": "connector-id", + "filtering": [ + { + "active": { + "advanced_snippet": { + "created_at": "2023-12-01T14:18:37.397819Z", + "updated_at": "2023-12-01T14:18:37.397819Z", + "value": {} + }, + "rules": [ + { + "created_at": "2023-12-01T14:18:37.397819Z", + "field": "_", + "id": "DEFAULT", + "order": 0, + "policy": "include", + "rule": "regex", + "updated_at": "2023-12-01T14:18:37.397819Z", + "value": ".*" + } + ], + "validation": { + "errors": [], + "state": "valid" + } + }, + "domain": "DEFAULT", + "draft": { + "advanced_snippet": { + "created_at": "2023-12-01T14:18:37.397819Z", + "updated_at": "2023-12-01T14:18:37.397819Z", + "value": {} + }, + "rules": [ + { + "created_at": "2023-12-01T14:18:37.397819Z", + "field": "_", + "id": "DEFAULT", + "order": 0, + "policy": "include", + "rule": "regex", + "updated_at": "2023-12-01T14:18:37.397819Z", + "value": ".*" + } + ], + "validation": { + "errors": [], + "state": "valid" + } + } + } + ], + "index_name": "search-connector", + "language": "english", + "pipeline": { + "extract_binary_content": true, + "name": "ent-search-generic-ingestion", + "reduce_whitespace": true, + "run_ml_inference": false + }, + "service_type": "service type", + "configuration": {} + }, + "created_at": "2023-12-01T14:18:43.07693Z", + "deleted_document_count": 10, + "error": "some-error", + "id": "HIC-JYwB9RqKhB7x_hIE", + "indexed_document_count": 10, + "indexed_document_volume": 10, + "job_type": "full", + "last_seen": "2023-12-01T14:18:43.07693Z", + "metadata": {}, + "started_at": "2023-12-01T14:18:43.07693Z", + "status": "canceling", + "total_document_count": 0, + "trigger_method": "scheduled", + "worker_hostname": "worker-hostname" + } + """); + + ConnectorSyncJob syncJob = ConnectorSyncJob.fromXContentBytes(new BytesArray(content), XContentType.JSON); + + assertThat(syncJob.getCancelationRequestedAt(), equalTo(Instant.parse("2023-12-01T14:19:39.394194Z"))); + assertThat(syncJob.getCanceledAt(), equalTo(Instant.parse("2023-12-01T14:19:39.394194Z"))); + assertThat(syncJob.getCompletedAt(), equalTo(Instant.parse("2023-12-01T14:19:39.394194Z"))); + + assertThat(syncJob.getConnector().getConnectorId(), equalTo("connector-id")); + assertThat(syncJob.getConnector().getFiltering(), hasSize(greaterThan(0))); + assertThat(syncJob.getConnector().getIndexName(), equalTo("search-connector")); + assertThat(syncJob.getConnector().getLanguage(), equalTo("english")); + assertThat(syncJob.getConnector().getPipeline(), notNullValue()); + + assertThat(syncJob.getCreatedAt(), equalTo(Instant.parse("2023-12-01T14:18:43.07693Z"))); + assertThat(syncJob.getDeletedDocumentCount(), equalTo(10L)); + assertThat(syncJob.getError(), equalTo("some-error")); + assertThat(syncJob.getId(), equalTo("HIC-JYwB9RqKhB7x_hIE")); + assertThat(syncJob.getIndexedDocumentCount(), equalTo(10L)); + assertThat(syncJob.getIndexedDocumentVolume(), equalTo(10L)); + assertThat(syncJob.getJobType(), equalTo(ConnectorSyncJobType.FULL)); + assertThat(syncJob.getLastSeen(), equalTo(Instant.parse("2023-12-01T14:18:43.07693Z"))); + assertThat(syncJob.getMetadata(), notNullValue()); + assertThat(syncJob.getStartedAt(), equalTo(Instant.parse("2023-12-01T14:18:43.07693Z"))); + assertThat(syncJob.getStatus(), equalTo(ConnectorSyncStatus.CANCELING)); + assertThat(syncJob.getTotalDocumentCount(), equalTo(0L)); + assertThat(syncJob.getTriggerMethod(), equalTo(ConnectorSyncJobTriggerMethod.SCHEDULED)); + assertThat(syncJob.getWorkerHostname(), equalTo("worker-hostname")); + } + + public void testFromXContent_WithAllNonOptionalFieldsSet_DoesNotThrow() throws IOException { + String content = XContentHelper.stripWhitespace(""" + { + "connector": { + "connector_id": "connector-id", + "filtering": [ + { + "active": { + "advanced_snippet": { + "created_at": "2023-12-01T14:18:37.397819Z", + "updated_at": "2023-12-01T14:18:37.397819Z", + "value": {} + }, + "rules": [ + { + "created_at": "2023-12-01T14:18:37.397819Z", + "field": "_", + "id": "DEFAULT", + "order": 0, + "policy": "include", + "rule": "regex", + "updated_at": "2023-12-01T14:18:37.397819Z", + "value": ".*" + } + ], + "validation": { + "errors": [], + "state": "valid" + } + }, + "domain": "DEFAULT", + "draft": { + "advanced_snippet": { + "created_at": "2023-12-01T14:18:37.397819Z", + "updated_at": "2023-12-01T14:18:37.397819Z", + "value": {} + }, + "rules": [ + { + "created_at": "2023-12-01T14:18:37.397819Z", + "field": "_", + "id": "DEFAULT", + "order": 0, + "policy": "include", + "rule": "regex", + "updated_at": "2023-12-01T14:18:37.397819Z", + "value": ".*" + } + ], + "validation": { + "errors": [], + "state": "valid" + } + } + } + ], + "index_name": "search-connector", + "language": "english", + "pipeline": { + "extract_binary_content": true, + "name": "ent-search-generic-ingestion", + "reduce_whitespace": true, + "run_ml_inference": false + }, + "service_type": "service type", + "configuration": {} + }, + "created_at": "2023-12-01T14:18:43.07693Z", + "deleted_document_count": 10, + "id": "HIC-JYwB9RqKhB7x_hIE", + "indexed_document_count": 10, + "indexed_document_volume": 10, + "job_type": "full", + "last_seen": "2023-12-01T14:18:43.07693Z", + "metadata": {}, + "status": "canceling", + "total_document_count": 0, + "trigger_method": "scheduled" + } + """); + + ConnectorSyncJob.fromXContentBytes(new BytesArray(content), XContentType.JSON); + } + private void assertTransportSerialization(ConnectorSyncJob testInstance) throws IOException { ConnectorSyncJob deserializedInstance = copyInstance(testInstance); assertNotSame(testInstance, deserializedInstance); diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionRequestBWCSerializingTests.java new file mode 100644 index 0000000000000..c0b7711474a0b --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionRequestBWCSerializingTests.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.syncjob.action; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; +import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; + +import java.io.IOException; + +public class GetConnectorSyncJobActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase< + GetConnectorSyncJobAction.Request> { + @Override + protected Writeable.Reader instanceReader() { + return GetConnectorSyncJobAction.Request::new; + } + + @Override + protected GetConnectorSyncJobAction.Request createTestInstance() { + return ConnectorSyncJobTestUtils.getRandomGetConnectorSyncJobRequest(); + } + + @Override + protected GetConnectorSyncJobAction.Request mutateInstance(GetConnectorSyncJobAction.Request instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected GetConnectorSyncJobAction.Request doParseInstance(XContentParser parser) throws IOException { + return GetConnectorSyncJobAction.Request.parse(parser); + } + + @Override + protected GetConnectorSyncJobAction.Request mutateInstanceForVersion( + GetConnectorSyncJobAction.Request instance, + TransportVersion version + ) { + return new GetConnectorSyncJobAction.Request(instance.getConnectorSyncJobId()); + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionResponseBWCSerializingTests.java new file mode 100644 index 0000000000000..00f6e7cf57fc1 --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionResponseBWCSerializingTests.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.syncjob.action; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xpack.application.connector.Connector; +import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; +import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; + +import java.io.IOException; +import java.util.List; + +public class GetConnectorSyncJobActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< + GetConnectorSyncJobAction.Response> { + + @Override + public NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(List.of(new NamedWriteableRegistry.Entry(Connector.class, Connector.NAME, Connector::new))); + } + + @Override + protected Writeable.Reader instanceReader() { + return GetConnectorSyncJobAction.Response::new; + } + + @Override + protected GetConnectorSyncJobAction.Response createTestInstance() { + return ConnectorSyncJobTestUtils.getRandomGetConnectorSyncJobResponse(); + } + + @Override + protected GetConnectorSyncJobAction.Response mutateInstance(GetConnectorSyncJobAction.Response instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected GetConnectorSyncJobAction.Response mutateInstanceForVersion( + GetConnectorSyncJobAction.Response instance, + TransportVersion version + ) { + return instance; + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionTests.java new file mode 100644 index 0000000000000..807f02124f32a --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionTests.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.syncjob.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobConstants; +import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +public class GetConnectorSyncJobActionTests extends ESTestCase { + + public void testValidate_WhenConnectorSyncJobIdIsPresent_ExpectNoValidationError() { + GetConnectorSyncJobAction.Request request = ConnectorSyncJobTestUtils.getRandomGetConnectorSyncJobRequest(); + ActionRequestValidationException exception = request.validate(); + + assertThat(exception, nullValue()); + } + + public void testValidate_WhenConnectorSyncJobIdIsEmpty_ExpectValidationError() { + GetConnectorSyncJobAction.Request requestWithMissingConnectorId = new GetConnectorSyncJobAction.Request(""); + ActionRequestValidationException exception = requestWithMissingConnectorId.validate(); + + assertThat(exception, notNullValue()); + assertThat(exception.getMessage(), containsString(ConnectorSyncJobConstants.EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE)); + } + +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportGetConnectorSyncJobActionTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportGetConnectorSyncJobActionTests.java new file mode 100644 index 0000000000000..7b83d008d92bc --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportGetConnectorSyncJobActionTests.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.syncjob.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.Transport; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; +import org.junit.Before; + +import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.mockito.Mockito.mock; + +public class TransportGetConnectorSyncJobActionTests extends ESSingleNodeTestCase { + + private static final Long TIMEOUT_SECONDS = 10L; + + private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + private TransportGetConnectorSyncJobAction action; + + @Before + public void setup() { + ClusterService clusterService = getInstanceFromNode(ClusterService.class); + + TransportService transportService = new TransportService( + Settings.EMPTY, + mock(Transport.class), + threadPool, + TransportService.NOOP_TRANSPORT_INTERCEPTOR, + x -> null, + null, + Collections.emptySet() + ); + + action = new TransportGetConnectorSyncJobAction(transportService, clusterService, mock(ActionFilters.class), client()); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + ThreadPool.terminate(threadPool, TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + + public void testGetConnectorSyncJob_ExpectNoWarnings() throws InterruptedException { + GetConnectorSyncJobAction.Request request = ConnectorSyncJobTestUtils.getRandomGetConnectorSyncJobRequest(); + + executeRequest(request); + + ensureNoWarnings(); + } + + private void executeRequest(GetConnectorSyncJobAction.Request request) throws InterruptedException { + final CountDownLatch latch = new CountDownLatch(1); + action.doExecute(mock(Task.class), request, ActionListener.wrap(response -> latch.countDown(), exception -> latch.countDown())); + + boolean requestTimedOut = latch.await(TIMEOUT_SECONDS, TimeUnit.SECONDS); + + assertTrue("Timeout waiting for get request", requestTimedOut); + } +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java index 060a137b69b7c..ebe27225becb1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java @@ -327,8 +327,7 @@ public static ExpectedResults loadCsvSpecValues(String csv) { for (int i = 0; i < row.size(); i++) { String value = row.get(i); if (value == null || value.trim().equalsIgnoreCase(NULL_VALUE)) { - value = null; - rowValues.add(columnTypes.get(i).convert(value)); + rowValues.add(null); continue; } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ints.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ints.csv-spec index 0f6fc42860750..3e28c8bc2cb9b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ints.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ints.csv-spec @@ -197,8 +197,7 @@ long:long |int:integer convertULToInt#[skip:-8.11.99, reason:ql exceptions were updated in 8.12] row ul = [2147483647, 9223372036854775808] | eval int = to_int(ul); warning:Line 1:57: evaluation of [to_int(ul)] failed, treating result as null. Only first 20 failures recorded. -// UL conversion to int dips into long; not the most efficient, but it's how SQL does it too. -warning:Line 1:57: org.elasticsearch.xpack.ql.InvalidArgumentException: [9223372036854775808] out of [long] range +warning:Line 1:57: org.elasticsearch.xpack.ql.InvalidArgumentException: [9223372036854775808] out of [integer] range ul:ul |int:integer [2147483647, 9223372036854775808]|2147483647 @@ -219,20 +218,29 @@ tf:boolean |t2i:integer |f2i:integer |tf2i:integer ; convertStringToInt -row int_str = "2147483647", int_dbl_str = "2147483647.2" | eval is2i = to_integer(int_str), ids2i = to_integer(int_dbl_str), overflow = to_integer("2147483648"), no_number = to_integer("foo"); -warning:Line 1:137: evaluation of [to_integer(\"2147483648\")] failed, treating result as null. Only first 20 failures recorded. -warning:Line 1:137: java.lang.NumberFormatException: For input string: \"2147483648\" -warning:Line 1:175: evaluation of [to_integer(\"foo\")] failed, treating result as null. Only first 20 failures recorded. -warning:Line 1:175: java.lang.NumberFormatException: For input string: \"foo\" +row int_str = "2147483647", int_dbl_str = "2147483646.2" | eval is2i = to_integer(int_str), ids2i = to_integer(int_dbl_str); -int_str:keyword |int_dbl_str:keyword |is2i:integer|ids2i:integer |overflow:integer |no_number:integer -2147483647 |2147483647.2 |2147483647 |2147483647 |null |null +int_str:keyword |int_dbl_str:keyword |is2i:integer|ids2i:integer +2147483647 |2147483646.2 |2147483647 |2147483646 +; + +convertStringToIntFail#[skip:-8.11.99, reason:double rounding in conversion updated in 8.12] +row str1 = "2147483647.2", str2 = "2147483648", non = "no number" | eval i1 = to_integer(str1), i2 = to_integer(str2), noi = to_integer(non); +warning:Line 1:79: evaluation of [to_integer(str1)] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:79: java.lang.NumberFormatException: For input string: \"2147483647.2\" +warning:Line 1:102: evaluation of [to_integer(str2)] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:102: java.lang.NumberFormatException: For input string: \"2147483648\" +warning:Line 1:126: evaluation of [to_integer(non)] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:126: java.lang.NumberFormatException: For input string: \"no number\" + +str1:keyword |str2:keyword |non:keyword |i1:integer |i2:integer |noi:integer +2147483647.2 |2147483648 |no number |null |null |null ; convertDoubleToInt#[skip:-8.11.99, reason:ql exceptions were updated in 8.12] row d = 123.4 | eval d2i = to_integer(d), overflow = to_integer(1e19); warning:Line 1:54: evaluation of [to_integer(1e19)] failed, treating result as null. Only first 20 failures recorded. -warning:Line 1:54: org.elasticsearch.xpack.ql.InvalidArgumentException: [1.0E19] out of [long] range +warning:Line 1:54: org.elasticsearch.xpack.ql.InvalidArgumentException: [1.0E19] out of [integer] range d:double |d2i:integer |overflow:integer 123.4 |123 |null diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec index b23e4d87fe52f..ffad468790998 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec @@ -5,6 +5,7 @@ v:long 1 ; +# TODO: switch this test to ``&format=csv&delimiter=|` output showFunctions#[skip:-8.11.99] show functions; @@ -71,27 +72,27 @@ sum |? sum(arg1:?) tan |"double tan(n:integer|long|double|unsigned_long)" |n |"integer|long|double|unsigned_long" | "" |double | "" | false | false tanh |"double tanh(n:integer|long|double|unsigned_long)" |n |"integer|long|double|unsigned_long" | "" |double | "" | false | false tau |? tau() | null | null | null |? | "" | null | false -to_bool |? to_bool(arg1:?) |arg1 |? | "" |? | "" | false | false -to_boolean |? to_boolean(arg1:?) |arg1 |? | "" |? | "" | false | false +to_bool |"boolean to_bool(v:boolean|keyword|text|double|long|unsigned_long|integer)" |v |"boolean|keyword|text|double|long|unsigned_long|integer" | |boolean | |false |false +to_boolean |"boolean to_boolean(v:boolean|keyword|text|double|long|unsigned_long|integer)" |v |"boolean|keyword|text|double|long|unsigned_long|integer" | |boolean | |false |false to_cartesianpoint |? to_cartesianpoint(arg1:?) |arg1 |? | "" |? | "" | false | false -to_datetime |? to_datetime(arg1:?) |arg1 |? | "" |? | "" | false | false -to_dbl |? to_dbl(arg1:?) |arg1 |? | "" |? | "" | false | false -to_degrees |? to_degrees(arg1:?) |arg1 |? | "" |? | "" | false | false -to_double |? to_double(arg1:?) |arg1 |? | "" |? | "" | false | false -to_dt |? to_dt(arg1:?) |arg1 |? | "" |? | "" | false | false +to_datetime |"date to_datetime(v:date|keyword|text|double|long|unsigned_long|integer)" |v |"date|keyword|text|double|long|unsigned_long|integer" | |date | |false |false +to_dbl |"double to_dbl(v:boolean|date|keyword|text|double|long|unsigned_long|integer)" |v |"boolean|date|keyword|text|double|long|unsigned_long|integer" | |double | |false |false +to_degrees |"double to_degrees(v:double|long|unsigned_long|integer)" |v |"double|long|unsigned_long|integer" | |double | |false |false +to_double |"double to_double(v:boolean|date|keyword|text|double|long|unsigned_long|integer)" |v |"boolean|date|keyword|text|double|long|unsigned_long|integer" | |double | |false |false +to_dt |"date to_dt(v:date|keyword|text|double|long|unsigned_long|integer)" |v |"date|keyword|text|double|long|unsigned_long|integer" | |date | |false |false to_geopoint |? to_geopoint(arg1:?) |arg1 |? | "" |? | "" | false | false -to_int |? to_int(arg1:?) |arg1 |? | "" |? | "" | false | false -to_integer |? to_integer(arg1:?) |arg1 |? | "" |? | "" | false | false -to_ip |? to_ip(arg1:?) |arg1 |? | "" |? | "" | false | false -to_long |? to_long(arg1:?) |arg1 |? | "" |? | "" | false | false -to_radians |? to_radians(arg1:?) |arg1 |? | "" |? | "" | false | false -to_str |"? to_str(v:unsigned_long|date|boolean|double|ip|text|integer|keyword|version|long|geo_point|cartesian_point)"|v |"unsigned_long|date|boolean|double|ip|text|integer|keyword|version|long|geo_point|cartesian_point" | "" |? | "" | false | false -to_string |"? to_string(v:unsigned_long|date|boolean|double|ip|text|integer|keyword|version|long|geo_point|cartesian_point)"|v |"unsigned_long|date|boolean|double|ip|text|integer|keyword|version|long|geo_point|cartesian_point" | "" |? | "" | false | false -to_ul |? to_ul(arg1:?) |arg1 |? | "" |? | "" | false | false -to_ulong |? to_ulong(arg1:?) |arg1 |? | "" |? | "" | false | false -to_unsigned_long |? to_unsigned_long(arg1:?) |arg1 |? | "" |? | "" | false | false -to_ver |"? to_ver(v:keyword|text|version)" |v |"keyword|text|version"| "" |? | "" | false | false -to_version |"? to_version(v:keyword|text|version)" |v |"keyword|text|version"| "" |? | "" | false | false +to_int |"integer to_int(v:boolean|date|keyword|text|double|long|unsigned_long|integer)" |v |"boolean|date|keyword|text|double|long|unsigned_long|integer" | |integer | |false |false +to_integer |"integer to_integer(v:boolean|date|keyword|text|double|long|unsigned_long|integer)" |v |"boolean|date|keyword|text|double|long|unsigned_long|integer" | |integer | |false |false +to_ip |"ip to_ip(v:ip|keyword|text)" |v |"ip|keyword|text" | |ip | |false |false +to_long |"long to_long(v:boolean|date|keyword|text|double|long|unsigned_long|integer|geo_point|cartesian_point)" |v |"boolean|date|keyword|text|double|long|unsigned_long|integer|geo_point|cartesian_point" | |long | |false |false +to_radians |"double to_radians(v:double|long|unsigned_long|integer)" |v |"double|long|unsigned_long|integer" | |double | |false |false +to_str |"keyword to_str(v:unsigned_long|date|boolean|double|ip|text|integer|keyword|version|long|geo_point|cartesian_point)" |v |"unsigned_long|date|boolean|double|ip|text|integer|keyword|version|long|geo_point|cartesian_point" | |keyword | |false |false +to_string |"keyword to_string(v:unsigned_long|date|boolean|double|ip|text|integer|keyword|version|long|geo_point|cartesian_point)" |v |"unsigned_long|date|boolean|double|ip|text|integer|keyword|version|long|geo_point|cartesian_point" | |keyword | |false |false +to_ul |"unsigned_long to_ul(v:boolean|date|keyword|text|double|long|unsigned_long|integer)" |v |"boolean|date|keyword|text|double|long|unsigned_long|integer" | |unsigned_long | |false |false +to_ulong |"unsigned_long to_ulong(v:boolean|date|keyword|text|double|long|unsigned_long|integer)" |v |"boolean|date|keyword|text|double|long|unsigned_long|integer" | |unsigned_long | |false |false +to_unsigned_long |"unsigned_long to_unsigned_long(v:boolean|date|keyword|text|double|long|unsigned_long|integer)" |v |"boolean|date|keyword|text|double|long|unsigned_long|integer" | |unsigned_long | |false |false +to_ver |"version to_ver(v:keyword|text|version)" |v |"keyword|text|version" | |version | |false |false +to_version |"version to_version(v:keyword|text|version)" |v |"keyword|text|version" | |version | |false |false trim |"keyword|text trim(str:keyword|text)" |str |"keyword|text" | "" |"keyword|text" |Removes leading and trailing whitespaces from a string.| false | false ; @@ -99,90 +100,90 @@ trim |"keyword|text trim(str:keyword|text)" showFunctionsSynopsis#[skip:-8.11.99] show functions | keep synopsis; -synopsis:keyword -"integer|long|double|unsigned_long abs(n:integer|long|double|unsigned_long)" -"double acos(n:integer|long|double|unsigned_long)" -"double asin(n:integer|long|double|unsigned_long)" -"double atan(n:integer|long|double|unsigned_long)" -"double atan2(y:integer|long|double|unsigned_long, x:integer|long|double|unsigned_long)" +synopsis:keyword +"integer|long|double|unsigned_long abs(n:integer|long|double|unsigned_long)" +"double acos(n:integer|long|double|unsigned_long)" +"double asin(n:integer|long|double|unsigned_long)" +"double atan(n:integer|long|double|unsigned_long)" +"double atan2(y:integer|long|double|unsigned_long, x:integer|long|double|unsigned_long)" "double|date auto_bucket(field:integer|long|double|date, buckets:integer, from:integer|long|double|date, to:integer|long|double|date)" -? avg(arg1:?) -? case(arg1:?, arg2...:?) -"? ceil(n:integer|long|double|unsigned_long)" -? cidr_match(arg1:?, arg2...:?) -? coalesce(arg1:?, arg2...:?) -? concat(arg1:?, arg2...:?) -"double cos(n:integer|long|double|unsigned_long)" -"double cosh(n:integer|long|double|unsigned_long)" -? count(arg1:?) -? count_distinct(arg1:?, arg2:?) -? date_extract(arg1:?, arg2:?) -? date_format(arg1:?, arg2:?) +? avg(arg1:?) +? case(arg1:?, arg2...:?) +"? ceil(n:integer|long|double|unsigned_long)" +? cidr_match(arg1:?, arg2...:?) +? coalesce(arg1:?, arg2...:?) +? concat(arg1:?, arg2...:?) +"double cos(n:integer|long|double|unsigned_long)" +"double cosh(n:integer|long|double|unsigned_long)" +? count(arg1:?) +? count_distinct(arg1:?, arg2:?) +? date_extract(arg1:?, arg2:?) +? date_format(arg1:?, arg2:?) "date date_parse(?datePattern:keyword, dateString:keyword|text)" -? date_trunc(arg1:?, arg2:?) -? e() -? ends_with(arg1:?, arg2:?) -"? floor(n:integer|long|double|unsigned_long)" -"? greatest(first:integer|long|double|boolean|keyword|text|ip|version, rest...:integer|long|double|boolean|keyword|text|ip|version)" -? is_finite(arg1:?) -? is_infinite(arg1:?) -? is_nan(arg1:?) -"? least(first:integer|long|double|boolean|keyword|text|ip|version, rest...:integer|long|double|boolean|keyword|text|ip|version)" -"? left(string:keyword, length:integer)" -? length(arg1:?) -"? log10(n:integer|long|double|unsigned_long)" +? date_trunc(arg1:?, arg2:?) +? e() +? ends_with(arg1:?, arg2:?) +"? floor(n:integer|long|double|unsigned_long)" +"? greatest(first:integer|long|double|boolean|keyword|text|ip|version, rest...:integer|long|double|boolean|keyword|text|ip|version)" +? is_finite(arg1:?) +? is_infinite(arg1:?) +? is_nan(arg1:?) +"? least(first:integer|long|double|boolean|keyword|text|ip|version, rest...:integer|long|double|boolean|keyword|text|ip|version)" +? left(string:keyword, length:integer) +? length(arg1:?) +"? log10(n:integer|long|double|unsigned_long)" "keyword|text ltrim(str:keyword|text)" -? max(arg1:?) -? median(arg1:?) -? median_absolute_deviation(arg1:?) -? min(arg1:?) -? mv_avg(arg1:?) +? max(arg1:?) +? median(arg1:?) +? median_absolute_deviation(arg1:?) +? min(arg1:?) +? mv_avg(arg1:?) "keyword mv_concat(v:text|keyword, delim:text|keyword)" "integer mv_count(v:unsigned_long|date|boolean|double|ip|text|integer|keyword|version|long|geo_point|cartesian_point)" "? mv_dedupe(v:boolean|date|double|ip|text|integer|keyword|version|long)" "? mv_max(v:unsigned_long|date|boolean|double|ip|text|integer|keyword|version|long)" -? mv_median(arg1:?) +? mv_median(arg1:?) "? mv_min(v:unsigned_long|date|boolean|double|ip|text|integer|keyword|version|long)" -? mv_sum(arg1:?) -? now() -? percentile(arg1:?, arg2:?) -? pi() -"? pow(base:integer|unsigned_long|long|double, exponent:integer|unsigned_long|long|double)" -"? replace(arg1:?, arg2:?, arg3:?)" -"? right(string:keyword, length:integer)" -? round(arg1:?, arg2:?) +? mv_sum(arg1:?) +? now() +? percentile(arg1:?, arg2:?) +? pi() +"? pow(base:integer|unsigned_long|long|double, exponent:integer|unsigned_long|long|double)" +? replace(arg1:?, arg2:?, arg3:?) +? right(string:keyword, length:integer) +? round(arg1:?, arg2:?) "keyword|text rtrim(str:keyword|text)" -"double sin(n:integer|long|double|unsigned_long)" +"double sin(n:integer|long|double|unsigned_long)" "double sinh(n:integer|long|double|unsigned_long)" -? split(arg1:?, arg2:?) -"? sqrt(n:integer|long|double|unsigned_long)" -? starts_with(arg1:?, arg2:?) -? substring(arg1:?, arg2:?, arg3:?) -? sum(arg1:?) -"double tan(n:integer|long|double|unsigned_long)" -"double tanh(n:integer|long|double|unsigned_long)" -? tau() -? to_bool(arg1:?) -? to_boolean(arg1:?) +? split(arg1:?, arg2:?) +"? sqrt(n:integer|long|double|unsigned_long)" +? starts_with(arg1:?, arg2:?) +? substring(arg1:?, arg2:?, arg3:?) +? sum(arg1:?) +"double tan(n:integer|long|double|unsigned_long)" +"double tanh(n:integer|long|double|unsigned_long)" +? tau() +"boolean to_bool(v:boolean|keyword|text|double|long|unsigned_long|integer)" +"boolean to_boolean(v:boolean|keyword|text|double|long|unsigned_long|integer)" ? to_cartesianpoint(arg1:?) -? to_datetime(arg1:?) -? to_dbl(arg1:?) -? to_degrees(arg1:?) -? to_double(arg1:?) -? to_dt(arg1:?) +"date to_datetime(v:date|keyword|text|double|long|unsigned_long|integer)" +"double to_dbl(v:boolean|date|keyword|text|double|long|unsigned_long|integer)" +"double to_degrees(v:double|long|unsigned_long|integer)" +"double to_double(v:boolean|date|keyword|text|double|long|unsigned_long|integer)" +"date to_dt(v:date|keyword|text|double|long|unsigned_long|integer)" ? to_geopoint(arg1:?) -? to_int(arg1:?) -? to_integer(arg1:?) -? to_ip(arg1:?) -? to_long(arg1:?) -? to_radians(arg1:?) -"? to_str(v:unsigned_long|date|boolean|double|ip|text|integer|keyword|version|long|geo_point|cartesian_point)" -"? to_string(v:unsigned_long|date|boolean|double|ip|text|integer|keyword|version|long|geo_point|cartesian_point)" -? to_ul(arg1:?) -? to_ulong(arg1:?) -? to_unsigned_long(arg1:?) -"? to_ver(v:keyword|text|version)" -"? to_version(v:keyword|text|version)" +"integer to_int(v:boolean|date|keyword|text|double|long|unsigned_long|integer)" +"integer to_integer(v:boolean|date|keyword|text|double|long|unsigned_long|integer)" +"ip to_ip(v:ip|keyword|text)" +"long to_long(v:boolean|date|keyword|text|double|long|unsigned_long|integer|geo_point|cartesian_point)" +"double to_radians(v:double|long|unsigned_long|integer)" +"keyword to_str(v:unsigned_long|date|boolean|double|ip|text|integer|keyword|version|long|geo_point|cartesian_point)" +"keyword to_string(v:unsigned_long|date|boolean|double|ip|text|integer|keyword|version|long|geo_point|cartesian_point)" +"unsigned_long to_ul(v:boolean|date|keyword|text|double|long|unsigned_long|integer)" +"unsigned_long to_ulong(v:boolean|date|keyword|text|double|long|unsigned_long|integer)" +"unsigned_long to_unsigned_long(v:boolean|date|keyword|text|double|long|unsigned_long|integer)" +"version to_ver(v:keyword|text|version)" +"version to_version(v:keyword|text|version)" "keyword|text trim(str:keyword|text)" ; diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromDoubleEvaluator.java index b7ff410d07c15..329269bafd9ba 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromDoubleEvaluator.java @@ -14,7 +14,6 @@ import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.xpack.ql.InvalidArgumentException; -import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import org.elasticsearch.xpack.ql.tree.Source; /** @@ -39,7 +38,7 @@ public Block evalVector(Vector v) { if (vector.isConstant()) { try { return driverContext.blockFactory().newConstantIntBlockWith(evalValue(vector, 0), positionCount); - } catch (InvalidArgumentException | QlIllegalArgumentException e) { + } catch (InvalidArgumentException e) { registerException(e); return driverContext.blockFactory().newConstantNullBlock(positionCount); } @@ -48,7 +47,7 @@ public Block evalVector(Vector v) { for (int p = 0; p < positionCount; p++) { try { builder.appendInt(evalValue(vector, p)); - } catch (InvalidArgumentException | QlIllegalArgumentException e) { + } catch (InvalidArgumentException e) { registerException(e); builder.appendNull(); } @@ -82,7 +81,7 @@ public Block evalBlock(Block b) { } builder.appendInt(value); valuesAppended = true; - } catch (InvalidArgumentException | QlIllegalArgumentException e) { + } catch (InvalidArgumentException e) { registerException(e); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromLongEvaluator.java index 742b057c06799..f9b3cb60dad2c 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromLongEvaluator.java @@ -14,7 +14,6 @@ import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.xpack.ql.InvalidArgumentException; -import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import org.elasticsearch.xpack.ql.tree.Source; /** @@ -39,7 +38,7 @@ public Block evalVector(Vector v) { if (vector.isConstant()) { try { return driverContext.blockFactory().newConstantIntBlockWith(evalValue(vector, 0), positionCount); - } catch (InvalidArgumentException | QlIllegalArgumentException e) { + } catch (InvalidArgumentException e) { registerException(e); return driverContext.blockFactory().newConstantNullBlock(positionCount); } @@ -48,7 +47,7 @@ public Block evalVector(Vector v) { for (int p = 0; p < positionCount; p++) { try { builder.appendInt(evalValue(vector, p)); - } catch (InvalidArgumentException | QlIllegalArgumentException e) { + } catch (InvalidArgumentException e) { registerException(e); builder.appendNull(); } @@ -82,7 +81,7 @@ public Block evalBlock(Block b) { } builder.appendInt(value); valuesAppended = true; - } catch (InvalidArgumentException | QlIllegalArgumentException e) { + } catch (InvalidArgumentException e) { registerException(e); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromStringEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromStringEvaluator.java index bff4d46b09dff..600fa293394f9 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromStringEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromStringEvaluator.java @@ -15,6 +15,7 @@ import org.elasticsearch.compute.data.Vector; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.ql.InvalidArgumentException; import org.elasticsearch.xpack.ql.tree.Source; /** @@ -40,7 +41,7 @@ public Block evalVector(Vector v) { if (vector.isConstant()) { try { return driverContext.blockFactory().newConstantIntBlockWith(evalValue(vector, 0, scratchPad), positionCount); - } catch (NumberFormatException e) { + } catch (InvalidArgumentException | NumberFormatException e) { registerException(e); return driverContext.blockFactory().newConstantNullBlock(positionCount); } @@ -49,7 +50,7 @@ public Block evalVector(Vector v) { for (int p = 0; p < positionCount; p++) { try { builder.appendInt(evalValue(vector, p, scratchPad)); - } catch (NumberFormatException e) { + } catch (InvalidArgumentException | NumberFormatException e) { registerException(e); builder.appendNull(); } @@ -84,7 +85,7 @@ public Block evalBlock(Block b) { } builder.appendInt(value); valuesAppended = true; - } catch (NumberFormatException e) { + } catch (InvalidArgumentException | NumberFormatException e) { registerException(e); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromUnsignedLongEvaluator.java index ccd1edc4aa6c2..34128e44f1500 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromUnsignedLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerFromUnsignedLongEvaluator.java @@ -14,7 +14,6 @@ import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.xpack.ql.InvalidArgumentException; -import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import org.elasticsearch.xpack.ql.tree.Source; /** @@ -39,7 +38,7 @@ public Block evalVector(Vector v) { if (vector.isConstant()) { try { return driverContext.blockFactory().newConstantIntBlockWith(evalValue(vector, 0), positionCount); - } catch (InvalidArgumentException | QlIllegalArgumentException e) { + } catch (InvalidArgumentException e) { registerException(e); return driverContext.blockFactory().newConstantNullBlock(positionCount); } @@ -48,7 +47,7 @@ public Block evalVector(Vector v) { for (int p = 0; p < positionCount; p++) { try { builder.appendInt(evalValue(vector, p)); - } catch (InvalidArgumentException | QlIllegalArgumentException e) { + } catch (InvalidArgumentException e) { registerException(e); builder.appendNull(); } @@ -82,7 +81,7 @@ public Block evalBlock(Block b) { } builder.appendInt(value); valuesAppended = true; - } catch (InvalidArgumentException | QlIllegalArgumentException e) { + } catch (InvalidArgumentException e) { registerException(e); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromIntEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromIntEvaluator.java index d3ccf82f2cb05..703f0729654a8 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromIntEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromIntEvaluator.java @@ -13,6 +13,7 @@ import org.elasticsearch.compute.data.Vector; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.ql.InvalidArgumentException; import org.elasticsearch.xpack.ql.tree.Source; /** @@ -35,11 +36,21 @@ public Block evalVector(Vector v) { IntVector vector = (IntVector) v; int positionCount = v.getPositionCount(); if (vector.isConstant()) { - return driverContext.blockFactory().newConstantLongBlockWith(evalValue(vector, 0), positionCount); + try { + return driverContext.blockFactory().newConstantLongBlockWith(evalValue(vector, 0), positionCount); + } catch (InvalidArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } } try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { for (int p = 0; p < positionCount; p++) { - builder.appendLong(evalValue(vector, p)); + try { + builder.appendLong(evalValue(vector, p)); + } catch (InvalidArgumentException e) { + registerException(e); + builder.appendNull(); + } } return builder.build(); } @@ -62,13 +73,17 @@ public Block evalBlock(Block b) { boolean positionOpened = false; boolean valuesAppended = false; for (int i = start; i < end; i++) { - long value = evalValue(block, i); - if (positionOpened == false && valueCount > 1) { - builder.beginPositionEntry(); - positionOpened = true; + try { + long value = evalValue(block, i); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendLong(value); + valuesAppended = true; + } catch (InvalidArgumentException e) { + registerException(e); } - builder.appendLong(value); - valuesAppended = true; } if (valuesAppended == false) { builder.appendNull(); diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromLongEvaluator.java index 2f01aef20edde..b43b961f5d34a 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromLongEvaluator.java @@ -12,6 +12,7 @@ import org.elasticsearch.compute.data.Vector; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.ql.InvalidArgumentException; import org.elasticsearch.xpack.ql.tree.Source; /** @@ -34,11 +35,21 @@ public Block evalVector(Vector v) { LongVector vector = (LongVector) v; int positionCount = v.getPositionCount(); if (vector.isConstant()) { - return driverContext.blockFactory().newConstantLongBlockWith(evalValue(vector, 0), positionCount); + try { + return driverContext.blockFactory().newConstantLongBlockWith(evalValue(vector, 0), positionCount); + } catch (InvalidArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } } try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { for (int p = 0; p < positionCount; p++) { - builder.appendLong(evalValue(vector, p)); + try { + builder.appendLong(evalValue(vector, p)); + } catch (InvalidArgumentException e) { + registerException(e); + builder.appendNull(); + } } return builder.build(); } @@ -61,13 +72,17 @@ public Block evalBlock(Block b) { boolean positionOpened = false; boolean valuesAppended = false; for (int i = start; i < end; i++) { - long value = evalValue(block, i); - if (positionOpened == false && valueCount > 1) { - builder.beginPositionEntry(); - positionOpened = true; + try { + long value = evalValue(block, i); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendLong(value); + valuesAppended = true; + } catch (InvalidArgumentException e) { + registerException(e); } - builder.appendLong(value); - valuesAppended = true; } if (valuesAppended == false) { builder.appendNull(); diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromStringEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromStringEvaluator.java index 4552154560421..5b46fe2bfc9bf 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromStringEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongFromStringEvaluator.java @@ -15,6 +15,7 @@ import org.elasticsearch.compute.data.Vector; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.ql.InvalidArgumentException; import org.elasticsearch.xpack.ql.tree.Source; /** @@ -40,7 +41,7 @@ public Block evalVector(Vector v) { if (vector.isConstant()) { try { return driverContext.blockFactory().newConstantLongBlockWith(evalValue(vector, 0, scratchPad), positionCount); - } catch (NumberFormatException e) { + } catch (InvalidArgumentException | NumberFormatException e) { registerException(e); return driverContext.blockFactory().newConstantNullBlock(positionCount); } @@ -49,7 +50,7 @@ public Block evalVector(Vector v) { for (int p = 0; p < positionCount; p++) { try { builder.appendLong(evalValue(vector, p, scratchPad)); - } catch (NumberFormatException e) { + } catch (InvalidArgumentException | NumberFormatException e) { registerException(e); builder.appendNull(); } @@ -84,7 +85,7 @@ public Block evalBlock(Block b) { } builder.appendLong(value); valuesAppended = true; - } catch (NumberFormatException e) { + } catch (InvalidArgumentException | NumberFormatException e) { registerException(e); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java index 0da3717f758bf..1772916ba801c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.convert; +import joptsimple.internal.Strings; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.elasticsearch.compute.data.Block; @@ -20,12 +22,18 @@ import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.Warnings; import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; +import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.function.Function; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isType; @@ -35,6 +43,15 @@ */ public abstract class AbstractConvertFunction extends UnaryScalarFunction implements EvaluatorMapper { + // the numeric types convert functions need to handle; the other numeric types are converted upstream to one of these + private static final List NUMERIC_TYPES = List.of( + DataTypes.INTEGER, + DataTypes.LONG, + DataTypes.UNSIGNED_LONG, + DataTypes.DOUBLE + ); + public static final List STRING_TYPES = DataTypes.types().stream().filter(EsqlDataTypes::isString).toList(); + protected AbstractConvertFunction(Source source, Expression field) { super(source, field); } @@ -56,13 +73,25 @@ protected final TypeResolution resolveType() { if (childrenResolved() == false) { return new TypeResolution("Unresolved children"); } - return isType( - field(), - factories()::containsKey, - sourceText(), - null, - factories().keySet().stream().map(dt -> dt.name().toLowerCase(Locale.ROOT)).sorted().toArray(String[]::new) - ); + return isType(field(), factories()::containsKey, sourceText(), null, supportedTypesNames(factories().keySet())); + } + + public static String supportedTypesNames(Set types) { + List supportedTypesNames = new ArrayList<>(types.size()); + HashSet supportTypes = new HashSet<>(types); + if (supportTypes.containsAll(NUMERIC_TYPES)) { + supportedTypesNames.add("numeric"); + NUMERIC_TYPES.forEach(supportTypes::remove); + } + + if (types.containsAll(STRING_TYPES)) { + supportedTypesNames.add("string"); + STRING_TYPES.forEach(supportTypes::remove); + } + + supportTypes.forEach(t -> supportedTypesNames.add(t.name().toLowerCase(Locale.ROOT))); + supportedTypesNames.sort(String::compareTo); + return Strings.join(supportedTypesNames, " or "); } @FunctionalInterface diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBoolean.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBoolean.java index 442c106042fa0..3a33e086d8fdd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBoolean.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBoolean.java @@ -9,6 +9,8 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; @@ -23,6 +25,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.LONG; +import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.util.NumericUtils.unsignedLongAsNumber; @@ -31,13 +34,18 @@ public class ToBoolean extends AbstractConvertFunction { private static final Map EVALUATORS = Map.ofEntries( Map.entry(BOOLEAN, (field, source) -> field), Map.entry(KEYWORD, ToBooleanFromStringEvaluator.Factory::new), + Map.entry(TEXT, ToBooleanFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToBooleanFromDoubleEvaluator.Factory::new), Map.entry(LONG, ToBooleanFromLongEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToBooleanFromUnsignedLongEvaluator.Factory::new), Map.entry(INTEGER, ToBooleanFromIntEvaluator.Factory::new) ); - public ToBoolean(Source source, Expression field) { + @FunctionInfo(returnType = "boolean") + public ToBoolean( + Source source, + @Param(name = "v", type = { "boolean", "keyword", "text", "double", "long", "unsigned_long", "integer" }) Expression field + ) { super(source, field); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java index 9910447708b44..c2f621433ca21 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java @@ -9,6 +9,8 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateParse; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.NodeInfo; @@ -23,6 +25,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.LONG; +import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; public class ToDatetime extends AbstractConvertFunction { @@ -31,12 +34,17 @@ public class ToDatetime extends AbstractConvertFunction { Map.entry(DATETIME, (field, source) -> field), Map.entry(LONG, (field, source) -> field), Map.entry(KEYWORD, ToDatetimeFromStringEvaluator.Factory::new), + Map.entry(TEXT, ToDatetimeFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToLongFromDoubleEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToLongFromUnsignedLongEvaluator.Factory::new), Map.entry(INTEGER, ToLongFromIntEvaluator.Factory::new) // CastIntToLongEvaluator would be a candidate, but not MV'd ); - public ToDatetime(Source source, Expression field) { + @FunctionInfo(returnType = "date") + public ToDatetime( + Source source, + @Param(name = "v", type = { "date", "keyword", "text", "double", "long", "unsigned_long", "integer" }) Expression field + ) { super(source, field); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegrees.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegrees.java index 6b0d638e875a0..44f8507d880d8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegrees.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegrees.java @@ -9,6 +9,8 @@ import org.elasticsearch.compute.ann.ConvertEvaluator; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; @@ -37,7 +39,8 @@ public class ToDegrees extends AbstractConvertFunction implements EvaluatorMappe ) ); - public ToDegrees(Source source, Expression field) { + @FunctionInfo(returnType = "double") + public ToDegrees(Source source, @Param(name = "v", type = { "double", "long", "unsigned_long", "integer" }) Expression field) { super(source, field); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java index e83a0eae8d7a8..7711f55d667ba 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java @@ -9,6 +9,8 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; @@ -23,6 +25,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.LONG; +import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.util.NumericUtils.unsignedLongAsNumber; @@ -33,12 +36,17 @@ public class ToDouble extends AbstractConvertFunction { Map.entry(BOOLEAN, ToDoubleFromBooleanEvaluator.Factory::new), Map.entry(DATETIME, ToDoubleFromLongEvaluator.Factory::new), // CastLongToDoubleEvaluator would be a candidate, but not MV'd Map.entry(KEYWORD, ToDoubleFromStringEvaluator.Factory::new), + Map.entry(TEXT, ToDoubleFromStringEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToDoubleFromUnsignedLongEvaluator.Factory::new), Map.entry(LONG, ToDoubleFromLongEvaluator.Factory::new), // CastLongToDoubleEvaluator would be a candidate, but not MV'd Map.entry(INTEGER, ToDoubleFromIntEvaluator.Factory::new) // CastIntToDoubleEvaluator would be a candidate, but not MV'd ); - public ToDouble(Source source, Expression field) { + @FunctionInfo(returnType = "double") + public ToDouble( + Source source, + @Param(name = "v", type = { "boolean", "date", "keyword", "text", "double", "long", "unsigned_long", "integer" }) Expression field + ) { super(source, field); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIP.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIP.java index 4829d39b09d65..97512a03fe2ec 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIP.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIP.java @@ -9,6 +9,8 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; @@ -19,16 +21,19 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.IP; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; +import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; import static org.elasticsearch.xpack.ql.util.StringUtils.parseIP; public class ToIP extends AbstractConvertFunction { private static final Map EVALUATORS = Map.ofEntries( Map.entry(IP, (field, source) -> field), - Map.entry(KEYWORD, ToIPFromStringEvaluator.Factory::new) + Map.entry(KEYWORD, ToIPFromStringEvaluator.Factory::new), + Map.entry(TEXT, ToIPFromStringEvaluator.Factory::new) ); - public ToIP(Source source, Expression field) { + @FunctionInfo(returnType = "ip") + public ToIP(Source source, @Param(name = "v", type = { "ip", "keyword", "text" }) Expression field) { super(source, field); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java index 480962ca27f86..a8e4ef804a2ba 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java @@ -9,8 +9,9 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.ql.InvalidArgumentException; -import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; @@ -19,7 +20,6 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.xpack.ql.type.DataTypeConverter.safeDoubleToLong; import static org.elasticsearch.xpack.ql.type.DataTypeConverter.safeToInt; import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; @@ -27,7 +27,9 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.LONG; +import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; +import static org.elasticsearch.xpack.ql.util.NumericUtils.unsignedLongAsNumber; public class ToInteger extends AbstractConvertFunction { @@ -36,12 +38,17 @@ public class ToInteger extends AbstractConvertFunction { Map.entry(BOOLEAN, ToIntegerFromBooleanEvaluator.Factory::new), Map.entry(DATETIME, ToIntegerFromLongEvaluator.Factory::new), Map.entry(KEYWORD, ToIntegerFromStringEvaluator.Factory::new), + Map.entry(TEXT, ToIntegerFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToIntegerFromDoubleEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToIntegerFromUnsignedLongEvaluator.Factory::new), Map.entry(LONG, ToIntegerFromLongEvaluator.Factory::new) ); - public ToInteger(Source source, Expression field) { + @FunctionInfo(returnType = "integer") + public ToInteger( + Source source, + @Param(name = "v", type = { "boolean", "date", "keyword", "text", "double", "long", "unsigned_long", "integer" }) Expression field + ) { super(source, field); } @@ -70,7 +77,7 @@ static int fromBoolean(boolean bool) { return bool ? 1 : 0; } - @ConvertEvaluator(extraName = "FromString", warnExceptions = { NumberFormatException.class }) + @ConvertEvaluator(extraName = "FromString", warnExceptions = { InvalidArgumentException.class, NumberFormatException.class }) static int fromKeyword(BytesRef in) { String asString = in.utf8ToString(); try { @@ -84,17 +91,22 @@ static int fromKeyword(BytesRef in) { } } - @ConvertEvaluator(extraName = "FromDouble", warnExceptions = { InvalidArgumentException.class, QlIllegalArgumentException.class }) + @ConvertEvaluator(extraName = "FromDouble", warnExceptions = { InvalidArgumentException.class }) static int fromDouble(double dbl) { - return fromLong(safeDoubleToLong(dbl)); + return safeToInt(dbl); } - @ConvertEvaluator(extraName = "FromUnsignedLong", warnExceptions = { InvalidArgumentException.class, QlIllegalArgumentException.class }) - static int fromUnsignedLong(long lng) { - return fromLong(ToLong.fromUnsignedLong(lng)); + @ConvertEvaluator(extraName = "FromUnsignedLong", warnExceptions = { InvalidArgumentException.class }) + static int fromUnsignedLong(long ul) { + Number n = unsignedLongAsNumber(ul); + int i = n.intValue(); + if (i != n.longValue()) { + throw new InvalidArgumentException("[{}] out of [integer] range", n); + } + return i; } - @ConvertEvaluator(extraName = "FromLong", warnExceptions = { InvalidArgumentException.class, QlIllegalArgumentException.class }) + @ConvertEvaluator(extraName = "FromLong", warnExceptions = { InvalidArgumentException.class }) static int fromLong(long lng) { return safeToInt(lng); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java index b66ad4f359607..0a2546297f038 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java @@ -9,6 +9,8 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.ql.InvalidArgumentException; import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import org.elasticsearch.xpack.ql.expression.Expression; @@ -29,6 +31,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.LONG; +import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.util.NumericUtils.unsignedLongAsNumber; @@ -41,12 +44,20 @@ public class ToLong extends AbstractConvertFunction { Map.entry(CARTESIAN_POINT, (fieldEval, source) -> fieldEval), Map.entry(BOOLEAN, ToLongFromBooleanEvaluator.Factory::new), Map.entry(KEYWORD, ToLongFromStringEvaluator.Factory::new), + Map.entry(TEXT, ToLongFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToLongFromDoubleEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToLongFromUnsignedLongEvaluator.Factory::new), Map.entry(INTEGER, ToLongFromIntEvaluator.Factory::new) // CastIntToLongEvaluator would be a candidate, but not MV'd ); - public ToLong(Source source, Expression field) { + @FunctionInfo(returnType = "long") + public ToLong( + Source source, + @Param( + name = "v", + type = { "boolean", "date", "keyword", "text", "double", "long", "unsigned_long", "integer", "geo_point", "cartesian_point" } + ) Expression field + ) { super(source, field); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadians.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadians.java index 9f39015a8e063..a1d2e1381109d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadians.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadians.java @@ -9,6 +9,8 @@ import org.elasticsearch.compute.ann.ConvertEvaluator; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; @@ -37,7 +39,8 @@ public class ToRadians extends AbstractConvertFunction implements EvaluatorMappe ) ); - public ToRadians(Source source, Expression field) { + @FunctionInfo(returnType = "double") + public ToRadians(Source source, @Param(name = "v", type = { "double", "long", "unsigned_long", "integer" }) Expression field) { super(source, field); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java index a37b2becc8595..41d8f87aee436 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java @@ -11,6 +11,7 @@ import org.elasticsearch.compute.ann.ConvertEvaluator; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.NodeInfo; @@ -55,6 +56,7 @@ public class ToString extends AbstractConvertFunction implements EvaluatorMapper Map.entry(CARTESIAN_POINT, ToStringFromCartesianPointEvaluator.Factory::new) ); + @FunctionInfo(returnType = "keyword") public ToString( Source source, @Param( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLong.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLong.java index 1b7ee01e50c54..cfa24cd6d8ff8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLong.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLong.java @@ -9,6 +9,8 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.ql.InvalidArgumentException; import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import org.elasticsearch.xpack.ql.expression.Expression; @@ -26,6 +28,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.LONG; +import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.util.NumericUtils.ONE_AS_UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.util.NumericUtils.ZERO_AS_UNSIGNED_LONG; @@ -38,12 +41,17 @@ public class ToUnsignedLong extends AbstractConvertFunction { Map.entry(DATETIME, ToUnsignedLongFromLongEvaluator.Factory::new), Map.entry(BOOLEAN, ToUnsignedLongFromBooleanEvaluator.Factory::new), Map.entry(KEYWORD, ToUnsignedLongFromStringEvaluator.Factory::new), + Map.entry(TEXT, ToUnsignedLongFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToUnsignedLongFromDoubleEvaluator.Factory::new), Map.entry(LONG, ToUnsignedLongFromLongEvaluator.Factory::new), Map.entry(INTEGER, ToUnsignedLongFromIntEvaluator.Factory::new) ); - public ToUnsignedLong(Source source, Expression field) { + @FunctionInfo(returnType = "unsigned_long") + public ToUnsignedLong( + Source source, + @Param(name = "v", type = { "boolean", "date", "keyword", "text", "double", "long", "unsigned_long", "integer" }) Expression field + ) { super(source, field); } @@ -72,7 +80,7 @@ static long fromBoolean(boolean bool) { return bool ? ONE_AS_UNSIGNED_LONG : ZERO_AS_UNSIGNED_LONG; } - @ConvertEvaluator(extraName = "FromString", warnExceptions = { NumberFormatException.class }) + @ConvertEvaluator(extraName = "FromString", warnExceptions = { InvalidArgumentException.class, NumberFormatException.class }) static long fromKeyword(BytesRef in) { String asString = in.utf8ToString(); return asLongUnsigned(safeToUnsignedLong(asString)); @@ -83,12 +91,12 @@ static long fromDouble(double dbl) { return asLongUnsigned(safeToUnsignedLong(dbl)); } - @ConvertEvaluator(extraName = "FromLong") + @ConvertEvaluator(extraName = "FromLong", warnExceptions = { InvalidArgumentException.class }) static long fromLong(long lng) { return asLongUnsigned(safeToUnsignedLong(lng)); } - @ConvertEvaluator(extraName = "FromInt") + @ConvertEvaluator(extraName = "FromInt", warnExceptions = { InvalidArgumentException.class }) static long fromInt(int i) { return fromLong(i); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersion.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersion.java index ad7712f33d947..34e8f695b23c3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersion.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersion.java @@ -9,6 +9,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.NodeInfo; @@ -31,6 +32,7 @@ public class ToVersion extends AbstractConvertFunction { Map.entry(TEXT, ToVersionFromStringEvaluator.Factory::new) ); + @FunctionInfo(returnType = "version") public ToVersion(Source source, @Param(name = "v", type = { "keyword", "text", "version" }) Expression v) { super(source, v); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index ba63afd8f1e4b..03a385592ac63 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -1292,8 +1292,7 @@ public void testRegexOnInt() { public void testUnsupportedTypesWithToString() { // DATE_PERIOD and TIME_DURATION types have been added, but not really patched through the engine; i.e. supported. - final String supportedTypes = "boolean, cartesian_point, datetime, double, geo_point, integer, ip, keyword, long, text, " - + "unsigned_long or version"; + final String supportedTypes = "boolean or cartesian_point or datetime or geo_point or ip or numeric or string or version"; verifyUnsupported( "row period = 1 year | eval to_string(period)", "line 1:28: argument of [to_string(period)] must be [" + supportedTypes + "], found value [period] type [date_period]" diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index 81f2fa98be8cc..f003170a7551d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -798,13 +798,70 @@ private static String typeErrorMessage(boolean includeOrdinal, List validTypes) { String named = NAMED_EXPECTED_TYPES.get(validTypes); if (named == null) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java index c1e9494541636..faf10d499127a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java @@ -11,6 +11,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Literal; @@ -219,19 +220,30 @@ public static void forUnaryInt( IntFunction expectedValue, int lowerBound, int upperBound, - List warnings + Function> expectedWarnings ) { unaryNumeric( suppliers, expectedEvaluatorToString, - DataTypes.INTEGER, intCases(lowerBound, upperBound), expectedType, n -> expectedValue.apply(n.intValue()), - warnings + n -> expectedWarnings.apply(n.intValue()) ); } + public static void forUnaryInt( + List suppliers, + String expectedEvaluatorToString, + DataType expectedType, + IntFunction expectedValue, + int lowerBound, + int upperBound, + List warnings + ) { + forUnaryInt(suppliers, expectedEvaluatorToString, expectedType, expectedValue, lowerBound, upperBound, unused -> warnings); + } + /** * Generate positive test cases for a unary function operating on an {@link DataTypes#LONG}. */ @@ -242,19 +254,30 @@ public static void forUnaryLong( LongFunction expectedValue, long lowerBound, long upperBound, - List warnings + Function> expectedWarnings ) { unaryNumeric( suppliers, expectedEvaluatorToString, - DataTypes.LONG, longCases(lowerBound, upperBound), expectedType, n -> expectedValue.apply(n.longValue()), - warnings + expectedWarnings ); } + public static void forUnaryLong( + List suppliers, + String expectedEvaluatorToString, + DataType expectedType, + LongFunction expectedValue, + long lowerBound, + long upperBound, + List warnings + ) { + forUnaryLong(suppliers, expectedEvaluatorToString, expectedType, expectedValue, lowerBound, upperBound, unused -> warnings); + } + /** * Generate positive test cases for a unary function operating on an {@link DataTypes#UNSIGNED_LONG}. */ @@ -265,19 +288,30 @@ public static void forUnaryUnsignedLong( Function expectedValue, BigInteger lowerBound, BigInteger upperBound, - List warnings + Function> expectedWarnings ) { unaryNumeric( suppliers, expectedEvaluatorToString, - DataTypes.UNSIGNED_LONG, ulongCases(lowerBound, upperBound), expectedType, n -> expectedValue.apply((BigInteger) n), - warnings + n -> expectedWarnings.apply((BigInteger) n) ); } + public static void forUnaryUnsignedLong( + List suppliers, + String expectedEvaluatorToString, + DataType expectedType, + Function expectedValue, + BigInteger lowerBound, + BigInteger upperBound, + List warnings + ) { + forUnaryUnsignedLong(suppliers, expectedEvaluatorToString, expectedType, expectedValue, lowerBound, upperBound, unused -> warnings); + } + /** * Generate positive test cases for a unary function operating on an {@link DataTypes#DOUBLE}. */ @@ -289,15 +323,26 @@ public static void forUnaryDouble( double lowerBound, double upperBound, List warnings + ) { + forUnaryDouble(suppliers, expectedEvaluatorToString, expectedType, expectedValue, lowerBound, upperBound, unused -> warnings); + } + + public static void forUnaryDouble( + List suppliers, + String expectedEvaluatorToString, + DataType expectedType, + DoubleFunction expectedValue, + double lowerBound, + double upperBound, + DoubleFunction> expectedWarnings ) { unaryNumeric( suppliers, expectedEvaluatorToString, - DataTypes.DOUBLE, doubleCases(lowerBound, upperBound), expectedType, n -> expectedValue.apply(n.doubleValue()), - warnings + n -> expectedWarnings.apply(n.doubleValue()) ); } @@ -311,15 +356,7 @@ public static void forUnaryBoolean( Function expectedValue, List warnings ) { - unary( - suppliers, - expectedEvaluatorToString, - DataTypes.BOOLEAN, - booleanCases(), - expectedType, - v -> expectedValue.apply((Boolean) v), - warnings - ); + unary(suppliers, expectedEvaluatorToString, booleanCases(), expectedType, v -> expectedValue.apply((Boolean) v), warnings); } /** @@ -335,7 +372,6 @@ public static void forUnaryDatetime( unaryNumeric( suppliers, expectedEvaluatorToString, - DataTypes.DATETIME, dateCases(), expectedType, n -> expectedValue.apply(Instant.ofEpochMilli(n.longValue())), @@ -356,7 +392,6 @@ public static void forUnaryGeoPoint( unaryNumeric( suppliers, expectedEvaluatorToString, - EsqlDataTypes.GEO_POINT, geoPointCases(), expectedType, n -> expectedValue.apply(n.longValue()), @@ -377,7 +412,6 @@ public static void forUnaryCartesianPoint( unaryNumeric( suppliers, expectedEvaluatorToString, - EsqlDataTypes.CARTESIAN_POINT, cartesianPointCases(), expectedType, n -> expectedValue.apply(n.longValue()), @@ -395,15 +429,7 @@ public static void forUnaryIp( Function expectedValue, List warnings ) { - unary( - suppliers, - expectedEvaluatorToString, - DataTypes.IP, - ipCases(), - expectedType, - v -> expectedValue.apply((BytesRef) v), - warnings - ); + unary(suppliers, expectedEvaluatorToString, ipCases(), expectedType, v -> expectedValue.apply((BytesRef) v), warnings); } /** @@ -414,21 +440,30 @@ public static void forUnaryStrings( String expectedEvaluatorToString, DataType expectedType, Function expectedValue, - List warnings + Function> expectedWarnings ) { - for (DataType type : EsqlDataTypes.types().stream().filter(EsqlDataTypes::isString).toList()) { + for (DataType type : AbstractConvertFunction.STRING_TYPES) { unary( suppliers, expectedEvaluatorToString, - type, stringCases(type), expectedType, v -> expectedValue.apply((BytesRef) v), - warnings + v -> expectedWarnings.apply((BytesRef) v) ); } } + public static void forUnaryStrings( + List suppliers, + String expectedEvaluatorToString, + DataType expectedType, + Function expectedValue, + List warnings + ) { + forUnaryStrings(suppliers, expectedEvaluatorToString, expectedType, expectedValue, unused -> warnings); + } + /** * Generate positive test cases for a unary function operating on an {@link DataTypes#VERSION}. */ @@ -442,7 +477,6 @@ public static void forUnaryVersion( unary( suppliers, expectedEvaluatorToString, - DataTypes.VERSION, versionCases(""), expectedType, v -> expectedValue.apply(new Version((BytesRef) v)), @@ -453,31 +487,39 @@ public static void forUnaryVersion( private static void unaryNumeric( List suppliers, String expectedEvaluatorToString, - DataType inputType, List valueSuppliers, DataType expectedOutputType, - Function expected, - List warnings + Function expectedValue, + Function> expectedWarnings ) { unary( suppliers, expectedEvaluatorToString, - inputType, valueSuppliers, expectedOutputType, - v -> expected.apply((Number) v), - warnings + v -> expectedValue.apply((Number) v), + v -> expectedWarnings.apply((Number) v) ); } - private static void unary( + private static void unaryNumeric( List suppliers, String expectedEvaluatorToString, - DataType inputType, List valueSuppliers, DataType expectedOutputType, - Function expected, + Function expected, List warnings + ) { + unaryNumeric(suppliers, expectedEvaluatorToString, valueSuppliers, expectedOutputType, expected, unused -> warnings); + } + + public static void unary( + List suppliers, + String expectedEvaluatorToString, + List valueSuppliers, + DataType expectedOutputType, + Function expectedValue, + Function> expectedWarnings ) { for (TypedDataSupplier supplier : valueSuppliers) { suppliers.add(new TestCaseSupplier(supplier.name(), List.of(supplier.type()), () -> { @@ -492,17 +534,29 @@ private static void unary( List.of(typed), expectedEvaluatorToString, expectedOutputType, - equalTo(expected.apply(value)) + equalTo(expectedValue.apply(value)) ); - for (String warning : warnings) { + for (String warning : expectedWarnings.apply(value)) { testCase = testCase.withWarning(warning); } return testCase; })); } + + } + + public static void unary( + List suppliers, + String expectedEvaluatorToString, + List valueSuppliers, + DataType expectedOutputType, + Function expected, + List warnings + ) { + unary(suppliers, expectedEvaluatorToString, valueSuppliers, expectedOutputType, expected, unused -> warnings); } - private static List intCases(int min, int max) { + public static List intCases(int min, int max) { List cases = new ArrayList<>(); if (0 <= max && 0 >= min) { cases.add(new TypedDataSupplier("<0 int>", () -> 0, DataTypes.INTEGER)); @@ -526,7 +580,7 @@ private static List intCases(int min, int max) { return cases; } - private static List longCases(long min, long max) { + public static List longCases(long min, long max) { List cases = new ArrayList<>(); if (0L <= max && 0L >= min) { cases.add(new TypedDataSupplier("<0 long>", () -> 0L, DataTypes.LONG)); @@ -551,7 +605,7 @@ private static List longCases(long min, long max) { return cases; } - private static List ulongCases(BigInteger min, BigInteger max) { + public static List ulongCases(BigInteger min, BigInteger max) { List cases = new ArrayList<>(); // Zero @@ -591,7 +645,7 @@ private static List ulongCases(BigInteger min, BigInteger max return cases; } - private static List doubleCases(double min, double max) { + public static List doubleCases(double min, double max) { List cases = new ArrayList<>(); // Zeros diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java new file mode 100644 index 0000000000000..b00cecd3f4ccc --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.function.Supplier; + +import static java.util.Collections.emptyList; + +public class ToBooleanTests extends AbstractFunctionTestCase { + public ToBooleanTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + final String read = "Attribute[channel=0]"; + final List suppliers = new ArrayList<>(); + + TestCaseSupplier.forUnaryBoolean(suppliers, read, DataTypes.BOOLEAN, b -> b, emptyList()); + + TestCaseSupplier.forUnaryInt( + suppliers, + "ToBooleanFromIntEvaluator[field=" + read + "]", + DataTypes.BOOLEAN, + i -> i != 0, + Integer.MIN_VALUE, + Integer.MAX_VALUE, + emptyList() + ); + TestCaseSupplier.forUnaryLong( + suppliers, + "ToBooleanFromLongEvaluator[field=" + read + "]", + DataTypes.BOOLEAN, + l -> l != 0, + Long.MIN_VALUE, + Long.MAX_VALUE, + emptyList() + ); + TestCaseSupplier.forUnaryUnsignedLong( + suppliers, + "ToBooleanFromUnsignedLongEvaluator[field=" + read + "]", + DataTypes.BOOLEAN, + ul -> ul.compareTo(BigInteger.ZERO) != 0, + BigInteger.ZERO, + UNSIGNED_LONG_MAX, + emptyList() + ); + TestCaseSupplier.forUnaryDouble( + suppliers, + "ToBooleanFromDoubleEvaluator[field=" + read + "]", + DataTypes.BOOLEAN, + d -> d != 0d, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY, + emptyList() + ); + TestCaseSupplier.forUnaryStrings( + suppliers, + "ToBooleanFromStringEvaluator[field=" + read + "]", + DataTypes.BOOLEAN, + bytesRef -> String.valueOf(bytesRef).toLowerCase(Locale.ROOT).equals("true"), + emptyList() + ); + + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); + } + + @Override + protected Expression build(Source source, List args) { + return new ToBoolean(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java new file mode 100644 index 0000000000000..c92c8712d1697 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java @@ -0,0 +1,152 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateParse; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.math.BigInteger; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static java.util.Collections.emptyList; + +public class ToDatetimeTests extends AbstractFunctionTestCase { + public ToDatetimeTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + final String read = "Attribute[channel=0]"; + final List suppliers = new ArrayList<>(); + + TestCaseSupplier.forUnaryDatetime(suppliers, read, DataTypes.DATETIME, Instant::toEpochMilli, emptyList()); + + TestCaseSupplier.forUnaryInt( + suppliers, + "ToLongFromIntEvaluator[field=" + read + "]", + DataTypes.DATETIME, + i -> ((Integer) i).longValue(), + Integer.MIN_VALUE, + Integer.MAX_VALUE, + emptyList() + ); + TestCaseSupplier.forUnaryLong(suppliers, read, DataTypes.DATETIME, l -> l, Long.MIN_VALUE, Long.MAX_VALUE, emptyList()); + TestCaseSupplier.forUnaryUnsignedLong( + suppliers, + "ToLongFromUnsignedLongEvaluator[field=" + read + "]", + DataTypes.DATETIME, + BigInteger::longValueExact, + BigInteger.ZERO, + BigInteger.valueOf(Long.MAX_VALUE), + emptyList() + ); + TestCaseSupplier.forUnaryUnsignedLong( + suppliers, + "ToLongFromUnsignedLongEvaluator[field=" + read + "]", + DataTypes.DATETIME, + bi -> null, + BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.TWO), + UNSIGNED_LONG_MAX, + bi -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.ql.InvalidArgumentException: [" + bi + "] out of [long] range" + ) + ); + TestCaseSupplier.forUnaryDouble( + suppliers, + "ToLongFromDoubleEvaluator[field=" + read + "]", + DataTypes.DATETIME, + d -> null, + Double.NEGATIVE_INFINITY, + -9.223372036854777E18, // a "convenient" value smaller than `(double) Long.MIN_VALUE` (== ...776E18) + d -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.ql.InvalidArgumentException: [" + d + "] out of [long] range" + ) + ); + TestCaseSupplier.forUnaryDouble( + suppliers, + "ToLongFromDoubleEvaluator[field=" + read + "]", + DataTypes.DATETIME, + d -> null, + 9.223372036854777E18, // a "convenient" value larger than `(double) Long.MAX_VALUE` (== ...776E18) + Double.POSITIVE_INFINITY, + d -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.ql.InvalidArgumentException: [" + d + "] out of [long] range" + ) + ); + TestCaseSupplier.forUnaryStrings( + suppliers, + "ToDatetimeFromStringEvaluator[field=" + read + "]", + DataTypes.DATETIME, + bytesRef -> null, + bytesRef -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.IllegalArgumentException: " + + (bytesRef.utf8ToString().isEmpty() + ? "cannot parse empty date" + : ("failed to parse date field [" + bytesRef.utf8ToString() + "] with format [yyyy-MM-dd'T'HH:mm:ss.SSS'Z']")) + ) + ); + TestCaseSupplier.unary( + suppliers, + "ToDatetimeFromStringEvaluator[field=" + read + "]", + List.of( + new TestCaseSupplier.TypedDataSupplier( + "", + // millis past "0001-01-01T00:00:00.000Z" to match the default formatter + () -> new BytesRef(Instant.ofEpochMilli(randomLongBetween(-62135596800000L, Long.MAX_VALUE)).toString()), + DataTypes.KEYWORD + ) + ), + DataTypes.DATETIME, + bytesRef -> DateParse.DEFAULT_FORMATTER.parseMillis(((BytesRef) bytesRef).utf8ToString()), + emptyList() + ); + TestCaseSupplier.unary( + suppliers, + "ToDatetimeFromStringEvaluator[field=" + read + "]", + List.of( + new TestCaseSupplier.TypedDataSupplier( + "", + // millis before "0001-01-01T00:00:00.000Z" + () -> new BytesRef(Instant.ofEpochMilli(randomLongBetween(Long.MIN_VALUE, -62135596800001L)).toString()), + DataTypes.KEYWORD + ) + ), + DataTypes.DATETIME, + bytesRef -> null, + bytesRef -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.IllegalArgumentException: failed to parse date field [" + + ((BytesRef) bytesRef).utf8ToString() + + "] with format [yyyy-MM-dd'T'HH:mm:ss.SSS'Z']" + ) + ); + + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); + } + + @Override + protected Expression build(Source source, List args) { + return new ToDatetime(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java new file mode 100644 index 0000000000000..a1c3c1f38aac5 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; + +public class ToDegreesTests extends AbstractFunctionTestCase { + public ToDegreesTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + // TODO multivalue fields + Function evaluatorName = eval -> "ToDegreesEvaluator[field=" + eval + "[field=Attribute[channel=0]]]"; + List suppliers = new ArrayList<>(); + + TestCaseSupplier.forUnaryInt( + suppliers, + evaluatorName.apply("ToDoubleFromIntEvaluator"), + DataTypes.DOUBLE, + Math::toDegrees, + Integer.MIN_VALUE, + Integer.MAX_VALUE, + List.of() + ); + TestCaseSupplier.forUnaryLong( + suppliers, + evaluatorName.apply("ToDoubleFromLongEvaluator"), + DataTypes.DOUBLE, + Math::toDegrees, + Long.MIN_VALUE, + Long.MAX_VALUE, + List.of() + ); + TestCaseSupplier.forUnaryUnsignedLong( + suppliers, + evaluatorName.apply("ToDoubleFromUnsignedLongEvaluator"), + DataTypes.DOUBLE, + ul -> Math.toDegrees(ul.doubleValue()), + BigInteger.ZERO, + UNSIGNED_LONG_MAX, + List.of() + ); + TestCaseSupplier.forUnaryDouble( + suppliers, + "ToDegreesEvaluator[field=Attribute[channel=0]]", + DataTypes.DOUBLE, + Math::toDegrees, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY, + List.of() + ); + + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); + } + + @Override + protected Expression build(Source source, List args) { + return new ToDegrees(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java new file mode 100644 index 0000000000000..ebcaf367b1226 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; + +public class ToDoubleTests extends AbstractFunctionTestCase { + public ToDoubleTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + // TODO multivalue fields + String read = "Attribute[channel=0]"; + Function evaluatorName = s -> "ToDoubleFrom" + s + "Evaluator[field=" + read + "]"; + List suppliers = new ArrayList<>(); + + TestCaseSupplier.forUnaryDouble( + suppliers, + read, + DataTypes.DOUBLE, + d -> d, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY, + List.of() + ); + + TestCaseSupplier.forUnaryBoolean(suppliers, evaluatorName.apply("Boolean"), DataTypes.DOUBLE, b -> b ? 1d : 0d, List.of()); + TestCaseSupplier.forUnaryDatetime( + suppliers, + evaluatorName.apply("Long"), + DataTypes.DOUBLE, + i -> (double) i.toEpochMilli(), + List.of() + ); + // random strings that don't look like a double + TestCaseSupplier.forUnaryStrings( + suppliers, + evaluatorName.apply("String"), + DataTypes.DOUBLE, + bytesRef -> null, + bytesRef -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.NumberFormatException: " + + (bytesRef.utf8ToString().isEmpty() ? "empty String" : ("For input string: \"" + bytesRef.utf8ToString() + "\"")) + ) + ); + TestCaseSupplier.forUnaryUnsignedLong( + suppliers, + evaluatorName.apply("UnsignedLong"), + DataTypes.DOUBLE, + BigInteger::doubleValue, + BigInteger.ZERO, + UNSIGNED_LONG_MAX, + List.of() + ); + TestCaseSupplier.forUnaryLong( + suppliers, + evaluatorName.apply("Long"), + DataTypes.DOUBLE, + l -> (double) l, + Long.MIN_VALUE, + Long.MAX_VALUE, + List.of() + ); + TestCaseSupplier.forUnaryInt( + suppliers, + evaluatorName.apply("Int"), + DataTypes.DOUBLE, + i -> (double) i, + Integer.MIN_VALUE, + Integer.MAX_VALUE, + List.of() + ); + + // strings of random numbers + TestCaseSupplier.unary( + suppliers, + evaluatorName.apply("String"), + TestCaseSupplier.castToDoubleSuppliersFromRange(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY) + .stream() + .map( + tds -> new TestCaseSupplier.TypedDataSupplier( + tds.name() + "as string", + () -> new BytesRef(tds.supplier().get().toString()), + DataTypes.KEYWORD + ) + ) + .toList(), + DataTypes.DOUBLE, + bytesRef -> Double.valueOf(((BytesRef) bytesRef).utf8ToString()), + List.of() + ); + + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); + } + + @Override + protected Expression build(Source source, List args) { + return new ToDouble(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java index 33a85f593ee6f..4294144e1cefe 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java @@ -17,16 +17,14 @@ import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.Source; -import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypes; import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; -import static org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier.stringCases; +import static java.util.Collections.emptyList; import static org.elasticsearch.xpack.ql.util.StringUtils.parseIP; -import static org.hamcrest.Matchers.equalTo; public class ToIPTests extends AbstractFunctionTestCase { public ToIPTests(@Name("TestCase") Supplier testCaseSupplier) { @@ -42,33 +40,27 @@ public static Iterable parameters() { // convert from IP to IP TestCaseSupplier.forUnaryIp(suppliers, read, DataTypes.IP, v -> v, List.of()); - // convert any kind of string to IP, with warnings. - for (TestCaseSupplier.TypedDataSupplier supplier : stringCases(DataTypes.KEYWORD)) { - suppliers.add(new TestCaseSupplier(supplier.name(), List.of(supplier.type()), () -> { - BytesRef value = (BytesRef) supplier.supplier().get(); - TestCaseSupplier.TypedData typed = new TestCaseSupplier.TypedData(value, supplier.type(), "value"); - TestCaseSupplier.TestCase testCase = new TestCaseSupplier.TestCase( - List.of(typed), - stringEvaluator, - DataTypes.IP, - equalTo(null) - ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") - .withWarning( - "Line -1:-1: java.lang.IllegalArgumentException: '" + value.utf8ToString() + "' is not an IP string literal." - ); - return testCase; - })); - } + // convert random string (i.e. not an IP representation) to IP `null`, with warnings. + TestCaseSupplier.forUnaryStrings( + suppliers, + stringEvaluator, + DataTypes.IP, + bytesRef -> null, + bytesRef -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.IllegalArgumentException: '" + bytesRef.utf8ToString() + "' is not an IP string literal." + ) + ); // convert valid IPs shaped as strings - DataType inputType = DataTypes.KEYWORD; - for (TestCaseSupplier.TypedDataSupplier ipGen : validIPsAsStrings()) { - suppliers.add(new TestCaseSupplier(ipGen.name(), List.of(inputType), () -> { - BytesRef ip = (BytesRef) ipGen.supplier().get(); - TestCaseSupplier.TypedData typed = new TestCaseSupplier.TypedData(ip, inputType, "value"); - return new TestCaseSupplier.TestCase(List.of(typed), stringEvaluator, DataTypes.IP, equalTo(parseIP(ip.utf8ToString()))); - })); - } + TestCaseSupplier.unary( + suppliers, + stringEvaluator, + validIPsAsStrings(), + DataTypes.IP, + bytesRef -> parseIP(((BytesRef) bytesRef).utf8ToString()), + emptyList() + ); // add null as parameter return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java new file mode 100644 index 0000000000000..4402c6d8529b4 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java @@ -0,0 +1,277 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.ql.type.DataTypeConverter.safeToInt; + +public class ToIntegerTests extends AbstractFunctionTestCase { + public ToIntegerTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + // TODO multivalue fields + String read = "Attribute[channel=0]"; + Function evaluatorName = s -> "ToIntegerFrom" + s + "Evaluator[field=" + read + "]"; + List suppliers = new ArrayList<>(); + + TestCaseSupplier.forUnaryInt(suppliers, read, DataTypes.INTEGER, i -> i, Integer.MIN_VALUE, Integer.MAX_VALUE, List.of()); + + TestCaseSupplier.forUnaryBoolean(suppliers, evaluatorName.apply("Boolean"), DataTypes.INTEGER, b -> b ? 1 : 0, List.of()); + + // datetimes that fall within Integer's range + TestCaseSupplier.unary( + suppliers, + evaluatorName.apply("Long"), + dateCases(0, Integer.MAX_VALUE), + DataTypes.INTEGER, + l -> ((Long) l).intValue(), + List.of() + ); + // datetimes that fall outside Integer's range + TestCaseSupplier.unary( + suppliers, + evaluatorName.apply("Long"), + dateCases(Integer.MAX_VALUE + 1L, Long.MAX_VALUE), + DataTypes.INTEGER, + l -> null, + l -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.ql.InvalidArgumentException: [" + l + "] out of [integer] range" + ) + ); + // random strings that don't look like an Integer + TestCaseSupplier.forUnaryStrings( + suppliers, + evaluatorName.apply("String"), + DataTypes.INTEGER, + bytesRef -> null, + bytesRef -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.NumberFormatException: For input string: \"" + bytesRef.utf8ToString() + "\"" + ) + ); + // from doubles within Integer's range + TestCaseSupplier.forUnaryDouble( + suppliers, + evaluatorName.apply("Double"), + DataTypes.INTEGER, + d -> safeToInt(Math.round(d)), + Integer.MIN_VALUE, + Integer.MAX_VALUE, + List.of() + ); + // from doubles outside Integer's range, negative + TestCaseSupplier.forUnaryDouble( + suppliers, + evaluatorName.apply("Double"), + DataTypes.INTEGER, + d -> null, + Double.NEGATIVE_INFINITY, + Integer.MIN_VALUE - 1d, + d -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.ql.InvalidArgumentException: [" + d + "] out of [integer] range" + ) + ); + // from doubles outside Integer's range, positive + TestCaseSupplier.forUnaryDouble( + suppliers, + evaluatorName.apply("Double"), + DataTypes.INTEGER, + d -> null, + Integer.MAX_VALUE + 1d, + Double.POSITIVE_INFINITY, + d -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.ql.InvalidArgumentException: [" + d + "] out of [integer] range" + ) + ); + + // from unsigned_long within Integer's range + TestCaseSupplier.forUnaryUnsignedLong( + suppliers, + evaluatorName.apply("UnsignedLong"), + DataTypes.INTEGER, + BigInteger::intValue, + BigInteger.ZERO, + BigInteger.valueOf(Integer.MAX_VALUE), + List.of() + ); + // from unsigned_long outside Integer's range + TestCaseSupplier.forUnaryUnsignedLong( + suppliers, + evaluatorName.apply("UnsignedLong"), + DataTypes.INTEGER, + ul -> null, + BigInteger.valueOf(Integer.MAX_VALUE).add(BigInteger.ONE), + UNSIGNED_LONG_MAX, + ul -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.ql.InvalidArgumentException: [" + ul + "] out of [integer] range" + + ) + ); + + // from long, within Integer's range + TestCaseSupplier.forUnaryLong( + suppliers, + evaluatorName.apply("Long"), + DataTypes.INTEGER, + l -> (int) l, + Integer.MIN_VALUE, + Integer.MAX_VALUE, + List.of() + ); + // from long, outside Integer's range, negative + TestCaseSupplier.forUnaryLong( + suppliers, + evaluatorName.apply("Long"), + DataTypes.INTEGER, + l -> null, + Long.MIN_VALUE, + Integer.MIN_VALUE - 1L, + l -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.ql.InvalidArgumentException: [" + l + "] out of [integer] range" + + ) + ); + // from long, outside Integer's range, positive + TestCaseSupplier.forUnaryLong( + suppliers, + evaluatorName.apply("Long"), + DataTypes.INTEGER, + l -> null, + Integer.MAX_VALUE + 1L, + Long.MAX_VALUE, + l -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.ql.InvalidArgumentException: [" + l + "] out of [integer] range" + ) + ); + + // strings of random ints within Integer's range + TestCaseSupplier.unary( + suppliers, + evaluatorName.apply("String"), + TestCaseSupplier.intCases(Integer.MIN_VALUE, Integer.MAX_VALUE) + .stream() + .map( + tds -> new TestCaseSupplier.TypedDataSupplier( + tds.name() + "as string", + () -> new BytesRef(tds.supplier().get().toString()), + DataTypes.KEYWORD + ) + ) + .toList(), + DataTypes.INTEGER, + bytesRef -> Integer.valueOf(((BytesRef) bytesRef).utf8ToString()), + List.of() + ); + // strings of random doubles within Integer's range + TestCaseSupplier.unary( + suppliers, + evaluatorName.apply("String"), + TestCaseSupplier.doubleCases(Integer.MIN_VALUE, Integer.MAX_VALUE) + .stream() + .map( + tds -> new TestCaseSupplier.TypedDataSupplier( + tds.name() + "as string", + () -> new BytesRef(tds.supplier().get().toString()), + DataTypes.KEYWORD + ) + ) + .toList(), + DataTypes.INTEGER, + bytesRef -> safeToInt(Math.round(Double.parseDouble(((BytesRef) bytesRef).utf8ToString()))), + List.of() + ); + // strings of random doubles outside Integer's range, negative + TestCaseSupplier.unary( + suppliers, + evaluatorName.apply("String"), + TestCaseSupplier.doubleCases(Double.NEGATIVE_INFINITY, Integer.MIN_VALUE - 1d) + .stream() + .map( + tds -> new TestCaseSupplier.TypedDataSupplier( + tds.name() + "as string", + () -> new BytesRef(tds.supplier().get().toString()), + DataTypes.KEYWORD + ) + ) + .toList(), + DataTypes.INTEGER, + bytesRef -> null, + bytesRef -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.NumberFormatException: For input string: \"" + ((BytesRef) bytesRef).utf8ToString() + "\"" + ) + ); + // strings of random doubles outside Integer's range, positive + TestCaseSupplier.unary( + suppliers, + evaluatorName.apply("String"), + TestCaseSupplier.doubleCases(Integer.MAX_VALUE + 1d, Double.POSITIVE_INFINITY) + .stream() + .map( + tds -> new TestCaseSupplier.TypedDataSupplier( + tds.name() + "as string", + () -> new BytesRef(tds.supplier().get().toString()), + DataTypes.KEYWORD + ) + ) + .toList(), + DataTypes.INTEGER, + bytesRef -> null, + bytesRef -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.NumberFormatException: For input string: \"" + ((BytesRef) bytesRef).utf8ToString() + "\"" + ) + ); + + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); + } + + @Override + protected Expression build(Source source, List args) { + return new ToInteger(source, args.get(0)); + } + + private static List dateCases(long min, long max) { + List dataSuppliers = new ArrayList<>(2); + if (min == 0L) { + dataSuppliers.add(new TestCaseSupplier.TypedDataSupplier("<1970-01-01T00:00:00Z>", () -> 0L, DataTypes.DATETIME)); + } + if (max <= Integer.MAX_VALUE) { + dataSuppliers.add(new TestCaseSupplier.TypedDataSupplier("<1970-01-25T20:31:23.647Z>", () -> 2147483647L, DataTypes.DATETIME)); + } + dataSuppliers.add( + new TestCaseSupplier.TypedDataSupplier("", () -> ESTestCase.randomLongBetween(min, max), DataTypes.DATETIME) + ); + return dataSuppliers; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java new file mode 100644 index 0000000000000..b153fa8489dee --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java @@ -0,0 +1,217 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.math.BigInteger; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; + +public class ToLongTests extends AbstractFunctionTestCase { + public ToLongTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + // TODO multivalue fields + String read = "Attribute[channel=0]"; + Function evaluatorName = s -> "ToLongFrom" + s + "Evaluator[field=" + read + "]"; + List suppliers = new ArrayList<>(); + + TestCaseSupplier.forUnaryLong(suppliers, read, DataTypes.LONG, l -> l, Long.MIN_VALUE, Long.MAX_VALUE, List.of()); + + TestCaseSupplier.forUnaryBoolean(suppliers, evaluatorName.apply("Boolean"), DataTypes.LONG, b -> b ? 1L : 0L, List.of()); + + // geo types + TestCaseSupplier.forUnaryGeoPoint(suppliers, read, DataTypes.LONG, i -> i, List.of()); + TestCaseSupplier.forUnaryCartesianPoint(suppliers, read, DataTypes.LONG, i -> i, List.of()); + // datetimes + TestCaseSupplier.forUnaryDatetime(suppliers, read, DataTypes.LONG, Instant::toEpochMilli, List.of()); + // random strings that don't look like a long + TestCaseSupplier.forUnaryStrings( + suppliers, + evaluatorName.apply("String"), + DataTypes.LONG, + bytesRef -> null, + bytesRef -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.NumberFormatException: For input string: \"" + bytesRef.utf8ToString() + "\"" + ) + ); + // from doubles within long's range + TestCaseSupplier.forUnaryDouble( + suppliers, + evaluatorName.apply("Double"), + DataTypes.LONG, + Math::round, + Long.MIN_VALUE, + Long.MAX_VALUE, + List.of() + ); + // from doubles outside long's range, negative + TestCaseSupplier.forUnaryDouble( + suppliers, + evaluatorName.apply("Double"), + DataTypes.LONG, + d -> null, + Double.NEGATIVE_INFINITY, + Long.MIN_VALUE - 1d, + d -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.ql.InvalidArgumentException: [" + d + "] out of [long] range" + ) + ); + // from doubles outside long's range, positive + TestCaseSupplier.forUnaryDouble( + suppliers, + evaluatorName.apply("Double"), + DataTypes.LONG, + d -> null, + Long.MAX_VALUE + 1d, + Double.POSITIVE_INFINITY, + d -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.ql.InvalidArgumentException: [" + d + "] out of [long] range" + ) + ); + + // from unsigned_long within long's range + TestCaseSupplier.forUnaryUnsignedLong( + suppliers, + evaluatorName.apply("UnsignedLong"), + DataTypes.LONG, + BigInteger::longValue, + BigInteger.ZERO, + BigInteger.valueOf(Long.MAX_VALUE), + List.of() + ); + TestCaseSupplier.forUnaryUnsignedLong( + suppliers, + evaluatorName.apply("UnsignedLong"), + DataTypes.LONG, + ul -> null, + BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE), + UNSIGNED_LONG_MAX, + ul -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.ql.InvalidArgumentException: [" + ul + "] out of [long] range" + + ) + ); + + // from integer + TestCaseSupplier.forUnaryInt( + suppliers, + evaluatorName.apply("Int"), + DataTypes.LONG, + l -> (long) l, + Integer.MIN_VALUE, + Integer.MAX_VALUE, + List.of() + ); + + // strings of random longs + TestCaseSupplier.unary( + suppliers, + evaluatorName.apply("String"), + TestCaseSupplier.longCases(Long.MIN_VALUE, Long.MAX_VALUE) + .stream() + .map( + tds -> new TestCaseSupplier.TypedDataSupplier( + tds.name() + "as string", + () -> new BytesRef(tds.supplier().get().toString()), + DataTypes.KEYWORD + ) + ) + .toList(), + DataTypes.LONG, + bytesRef -> Long.valueOf(((BytesRef) bytesRef).utf8ToString()), + List.of() + ); + // strings of random doubles within long's range + TestCaseSupplier.unary( + suppliers, + evaluatorName.apply("String"), + TestCaseSupplier.doubleCases(Long.MIN_VALUE, Long.MAX_VALUE) + .stream() + .map( + tds -> new TestCaseSupplier.TypedDataSupplier( + tds.name() + "as string", + () -> new BytesRef(tds.supplier().get().toString()), + DataTypes.KEYWORD + ) + ) + .toList(), + DataTypes.LONG, + bytesRef -> Math.round(Double.parseDouble(((BytesRef) bytesRef).utf8ToString())), + List.of() + ); + // strings of random doubles outside integer's range, negative + TestCaseSupplier.unary( + suppliers, + evaluatorName.apply("String"), + TestCaseSupplier.doubleCases(Double.NEGATIVE_INFINITY, Long.MIN_VALUE - 1d) + .stream() + .map( + tds -> new TestCaseSupplier.TypedDataSupplier( + tds.name() + "as string", + () -> new BytesRef(tds.supplier().get().toString()), + DataTypes.KEYWORD + ) + ) + .toList(), + DataTypes.LONG, + bytesRef -> null, + bytesRef -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.NumberFormatException: For input string: \"" + ((BytesRef) bytesRef).utf8ToString() + "\"" + ) + ); + // strings of random doubles outside integer's range, positive + TestCaseSupplier.unary( + suppliers, + evaluatorName.apply("String"), + TestCaseSupplier.doubleCases(Long.MAX_VALUE + 1d, Double.POSITIVE_INFINITY) + .stream() + .map( + tds -> new TestCaseSupplier.TypedDataSupplier( + tds.name() + "as string", + () -> new BytesRef(tds.supplier().get().toString()), + DataTypes.KEYWORD + ) + ) + .toList(), + DataTypes.LONG, + bytesRef -> null, + bytesRef -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.NumberFormatException: For input string: \"" + ((BytesRef) bytesRef).utf8ToString() + "\"" + ) + ); + + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); + } + + @Override + protected Expression build(Source source, List args) { + return new ToLong(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java new file mode 100644 index 0000000000000..ffd1a2734d75f --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; + +public class ToRadiansTests extends AbstractFunctionTestCase { + public ToRadiansTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + // TODO multivalue fields + Function evaluatorName = eval -> "ToRadiansEvaluator[field=" + eval + "[field=Attribute[channel=0]]]"; + List suppliers = new ArrayList<>(); + + TestCaseSupplier.forUnaryInt( + suppliers, + evaluatorName.apply("ToDoubleFromIntEvaluator"), + DataTypes.DOUBLE, + Math::toRadians, + Integer.MIN_VALUE, + Integer.MAX_VALUE, + List.of() + ); + TestCaseSupplier.forUnaryLong( + suppliers, + evaluatorName.apply("ToDoubleFromLongEvaluator"), + DataTypes.DOUBLE, + Math::toRadians, + Long.MIN_VALUE, + Long.MAX_VALUE, + List.of() + ); + TestCaseSupplier.forUnaryUnsignedLong( + suppliers, + evaluatorName.apply("ToDoubleFromUnsignedLongEvaluator"), + DataTypes.DOUBLE, + ul -> Math.toRadians(ul.doubleValue()), + BigInteger.ZERO, + UNSIGNED_LONG_MAX, + List.of() + ); + TestCaseSupplier.forUnaryDouble( + suppliers, + "ToRadiansEvaluator[field=Attribute[channel=0]]", + DataTypes.DOUBLE, + Math::toRadians, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY, + List.of() + ); + + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); + } + + @Override + protected Expression build(Source source, List args) { + return new ToRadians(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java new file mode 100644 index 0000000000000..080424602703d --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java @@ -0,0 +1,258 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataTypes; +import org.elasticsearch.xpack.ql.util.NumericUtils; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.ql.type.DataTypeConverter.safeToUnsignedLong; +import static org.elasticsearch.xpack.ql.util.NumericUtils.ONE_AS_UNSIGNED_LONG; +import static org.elasticsearch.xpack.ql.util.NumericUtils.UNSIGNED_LONG_MAX_AS_DOUBLE; +import static org.elasticsearch.xpack.ql.util.NumericUtils.ZERO_AS_UNSIGNED_LONG; +import static org.elasticsearch.xpack.ql.util.NumericUtils.asLongUnsigned; + +public class ToUnsignedLongTests extends AbstractFunctionTestCase { + public ToUnsignedLongTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + // TODO multivalue fields + String read = "Attribute[channel=0]"; + Function evaluatorName = s -> "ToUnsignedLongFrom" + s + "Evaluator[field=" + read + "]"; + List suppliers = new ArrayList<>(); + + TestCaseSupplier.forUnaryUnsignedLong( + suppliers, + read, + DataTypes.UNSIGNED_LONG, + NumericUtils::asLongUnsigned, + BigInteger.ZERO, + UNSIGNED_LONG_MAX, + List.of() + ); + + TestCaseSupplier.forUnaryBoolean( + suppliers, + evaluatorName.apply("Boolean"), + DataTypes.UNSIGNED_LONG, + b -> b ? ONE_AS_UNSIGNED_LONG : ZERO_AS_UNSIGNED_LONG, + List.of() + ); + + // datetimes + TestCaseSupplier.forUnaryDatetime( + suppliers, + evaluatorName.apply("Long"), + DataTypes.UNSIGNED_LONG, + instant -> asLongUnsigned(instant.toEpochMilli()), + List.of() + ); + // random strings that don't look like an unsigned_long + TestCaseSupplier.forUnaryStrings(suppliers, evaluatorName.apply("String"), DataTypes.UNSIGNED_LONG, bytesRef -> null, bytesRef -> { + // BigDecimal, used to parse unsigned_longs will throw NFEs with different messages depending on empty string, first + // non-number character after a number-looking like prefix, or string starting with "e", maybe others -- safer to take + // this shortcut here. + Exception e = expectThrows(NumberFormatException.class, () -> new BigDecimal(bytesRef.utf8ToString())); + return List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.NumberFormatException: " + e.getMessage() + ); + }); + // from doubles within unsigned_long's range + TestCaseSupplier.forUnaryDouble( + suppliers, + evaluatorName.apply("Double"), + DataTypes.UNSIGNED_LONG, + d -> asLongUnsigned(BigDecimal.valueOf(d).toBigInteger()), // note: not: new BigDecimal(d).toBigInteger + 0d, + UNSIGNED_LONG_MAX_AS_DOUBLE, + List.of() + ); + // from doubles outside unsigned_long's range, negative + TestCaseSupplier.forUnaryDouble( + suppliers, + evaluatorName.apply("Double"), + DataTypes.UNSIGNED_LONG, + d -> null, + Double.NEGATIVE_INFINITY, + -1d, + d -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.ql.InvalidArgumentException: [" + d + "] out of [unsigned_long] range" + ) + ); + // from doubles outside Long's range, positive + TestCaseSupplier.forUnaryDouble( + suppliers, + evaluatorName.apply("Double"), + DataTypes.UNSIGNED_LONG, + d -> null, + UNSIGNED_LONG_MAX_AS_DOUBLE + 10e5, + Double.POSITIVE_INFINITY, + d -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.ql.InvalidArgumentException: [" + d + "] out of [unsigned_long] range" + ) + ); + + // from long within unsigned_long's range + TestCaseSupplier.forUnaryLong( + suppliers, + evaluatorName.apply("Long"), + DataTypes.UNSIGNED_LONG, + NumericUtils::asLongUnsigned, + 0L, + Long.MAX_VALUE, + List.of() + ); + // from long outside unsigned_long's range + TestCaseSupplier.forUnaryLong( + suppliers, + evaluatorName.apply("Long"), + DataTypes.UNSIGNED_LONG, + unused -> null, + Long.MIN_VALUE, + -1L, + l -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.ql.InvalidArgumentException: [" + l + "] out of [unsigned_long] range" + ) + ); + + // from int within unsigned_long's range + TestCaseSupplier.forUnaryInt( + suppliers, + evaluatorName.apply("Int"), + DataTypes.UNSIGNED_LONG, + NumericUtils::asLongUnsigned, + 0, + Integer.MAX_VALUE, + List.of() + ); + // from int outside unsigned_long's range + TestCaseSupplier.forUnaryInt( + suppliers, + evaluatorName.apply("Int"), + DataTypes.UNSIGNED_LONG, + unused -> null, + Integer.MIN_VALUE, + -1, + l -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.ql.InvalidArgumentException: [" + l + "] out of [unsigned_long] range" + ) + ); + + // strings of random unsigned_longs + TestCaseSupplier.unary( + suppliers, + evaluatorName.apply("String"), + TestCaseSupplier.ulongCases(BigInteger.ZERO, UNSIGNED_LONG_MAX) + .stream() + .map( + tds -> new TestCaseSupplier.TypedDataSupplier( + tds.name() + "as string", + () -> new BytesRef(tds.supplier().get().toString()), + DataTypes.KEYWORD + ) + ) + .toList(), + DataTypes.UNSIGNED_LONG, + bytesRef -> asLongUnsigned(safeToUnsignedLong(((BytesRef) bytesRef).utf8ToString())), + List.of() + ); + // strings of random doubles within unsigned_long's range + TestCaseSupplier.unary( + suppliers, + evaluatorName.apply("String"), + TestCaseSupplier.doubleCases(0, UNSIGNED_LONG_MAX_AS_DOUBLE) + .stream() + .map( + tds -> new TestCaseSupplier.TypedDataSupplier( + tds.name() + "as string", + () -> new BytesRef(tds.supplier().get().toString()), + DataTypes.KEYWORD + ) + ) + .toList(), + DataTypes.UNSIGNED_LONG, + bytesRef -> asLongUnsigned(safeToUnsignedLong(((BytesRef) bytesRef).utf8ToString())), + List.of() + ); + // strings of random doubles outside unsigned_long's range, negative + TestCaseSupplier.unary( + suppliers, + evaluatorName.apply("String"), + TestCaseSupplier.doubleCases(Double.NEGATIVE_INFINITY, -1d) + .stream() + .map( + tds -> new TestCaseSupplier.TypedDataSupplier( + tds.name() + "as string", + () -> new BytesRef(tds.supplier().get().toString()), + DataTypes.KEYWORD + ) + ) + .toList(), + DataTypes.UNSIGNED_LONG, + bytesRef -> null, + bytesRef -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.ql.InvalidArgumentException: [" + + ((BytesRef) bytesRef).utf8ToString() + + "] out of [unsigned_long] range" + ) + ); + // strings of random doubles outside Integer's range, positive + TestCaseSupplier.unary( + suppliers, + evaluatorName.apply("String"), + TestCaseSupplier.doubleCases(UNSIGNED_LONG_MAX_AS_DOUBLE + 10e5, Double.POSITIVE_INFINITY) + .stream() + .map( + tds -> new TestCaseSupplier.TypedDataSupplier( + tds.name() + "as string", + () -> new BytesRef(tds.supplier().get().toString()), + DataTypes.KEYWORD + ) + ) + .toList(), + DataTypes.UNSIGNED_LONG, + bytesRef -> null, + bytesRef -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.ql.InvalidArgumentException: [" + + ((BytesRef) bytesRef).utf8ToString() + + "] out of [unsigned_long] range" + ) + ); + + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); + } + + @Override + protected Expression build(Source source, List args) { + return new ToUnsignedLong(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java index fefa397f7c77f..c6e2abae14443 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java @@ -13,7 +13,6 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataType; @@ -24,8 +23,6 @@ import java.util.List; import java.util.function.Supplier; -import static org.hamcrest.Matchers.equalTo; - public class ToVersionTests extends AbstractFunctionTestCase { public ToVersionTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); @@ -37,9 +34,12 @@ public static Iterable parameters() { String read = "Attribute[channel=0]"; String stringEvaluator = "ToVersionFromStringEvaluator[field=" + read + "]"; List suppliers = new ArrayList<>(); + // Converting and IP to an IP doesn't change anything. Everything should succeed. - TestCaseSupplier.forUnaryVersion(suppliers, read, DataTypes.VERSION, v -> v.toBytesRef(), List.of()); - // None of the random strings ever look like versions so they should all become "invalid" versions + TestCaseSupplier.forUnaryVersion(suppliers, read, DataTypes.VERSION, Version::toBytesRef, List.of()); + + // None of the random strings ever look like versions so they should all become "invalid" versions: + // https://github.com/elastic/elasticsearch/issues/98989 // TODO should this return null with warnings? they aren't version shaped at all. TestCaseSupplier.forUnaryStrings( suppliers, @@ -48,20 +48,19 @@ public static Iterable parameters() { bytesRef -> new Version(bytesRef.utf8ToString()).toBytesRef(), List.of() ); + // But strings that are shaped like versions do parse to valid versions - for (DataType inputType : EsqlDataTypes.types().stream().filter(EsqlDataTypes::isString).toList()) { - for (TestCaseSupplier.TypedDataSupplier versionGen : TestCaseSupplier.versionCases(inputType.typeName() + " ")) { - suppliers.add(new TestCaseSupplier(versionGen.name(), List.of(inputType), () -> { - BytesRef encodedVersion = (BytesRef) versionGen.supplier().get(); - TestCaseSupplier.TypedData typed = new TestCaseSupplier.TypedData( - new BytesRef(new Version(encodedVersion).toString()), - inputType, - "value" - ); - return new TestCaseSupplier.TestCase(List.of(typed), stringEvaluator, DataTypes.VERSION, equalTo(encodedVersion)); - })); - } + for (DataType inputType : AbstractConvertFunction.STRING_TYPES) { + TestCaseSupplier.unary( + suppliers, + read, + TestCaseSupplier.versionCases(inputType.typeName() + " "), + DataTypes.VERSION, + bytesRef -> new Version((BytesRef) bytesRef).toBytesRef(), + List.of() + ); } + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java index 3adc63c9863cb..e08224aaffdd5 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java @@ -32,6 +32,7 @@ import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.ScalingExecutorBuilder; import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; import org.elasticsearch.xpack.core.inference.action.DeleteInferenceModelAction; import org.elasticsearch.xpack.core.inference.action.GetInferenceModelAction; import org.elasticsearch.xpack.core.inference.action.InferenceAction; @@ -39,6 +40,7 @@ import org.elasticsearch.xpack.inference.action.TransportDeleteInferenceModelAction; import org.elasticsearch.xpack.inference.action.TransportGetInferenceModelAction; import org.elasticsearch.xpack.inference.action.TransportInferenceAction; +import org.elasticsearch.xpack.inference.action.TransportInferenceUsageAction; import org.elasticsearch.xpack.inference.action.TransportPutInferenceModelAction; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.HttpSettings; @@ -86,7 +88,8 @@ public InferencePlugin(Settings settings) { new ActionHandler<>(InferenceAction.INSTANCE, TransportInferenceAction.class), new ActionHandler<>(GetInferenceModelAction.INSTANCE, TransportGetInferenceModelAction.class), new ActionHandler<>(PutInferenceModelAction.INSTANCE, TransportPutInferenceModelAction.class), - new ActionHandler<>(DeleteInferenceModelAction.INSTANCE, TransportDeleteInferenceModelAction.class) + new ActionHandler<>(DeleteInferenceModelAction.INSTANCE, TransportDeleteInferenceModelAction.class), + new ActionHandler<>(XPackUsageFeatureAction.INFERENCE, TransportInferenceUsageAction.class) ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceUsageAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceUsageAction.java new file mode 100644 index 0000000000000..54452d8a7ed68 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceUsageAction.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.OriginSettingClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.protocol.xpack.XPackUsageRequest; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction; +import org.elasticsearch.xpack.core.inference.InferenceFeatureSetUsage; +import org.elasticsearch.xpack.core.inference.action.GetInferenceModelAction; + +import java.util.Map; +import java.util.TreeMap; + +import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; + +public class TransportInferenceUsageAction extends XPackUsageFeatureTransportAction { + + private final Client client; + + @Inject + public TransportInferenceUsageAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + Client client + ) { + super( + XPackUsageFeatureAction.INFERENCE.name(), + transportService, + clusterService, + threadPool, + actionFilters, + indexNameExpressionResolver + ); + this.client = new OriginSettingClient(client, ML_ORIGIN); + } + + @Override + protected void masterOperation( + Task task, + XPackUsageRequest request, + ClusterState state, + ActionListener listener + ) throws Exception { + GetInferenceModelAction.Request getInferenceModelAction = new GetInferenceModelAction.Request("_all", TaskType.ANY); + client.execute(GetInferenceModelAction.INSTANCE, getInferenceModelAction, ActionListener.wrap(response -> { + Map stats = new TreeMap<>(); + for (ModelConfigurations model : response.getModels()) { + String statKey = model.getService() + ":" + model.getTaskType().name(); + InferenceFeatureSetUsage.ModelStats stat = stats.computeIfAbsent( + statKey, + key -> new InferenceFeatureSetUsage.ModelStats(model.getService(), model.getTaskType()) + ); + stat.add(); + } + InferenceFeatureSetUsage usage = new InferenceFeatureSetUsage(stats.values()); + listener.onResponse(new XPackUsageFeatureResponse(usage)); + }, listener::onFailure)); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModel.java index 5e2c352d88a01..02c1e41e0374a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModel.java @@ -84,8 +84,11 @@ public ExecutableAction accept(OpenAiActionVisitor creator, Map } public OpenAiEmbeddingsModel overrideWith(Map taskSettings) { - var requestTaskSettings = OpenAiEmbeddingsRequestTaskSettings.fromMap(taskSettings); + if (taskSettings == null || taskSettings.isEmpty()) { + return this; + } + var requestTaskSettings = OpenAiEmbeddingsRequestTaskSettings.fromMap(taskSettings); return new OpenAiEmbeddingsModel(this, getTaskSettings().overrideWith(requestTaskSettings)); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceUsageActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceUsageActionTests.java new file mode 100644 index 0000000000000..b0c59fe160be3 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceUsageActionTests.java @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ServiceSettings; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.protocol.xpack.XPackUsageRequest; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; +import org.elasticsearch.xpack.core.inference.InferenceFeatureSetUsage; +import org.elasticsearch.xpack.core.inference.action.GetInferenceModelAction; +import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource; +import org.junit.After; +import org.junit.Before; + +import java.util.List; + +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.core.Is.is; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TransportInferenceUsageActionTests extends ESTestCase { + + private Client client; + private TransportInferenceUsageAction action; + + @Before + public void init() { + client = mock(Client.class); + ThreadPool threadPool = new TestThreadPool("test"); + when(client.threadPool()).thenReturn(threadPool); + + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(mock(ThreadPool.class)); + + action = new TransportInferenceUsageAction( + transportService, + mock(ClusterService.class), + mock(ThreadPool.class), + mock(ActionFilters.class), + mock(IndexNameExpressionResolver.class), + client + ); + } + + @After + public void close() { + client.threadPool().shutdown(); + } + + public void test() throws Exception { + doAnswer(invocation -> { + @SuppressWarnings("unchecked") + var listener = (ActionListener) invocation.getArguments()[2]; + listener.onResponse( + new GetInferenceModelAction.Response( + List.of( + new ModelConfigurations("model-001", TaskType.TEXT_EMBEDDING, "openai", mock(ServiceSettings.class)), + new ModelConfigurations("model-002", TaskType.TEXT_EMBEDDING, "openai", mock(ServiceSettings.class)), + new ModelConfigurations("model-003", TaskType.SPARSE_EMBEDDING, "hugging_face_elser", mock(ServiceSettings.class)), + new ModelConfigurations("model-004", TaskType.TEXT_EMBEDDING, "openai", mock(ServiceSettings.class)), + new ModelConfigurations("model-005", TaskType.SPARSE_EMBEDDING, "openai", mock(ServiceSettings.class)), + new ModelConfigurations("model-006", TaskType.SPARSE_EMBEDDING, "hugging_face_elser", mock(ServiceSettings.class)) + ) + ) + ); + return Void.TYPE; + }).when(client).execute(any(GetInferenceModelAction.class), any(), any()); + + PlainActionFuture future = new PlainActionFuture<>(); + action.masterOperation(mock(Task.class), mock(XPackUsageRequest.class), mock(ClusterState.class), future); + + BytesStreamOutput out = new BytesStreamOutput(); + future.get().getUsage().writeTo(out); + XPackFeatureSet.Usage usage = new InferenceFeatureSetUsage(out.bytes().streamInput()); + + assertThat(usage.name(), is(XPackField.INFERENCE)); + assertTrue(usage.enabled()); + assertTrue(usage.available()); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + usage.toXContent(builder, ToXContent.EMPTY_PARAMS); + XContentSource source = new XContentSource(builder); + assertThat(source.getValue("models"), hasSize(3)); + assertThat(source.getValue("models.0.service"), is("hugging_face_elser")); + assertThat(source.getValue("models.0.task_type"), is("SPARSE_EMBEDDING")); + assertThat(source.getValue("models.0.count"), is(2)); + assertThat(source.getValue("models.1.service"), is("openai")); + assertThat(source.getValue("models.1.task_type"), is("SPARSE_EMBEDDING")); + assertThat(source.getValue("models.1.count"), is(1)); + assertThat(source.getValue("models.2.service"), is("openai")); + assertThat(source.getValue("models.2.task_type"), is("TEXT_EMBEDDING")); + assertThat(source.getValue("models.2.count"), is(3)); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/SparseEmbeddingResultsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/SparseEmbeddingResultsTests.java index 0a8bfd20caaf1..6f8fa0c453d09 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/SparseEmbeddingResultsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/SparseEmbeddingResultsTests.java @@ -11,12 +11,14 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; +import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; +import static org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfig.DEFAULT_RESULTS_FIELD; import static org.hamcrest.Matchers.is; public class SparseEmbeddingResultsTests extends AbstractWireSerializingTestCase { @@ -151,6 +153,25 @@ public void testToXContent_CreatesTheRightFormatForMultipleEmbeddings() throws I }""")); } + public void testTransformToCoordinationFormat() { + var results = createSparseResult( + List.of( + createEmbedding(List.of(new SparseEmbeddingResults.WeightedToken("token", 0.1F)), false), + createEmbedding(List.of(new SparseEmbeddingResults.WeightedToken("token2", 0.2F)), true) + ) + ).transformToCoordinationFormat(); + + assertThat( + results, + is( + List.of( + new TextExpansionResults(DEFAULT_RESULTS_FIELD, List.of(new TextExpansionResults.WeightedToken("token", 0.1F)), false), + new TextExpansionResults(DEFAULT_RESULTS_FIELD, List.of(new TextExpansionResults.WeightedToken("token2", 0.2F)), true) + ) + ) + ); + } + public record EmbeddingExpectation(Map tokens, boolean isTruncated) {} public static Map buildExpectation(List embeddings) { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingResultsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingResultsTests.java index 71d14e09872fd..09d9894d98853 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingResultsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingResultsTests.java @@ -100,6 +100,30 @@ public void testToXContent_CreatesTheRightFormatForMultipleEmbeddings() throws I }""")); } + public void testTransformToCoordinationFormat() { + var results = new TextEmbeddingResults( + List.of(new TextEmbeddingResults.Embedding(List.of(0.1F, 0.2F)), new TextEmbeddingResults.Embedding(List.of(0.3F, 0.4F))) + ).transformToCoordinationFormat(); + + assertThat( + results, + is( + List.of( + new org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingResults( + TextEmbeddingResults.TEXT_EMBEDDING, + new double[] { 0.1F, 0.2F }, + false + ), + new org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingResults( + TextEmbeddingResults.TEXT_EMBEDDING, + new double[] { 0.3F, 0.4F }, + false + ) + ) + ) + ); + } + @Override protected Writeable.Reader instanceReader() { return TextEmbeddingResults::new; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModelTests.java index 96ced66723f04..62cb609a59d2a 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModelTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModelTests.java @@ -14,8 +14,11 @@ import org.elasticsearch.xpack.inference.services.openai.OpenAiServiceSettings; import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; +import java.util.Map; + import static org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsRequestTaskSettingsTests.getRequestTaskSettingsMap; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.sameInstance; public class OpenAiEmbeddingsModelTests extends ESTestCase { @@ -28,6 +31,22 @@ public void testOverrideWith_OverridesUser() { assertThat(overriddenModel, is(createModel("url", "org", "api_key", "model_name", "user_override"))); } + public void testOverrideWith_EmptyMap() { + var model = createModel("url", "org", "api_key", "model_name", null); + + var requestTaskSettingsMap = Map.of(); + + var overriddenModel = model.overrideWith(requestTaskSettingsMap); + assertThat(overriddenModel, sameInstance(model)); + } + + public void testOverrideWith_NullMap() { + var model = createModel("url", "org", "api_key", "model_name", null); + + var overriddenModel = model.overrideWith(null); + assertThat(overriddenModel, sameInstance(model)); + } + public static OpenAiEmbeddingsModel createModel( String url, @Nullable String org, diff --git a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle index fca019a6fc689..bf6ab9ed7d77e 100644 --- a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle +++ b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle @@ -17,7 +17,7 @@ testClusters.configureEach { setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.history_index_enabled', 'false' setting 'slm.history_index_enabled', 'false' - requiresFeature 'es.inference_rescorer_feature_flag_enabled', Version.fromString("8.10.0") + requiresFeature 'es.learn_to_rank_feature_flag_enabled', Version.fromString("8.10.0") } if (BuildParams.inFipsJvm){ diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle index b28e6bec462b9..b8b706353d624 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle @@ -181,7 +181,7 @@ tasks.named("yamlRestTest").configure { 'ml/inference_crud/Test put nlp model config with vocabulary set', 'ml/inference_crud/Test put model model aliases with nlp model', 'ml/inference_processor/Test create processor with missing mandatory fields', - 'ml/inference_rescore/Test rescore with missing model', + 'ml/learn_to_rank_rescorer/Test rescore with missing model', 'ml/inference_stats_crud/Test get stats given missing trained model', 'ml/inference_stats_crud/Test get stats given expression without matches and allow_no_match is false', 'ml/jobs_crud/Test cannot create job with model snapshot id set', @@ -258,5 +258,5 @@ testClusters.configureEach { user username: "no_ml", password: "x-pack-test-password", role: "minimal" setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' - requiresFeature 'es.inference_rescorer_feature_flag_enabled', Version.fromString("8.10.0") + requiresFeature 'es.learn_to_rank_feature_flag_enabled', Version.fromString("8.10.0") } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index db23e7796f862..d0f7302105768 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -324,8 +324,8 @@ import org.elasticsearch.xpack.ml.inference.deployment.DeploymentManager; import org.elasticsearch.xpack.ml.inference.ingest.InferenceProcessor; import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService; -import org.elasticsearch.xpack.ml.inference.ltr.InferenceRescorerFeature; import org.elasticsearch.xpack.ml.inference.ltr.LearnToRankRescorerBuilder; +import org.elasticsearch.xpack.ml.inference.ltr.LearnToRankRescorerFeature; import org.elasticsearch.xpack.ml.inference.ltr.LearnToRankService; import org.elasticsearch.xpack.ml.inference.modelsize.MlModelSizeNamedXContentProvider; import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider; @@ -886,8 +886,7 @@ private static void reportClashingNodeAttribute(String attrName) { @Override public List> getRescorers() { - if (enabled && InferenceRescorerFeature.isEnabled()) { - // Inference rescorer requires access to the model loading service + if (enabled && LearnToRankRescorerFeature.isEnabled()) { return List.of( new RescorerSpec<>( LearnToRankRescorerBuilder.NAME, @@ -1798,7 +1797,7 @@ public List getNamedXContent() { ); namedXContent.addAll(new CorrelationNamedContentProvider().getNamedXContentParsers()); // LTR Combine with Inference named content provider when feature flag is removed - if (InferenceRescorerFeature.isEnabled()) { + if (LearnToRankRescorerFeature.isEnabled()) { namedXContent.addAll(new MlLTRNamedXContentProvider().getNamedXContentParsers()); } return namedXContent; @@ -1886,7 +1885,7 @@ public List getNamedWriteables() { namedWriteables.addAll(new CorrelationNamedContentProvider().getNamedWriteables()); namedWriteables.addAll(new ChangePointNamedContentProvider().getNamedWriteables()); // LTR Combine with Inference named content provider when feature flag is removed - if (InferenceRescorerFeature.isEnabled()) { + if (LearnToRankRescorerFeature.isEnabled()) { namedWriteables.addAll(new MlLTRNamedXContentProvider().getNamedWriteables()); } return namedWriteables; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCoordinatedInferenceAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCoordinatedInferenceAction.java index d90c9ec807495..13e04772683eb 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCoordinatedInferenceAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCoordinatedInferenceAction.java @@ -182,7 +182,7 @@ private void replaceErrorOnMissing( } static InferModelAction.Response translateInferenceServiceResponse(InferenceServiceResults inferenceResults) { - var legacyResults = new ArrayList(inferenceResults.transformToLegacyFormat()); + var legacyResults = new ArrayList(inferenceResults.transformToCoordinationFormat()); return new InferModelAction.Response(legacyResults, null, false); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/InferenceRescorerFeature.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearnToRankRescorerFeature.java similarity index 61% rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/InferenceRescorerFeature.java rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearnToRankRescorerFeature.java index 8a26714c7c06b..18b2c6fe5ff3f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/InferenceRescorerFeature.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearnToRankRescorerFeature.java @@ -10,19 +10,19 @@ import org.elasticsearch.common.util.FeatureFlag; /** - * Inference rescorer feature flag. When the feature is complete, this flag will be removed. + * Learn to rank feature flag. When the feature is complete, this flag will be removed. * * Upon removal, ensure transport serialization is all corrected for future BWC. * * See {@link LearnToRankRescorerBuilder} */ -public class InferenceRescorerFeature { +public class LearnToRankRescorerFeature { - private InferenceRescorerFeature() {} + private LearnToRankRescorerFeature() {} - private static final FeatureFlag INFERENCE_RESCORE_FEATURE_FLAG = new FeatureFlag("inference_rescorer"); + private static final FeatureFlag LEARN_TO_RANK = new FeatureFlag("learn_to_rank"); public static boolean isEnabled() { - return INFERENCE_RESCORE_FEATURE_FLAG.isEnabled(); + return LEARN_TO_RANK.isEnabled(); } } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java index 5d834e0303a37..df6fded49e6bb 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java @@ -130,6 +130,7 @@ public static SegmentInfo wrap(SegmentInfo segmentInfo) { segmentInfo.name, segmentInfo.maxDoc(), segmentInfo.getUseCompoundFile(), + segmentInfo.getHasBlocks(), codec, segmentInfo.getDiagnostics(), segmentInfo.getId(), diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50SegmentInfoFormat.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50SegmentInfoFormat.java index cf4437a230c0d..a260722ee3501 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50SegmentInfoFormat.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50SegmentInfoFormat.java @@ -70,7 +70,20 @@ public SegmentInfo read(Directory dir, String segment, byte[] segmentID, IOConte final Set files = input.readSetOfStrings(); final Map attributes = input.readMapOfStrings(); - si = new SegmentInfo(dir, version, null, segment, docCount, isCompoundFile, null, diagnostics, segmentID, attributes, null); + si = new SegmentInfo( + dir, + version, + null, + segment, + docCount, + isCompoundFile, + false, + null, + diagnostics, + segmentID, + attributes, + null + ); si.setFiles(files); } catch (Throwable exception) { priorE = exception; diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62SegmentInfoFormat.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62SegmentInfoFormat.java index b700c39591819..5416b1a9fbc5a 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62SegmentInfoFormat.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62SegmentInfoFormat.java @@ -210,6 +210,7 @@ public SegmentInfo read(Directory dir, String segment, byte[] segmentID, IOConte segment, docCount, isCompoundFile, + false, null, diagnostics, segmentID, diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java index bb7fa9cf8c03a..87f30a89577c2 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java @@ -382,6 +382,14 @@ public static int safeToInt(long x) { return (int) x; } + public static int safeToInt(double x) { + if (x > Integer.MAX_VALUE || x < Integer.MIN_VALUE) { + throw new InvalidArgumentException("[{}] out of [integer] range", x); + } + // cast is safe, double can represent all of int's range + return (int) Math.round(x); + } + public static long safeDoubleToLong(double x) { if (x > Long.MAX_VALUE || x < Long.MIN_VALUE) { throw new InvalidArgumentException("[{}] out of [long] range", x); diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInput.java index aab3e83a4f496..ea85a91677c46 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInput.java @@ -341,7 +341,7 @@ public String toString() { private InputStream openBlobStream(int part, long pos, long length) throws IOException { assert MetadataCachingIndexInput.assertCurrentThreadMayAccessBlobStore(); stats.addBlobStoreBytesRequested(length); - return blobContainer.readBlob(OperationPurpose.SNAPSHOT, fileInfo.partName(part), pos, length); + return blobContainer.readBlob(OperationPurpose.SNAPSHOT_DATA, fileInfo.partName(part), pos, length); } private static class StreamForSequentialReads implements Closeable { diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java index 2b61dc18e266c..e9f4ab11c9b7c 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java @@ -528,7 +528,7 @@ protected InputStream openInputStreamFromBlobStore(final long position, final lo assert position + readLength <= fileInfo.length() : "cannot read [" + position + "-" + (position + readLength) + "] from [" + fileInfo + "]"; stats.addBlobStoreBytesRequested(readLength); - return directory.blobContainer().readBlob(OperationPurpose.SNAPSHOT, fileInfo.name(), position, readLength); + return directory.blobContainer().readBlob(OperationPurpose.SNAPSHOT_DATA, fileInfo.name(), position, readLength); } return openInputStreamMultipleParts(position, readLength); } @@ -558,7 +558,7 @@ protected InputStream openSlice(int slice) throws IOException { ? getRelativePositionInPart(position + readLength - 1) + 1 : fileInfo.partBytes(currentPart); return directory.blobContainer() - .readBlob(OperationPurpose.SNAPSHOT, fileInfo.partName(currentPart), startInPart, endInPart - startInPart); + .readBlob(OperationPurpose.SNAPSHOT_DATA, fileInfo.partName(currentPart), startInPart, endInPart - startInPart); } }; } diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 5412e7d05f27f..3409f549cb579 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -127,6 +127,7 @@ public class Constants { "cluster:admin/xpack/connector/get", "cluster:admin/xpack/connector/list", "cluster:admin/xpack/connector/put", + "cluster:admin/xpack/connector/update_error", "cluster:admin/xpack/connector/update_filtering", "cluster:admin/xpack/connector/update_last_seen", "cluster:admin/xpack/connector/update_last_sync_stats", @@ -135,6 +136,7 @@ public class Constants { "cluster:admin/xpack/connector/sync_job/post", "cluster:admin/xpack/connector/sync_job/delete", "cluster:admin/xpack/connector/sync_job/check_in", + "cluster:admin/xpack/connector/sync_job/get", "cluster:admin/xpack/connector/sync_job/cancel", "cluster:admin/xpack/deprecation/info", "cluster:admin/xpack/deprecation/nodes/info", @@ -416,6 +418,7 @@ public class Constants { "cluster:monitor/xpack/usage/graph", "cluster:monitor/xpack/usage/health_api", "cluster:monitor/xpack/usage/ilm", + "cluster:monitor/xpack/usage/inference", "cluster:monitor/xpack/usage/logstash", "cluster:monitor/xpack/usage/ml", "cluster:monitor/xpack/usage/monitoring", diff --git a/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java b/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java index a0e0fd621ba46..3fd8e952d626e 100644 --- a/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java +++ b/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java @@ -43,7 +43,7 @@ public class XPackRestIT extends AbstractXPackRestTest { .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB") .user("x_pack_rest_user", "x-pack-test-password") .feature(FeatureFlag.TIME_SERIES_MODE) - .feature(FeatureFlag.INFERENCE_RESCORER) + .feature(FeatureFlag.LEARN_TO_RANK) .configFile("testnode.pem", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem")) .configFile("testnode.crt", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt")) .configFile("service_tokens", Resource.fromClasspath("service_tokens"))