From 8f359a91fa99301308a60a0d78f26ba58a2f70f2 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 10 Oct 2023 10:43:31 +0100 Subject: [PATCH 01/15] Add known-issue docs for snapshot downgrades bug (#100412) (#100576) --- docs/reference/release-notes/8.10.0.asciidoc | 30 ++++++++++++++++++++ docs/reference/release-notes/8.10.1.asciidoc | 6 ++++ docs/reference/release-notes/8.10.2.asciidoc | 6 +++- 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/docs/reference/release-notes/8.10.0.asciidoc b/docs/reference/release-notes/8.10.0.asciidoc index ea0af6c485f25..b8a57c702b8e7 100644 --- a/docs/reference/release-notes/8.10.0.asciidoc +++ b/docs/reference/release-notes/8.10.0.asciidoc @@ -3,6 +3,36 @@ Also see <>. +[[known-issues-8.10.0]] +[float] +=== Known issues + +// tag::repositorydata-format-change[] +* Snapshot-based downgrades ++ +The snapshot repository format changed in a manner that prevents earlier +versions of Elasticsearch from reading the repository contents if it contains +snapshots from this version and the last cluster to write to this repository +was in the 8.10 series. This will prevent you from reverting an upgrade to the +8.10 series by restoring a snapshot taken before the upgrade. ++ +Snapshot repositories written by clusters running versions 8.11.0 and later are +compatible with all earlier versions. Moreover, clusters running version 8.11.0 +or later will also automatically repair the repository format the first time +they write to the repository to take or delete a snapshot, making it so that +all earlier versions can read its contents again. ++ +If you wish to downgrade to a version prior to 8.9.0, take or delete a snapshot +using a cluster running version 8.11.0 or later to repair the repository format +first. If you cannot repair the repository in this way, first delete all the +snapshots in the repository taken with version 8.9.0 or later. To do this will +require using a cluster running version 8.10.0 or later. ++ +If you wish to downgrade to a version in the 8.9 series, you must take or +delete a snapshot using a cluster running version 8.11.0 or later to repair the +repository format first. +// end::repositorydata-format-change[] + [[breaking-8.10.0]] [float] === Breaking changes diff --git a/docs/reference/release-notes/8.10.1.asciidoc b/docs/reference/release-notes/8.10.1.asciidoc index de890ccadd772..d049d5b33b1f7 100644 --- a/docs/reference/release-notes/8.10.1.asciidoc +++ b/docs/reference/release-notes/8.10.1.asciidoc @@ -3,6 +3,12 @@ Also see <>. +[[known-issues-8.10.1]] +[float] +=== Known issues + +include::8.10.0.asciidoc[tag=repositorydata-format-change] + [[bug-8.10.1]] [float] === Bug fixes diff --git a/docs/reference/release-notes/8.10.2.asciidoc b/docs/reference/release-notes/8.10.2.asciidoc index 248f07530b8a9..c428b4534fe79 100644 --- a/docs/reference/release-notes/8.10.2.asciidoc +++ b/docs/reference/release-notes/8.10.2.asciidoc @@ -1,6 +1,10 @@ [[release-notes-8.10.2]] == {es} version 8.10.2 -8.10.2 contains no significant changes. +[[known-issues-8.10.2]] +[float] +=== Known issues + +include::8.10.0.asciidoc[tag=repositorydata-format-change] Also see <>. From 12f7321e81b209a8e265297dfe46e0aace62f81d Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 10 Oct 2023 12:04:21 +0100 Subject: [PATCH 02/15] AwaitsFix for #100587 --- .../apm/internal/metrics/InstrumentsConcurrencyTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/metrics/InstrumentsConcurrencyTests.java b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/metrics/InstrumentsConcurrencyTests.java index 51285894f27ee..db4e234418612 100644 --- a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/metrics/InstrumentsConcurrencyTests.java +++ b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/metrics/InstrumentsConcurrencyTests.java @@ -88,6 +88,7 @@ public ObservableLongCounter buildWithCallback(Consumer Date: Tue, 10 Oct 2023 09:59:40 -0400 Subject: [PATCH 03/15] [DOCS] Update links to migrating Search docs (#100237) (#100560) Some Search docs are migrating from Enterprise Search to Elasticsearch. Update links to these docs to use the new locations. Also update titles and surrounding text. --- .../apis/post-analytics-collection-event.asciidoc | 2 +- docs/reference/ingest.asciidoc | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc b/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc index d20134644dde4..84d9cb5351799 100644 --- a/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc +++ b/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc @@ -40,8 +40,8 @@ Post an event to an Analytics Collection. [[post-analytics-collection-event-request-body]] ==== {api-request-body-title} -Full request body parameters can be found in {enterprise-search-ref}/analytics-events-reference.html[Enterprise Analytics Events^]. +Full request body parameters can be found in: <>. [[post-analytics-collection-event-prereqs]] ==== {api-prereq-title} diff --git a/docs/reference/ingest.asciidoc b/docs/reference/ingest.asciidoc index c0ef4d852b84d..ddba7c4e775ce 100644 --- a/docs/reference/ingest.asciidoc +++ b/docs/reference/ingest.asciidoc @@ -448,11 +448,11 @@ configuration. See {fleet-guide}/install-standalone-elastic-agent.html[Install s [discrete] [[pipelines-in-enterprise-search]] -=== Pipelines in Enterprise Search +=== Pipelines for search indices -When you create Elasticsearch indices for {enterprise-search-ref}/index.html[Enterprise Search^] use cases, for example, using the {enterprise-search-ref}/crawler.html[web crawler^] or {enterprise-search-ref}/connectors.html[connectors^], these indices are automatically set up with specific ingest pipelines. +When you create Elasticsearch indices for search use cases, for example, using the {enterprise-search-ref}/crawler.html[web crawler^] or {enterprise-search-ref}/connectors.html[connectors^], these indices are automatically set up with specific ingest pipelines. These processors help optimize your content for search. -Refer to the {enterprise-search-ref}/ingest-pipelines.html[Enterprise Search documentation^] for more information. +See <> for more information. [discrete] [[access-source-fields]] From 29a1692970a94b3df5103ace6e9bcc522bf6611d Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 10 Oct 2023 15:00:59 +0100 Subject: [PATCH 04/15] MatchingDirectoryReader should not use a threaded searcher (#100527) (#100597) MatchingDirectoryReader is a test reader wrapper that filters out documents matching a particular query. For each leaf, we create an IndexSearcher, execute the query against it and then use that as a filter for the leaf. This searcher is created using LuceneTestCase.newSearcher() and as such may be multi- threaded, which triggers extra index checks. For tests that are expecting certain methods to be called against internal readers a given number of times, these extra checks can add additional calls which then lead to a failure of test assumptions. Because this IndexSearcher is only executed against a single leaf it will only ever use a single thread, and so we can explicitly disable threading here. Fixes #100487 Fixes #99916 Fixes #100460 --- .../java/org/elasticsearch/index/engine/EngineTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index 03e61d5cb3037..811967c45c999 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -1513,7 +1513,7 @@ public MatchingDirectoryReader(DirectoryReader in, Query query) throws IOExcepti @Override public LeafReader wrap(LeafReader leaf) { try { - final IndexSearcher searcher = newSearcher(leaf, false); + final IndexSearcher searcher = newSearcher(leaf, false, true, false); searcher.setQueryCache(null); final Weight weight = searcher.createWeight(query, ScoreMode.COMPLETE_NO_SCORES, 1.0f); final Scorer scorer = weight.scorer(leaf.getContext()); From 0a5e41596652d65fa4a1cd51f692f290668d4b25 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Tue, 10 Oct 2023 08:02:55 -0700 Subject: [PATCH 05/15] Allow VectorFixedBuilder to become released without build (#100567) (#100607) VectorFixedBuilder can be closed without calling build and we should move its state to released in this case. --- .../elasticsearch/compute/data/BooleanVectorFixedBuilder.java | 1 + .../org/elasticsearch/compute/data/DoubleVectorFixedBuilder.java | 1 + .../org/elasticsearch/compute/data/IntVectorFixedBuilder.java | 1 + .../org/elasticsearch/compute/data/LongVectorFixedBuilder.java | 1 + .../org/elasticsearch/compute/data/X-VectorFixedBuilder.java.st | 1 + 5 files changed, 5 insertions(+) diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorFixedBuilder.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorFixedBuilder.java index 93ff57f2336bd..33daff853eecb 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorFixedBuilder.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorFixedBuilder.java @@ -69,6 +69,7 @@ public BooleanVector build() { public void close() { if (nextIndex >= 0) { // If nextIndex < 0 we've already built the vector + nextIndex = -1; blockFactory.adjustBreaker(-preAdjustedBytes, false); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorFixedBuilder.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorFixedBuilder.java index aa698a86b9c4e..7353515e8ffd8 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorFixedBuilder.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorFixedBuilder.java @@ -69,6 +69,7 @@ public DoubleVector build() { public void close() { if (nextIndex >= 0) { // If nextIndex < 0 we've already built the vector + nextIndex = -1; blockFactory.adjustBreaker(-preAdjustedBytes, false); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorFixedBuilder.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorFixedBuilder.java index 3ee3bfb40d6fa..a4755addf0b16 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorFixedBuilder.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorFixedBuilder.java @@ -69,6 +69,7 @@ public IntVector build() { public void close() { if (nextIndex >= 0) { // If nextIndex < 0 we've already built the vector + nextIndex = -1; blockFactory.adjustBreaker(-preAdjustedBytes, false); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorFixedBuilder.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorFixedBuilder.java index c95b0d07bec55..4a11012e769d8 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorFixedBuilder.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorFixedBuilder.java @@ -69,6 +69,7 @@ public LongVector build() { public void close() { if (nextIndex >= 0) { // If nextIndex < 0 we've already built the vector + nextIndex = -1; blockFactory.adjustBreaker(-preAdjustedBytes, false); } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorFixedBuilder.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorFixedBuilder.java.st index 69993d0945fe3..dfe5bb7622b2a 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorFixedBuilder.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorFixedBuilder.java.st @@ -69,6 +69,7 @@ final class $Type$VectorFixedBuilder implements $Type$Vector.FixedBuilder { public void close() { if (nextIndex >= 0) { // If nextIndex < 0 we've already built the vector + nextIndex = -1; blockFactory.adjustBreaker(-preAdjustedBytes, false); } } From 8bca914993409b58033fcc268d4add88814c5c3f Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Tue, 10 Oct 2023 16:52:07 +0100 Subject: [PATCH 06/15] ESQL Fix array Block ramBytesUsed (#100578) (#100612) This commit removes the MvOrdering enum from array Block ramBytesUsed. The enum value is shared. --- .../org/elasticsearch/compute/data/BooleanArrayBlock.java | 3 +-- .../org/elasticsearch/compute/data/BytesRefArrayBlock.java | 3 +-- .../org/elasticsearch/compute/data/DoubleArrayBlock.java | 3 +-- .../org/elasticsearch/compute/data/IntArrayBlock.java | 3 +-- .../org/elasticsearch/compute/data/LongArrayBlock.java | 3 +-- .../java/org/elasticsearch/compute/data/X-ArrayBlock.java.st | 3 +-- .../org/elasticsearch/compute/data/BlockAccountingTests.java | 2 ++ 7 files changed, 8 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java index adf1282c21fb0..9a66bf00fa71f 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java @@ -93,8 +93,7 @@ public BooleanBlock expand() { public static long ramBytesEstimated(boolean[] values, int[] firstValueIndexes, BitSet nullsMask) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) - + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); - // TODO mvordering is shared + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java index f46615307f767..9e6631b6807c6 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java @@ -96,8 +96,7 @@ public BytesRefBlock expand() { public static long ramBytesEstimated(BytesRefArray values, int[] firstValueIndexes, BitSet nullsMask) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) - + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); - // TODO mvordering is shared + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java index b0d77dd71271e..f9e1fe0c6e199 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java @@ -93,8 +93,7 @@ public DoubleBlock expand() { public static long ramBytesEstimated(double[] values, int[] firstValueIndexes, BitSet nullsMask) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) - + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); - // TODO mvordering is shared + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java index 97791a03c6044..95344bd8367c0 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java @@ -93,8 +93,7 @@ public IntBlock expand() { public static long ramBytesEstimated(int[] values, int[] firstValueIndexes, BitSet nullsMask) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) - + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); - // TODO mvordering is shared + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java index dddc5296e471e..a45abb1ed9248 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java @@ -93,8 +93,7 @@ public LongBlock expand() { public static long ramBytesEstimated(long[] values, int[] firstValueIndexes, BitSet nullsMask) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) - + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); - // TODO mvordering is shared + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st index 1f9fb93bc65c6..6a8185b43ecab 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st @@ -114,8 +114,7 @@ $endif$ public static long ramBytesEstimated($if(BytesRef)$BytesRefArray$else$$type$[]$endif$ values, int[] firstValueIndexes, BitSet nullsMask) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) - + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); - // TODO mvordering is shared + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask); } @Override diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java index 05f77357b9184..1cdafcb627f9e 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java @@ -251,6 +251,8 @@ public long accumulateObject(Object o, long shallowSize, Map fiel } else { queue.add(entry.getValue()); } + } else if (o instanceof AbstractArrayBlock && entry.getValue() instanceof Block.MvOrdering) { + // skip; MvOrdering is an enum, so instances are shared } else { queue.add(entry.getValue()); } From a74422451d474887b5b1efe7aa4a423d1974f4b1 Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Tue, 10 Oct 2023 08:53:15 -0700 Subject: [PATCH 07/15] Update 8.10.3.asciidoc (#100590) (#100614) (#100619) * Update 8.10.3.asciidoc * Update docs/reference/release-notes/8.10.3.asciidoc Co-authored-by: David Turner * Update docs/reference/release-notes/8.10.3.asciidoc --------- Co-authored-by: David Turner (cherry picked from commit 5ba96f2e2e766324a379fe1c93fff05c056211b3) --- docs/reference/release-notes/8.10.3.asciidoc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/reference/release-notes/8.10.3.asciidoc b/docs/reference/release-notes/8.10.3.asciidoc index a09beb26b4d27..b7828f52ad082 100644 --- a/docs/reference/release-notes/8.10.3.asciidoc +++ b/docs/reference/release-notes/8.10.3.asciidoc @@ -1,7 +1,11 @@ [[release-notes-8.10.3]] == {es} version 8.10.3 -coming[8.10.3] +[[known-issues-8.10.3]] +[float] +=== Known issues + +include::8.10.0.asciidoc[tag=repositorydata-format-change] Also see <>. From 64cae6209a44bef974140355aa4d7c4e491cb353 Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Tue, 10 Oct 2023 16:54:39 +0100 Subject: [PATCH 08/15] Fix NPE in DocVector ramBytesUsed (#100575) (#100613) This commit fixes a potential NPE in DocVector::ramBytesUsed. --- .../org/elasticsearch/compute/data/DocVector.java | 6 +++++- .../elasticsearch/compute/data/DocVectorTests.java | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java index 8abf0678593ec..44819359e8e44 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java @@ -203,6 +203,10 @@ public boolean equals(Object obj) { return shards.equals(other.shards) && segments.equals(other.segments) && docs.equals(other.docs); } + private static long ramBytesOrZero(int[] array) { + return array == null ? 0 : RamUsageEstimator.shallowSizeOf(array); + } + public static long ramBytesEstimated( IntVector shards, IntVector segments, @@ -211,7 +215,7 @@ public static long ramBytesEstimated( int[] shardSegmentDocMapBackwards ) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(shards) + RamUsageEstimator.sizeOf(segments) + RamUsageEstimator.sizeOf(docs) - + RamUsageEstimator.shallowSizeOf(shardSegmentDocMapForwards) + RamUsageEstimator.shallowSizeOf(shardSegmentDocMapBackwards); + + ramBytesOrZero(shardSegmentDocMapForwards) + ramBytesOrZero(shardSegmentDocMapBackwards); } @Override diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/DocVectorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/DocVectorTests.java index 350425840a598..e2eff15fcb769 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/DocVectorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/DocVectorTests.java @@ -150,6 +150,17 @@ public void testCannotDoubleRelease() { assertThat(e.getMessage(), containsString("can't build page out of released blocks")); } + public void testRamBytesUsedWithout() { + DocVector docs = new DocVector( + IntBlock.newConstantBlockWith(0, 1).asVector(), + IntBlock.newConstantBlockWith(0, 1).asVector(), + IntBlock.newConstantBlockWith(0, 1).asVector(), + false + ); + assertThat(docs.singleSegmentNonDecreasing(), is(false)); + docs.ramBytesUsed(); // ensure non-singleSegmentNonDecreasing handles nulls in ramByteUsed + } + IntVector intRange(int startInclusive, int endExclusive) { return IntVector.range(startInclusive, endExclusive, BlockFactory.getNonBreakingInstance()); } From fe150af3d678d9a9f85cdf04ec95731a255bc851 Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Tue, 10 Oct 2023 17:02:52 +0100 Subject: [PATCH 09/15] Re-enable org.elasticsearch.xpack.esql.action.EsqlActionIT.testFilterWithNullAndEvalFromIndex (#100604) (#100616) This commit re-enables org.elasticsearch.xpack.esql.action.EsqlActionIT.testFilterWithNullAndEvalFromIndex, which now passes successfully. --- .../java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java index fd4fe13b9c1b1..f10ca17d741d8 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java @@ -574,7 +574,6 @@ public void testStringLength() { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99826") public void testFilterWithNullAndEvalFromIndex() { // append entry, with an absent count, to the index client().prepareBulk().add(new IndexRequest("test").id("no_count").source("data", 12, "data_d", 2d, "color", "red")).get(); From 3bc97049935e5d329396aa86dea3d6cd5890bb73 Mon Sep 17 00:00:00 2001 From: Jason Bryan Date: Tue, 10 Oct 2023 12:50:43 -0400 Subject: [PATCH 10/15] Bump versions after 7.17.14 release --- .buildkite/pipelines/intake.yml | 2 +- .buildkite/pipelines/periodic-packaging.yml | 16 ++++++++++++++++ .buildkite/pipelines/periodic.yml | 11 +++++++++++ .ci/bwcVersions | 1 + .ci/snapshotBwcVersions | 2 +- .../src/main/java/org/elasticsearch/Version.java | 1 + 6 files changed, 31 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index 07cf55e0b3adf..dacb30376d2ec 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -40,7 +40,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["7.17.14", "8.10.3", "8.11.0"] + BWC_VERSION: ["7.17.15", "8.10.3", "8.11.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 6a2492fb03ef9..ab8037a8d2d3b 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -1056,6 +1056,22 @@ steps: env: BWC_VERSION: 7.17.14 + - label: "{{matrix.image}} / 7.17.15 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.15 + timeout_in_minutes: 300 + matrix: + setup: + image: + - rocky-8 + - ubuntu-2004 + agents: + provider: gcp + image: family/elasticsearch-{{matrix.image}} + machineType: custom-16-32768 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 7.17.15 + - label: "{{matrix.image}} / 8.0.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.0 timeout_in_minutes: 300 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index f0a3cfdfd1f50..bbfa1c0cb9b45 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -642,6 +642,16 @@ steps: buildDirectory: /dev/shm/bk env: BWC_VERSION: 7.17.14 + - label: 7.17.15 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.15#bwcTest + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 7.17.15 - label: 8.0.0 / bwc command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.0.0#bwcTest timeout_in_minutes: 300 @@ -1176,6 +1186,7 @@ steps: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 + buildDirectory: /dev/shm/bk - label: third-party / s3 command: | export amazon_s3_bucket=elasticsearch-ci.us-west-2 diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 90ba9b8131dcd..a2487240e55c9 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -63,6 +63,7 @@ BWC_VERSION: - "7.17.12" - "7.17.13" - "7.17.14" + - "7.17.15" - "8.0.0" - "8.0.1" - "8.1.0" diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index 1bc1fa321d9da..69322c4bf1826 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,4 +1,4 @@ BWC_VERSION: - - "7.17.14" + - "7.17.15" - "8.10.3" - "8.11.0" diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 591f93a34726f..7f0e1b9f9f558 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -114,6 +114,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_7_17_12 = new Version(7_17_12_99); public static final Version V_7_17_13 = new Version(7_17_13_99); public static final Version V_7_17_14 = new Version(7_17_14_99); + public static final Version V_7_17_15 = new Version(7_17_15_99); public static final Version V_8_0_0 = new Version(8_00_00_99); public static final Version V_8_0_1 = new Version(8_00_01_99); public static final Version V_8_1_0 = new Version(8_01_00_99); From f72906288dce763bdbcc278441082fc72b6588ed Mon Sep 17 00:00:00 2001 From: Jason Bryan Date: Tue, 10 Oct 2023 12:53:29 -0400 Subject: [PATCH 11/15] Prune changelogs after 7.17.14 release --- docs/changelog/100106.yaml | 5 ----- docs/changelog/100134.yaml | 5 ----- docs/changelog/100179.yaml | 6 ------ docs/changelog/100207.yaml | 5 ----- docs/changelog/100284.yaml | 5 ----- docs/changelog/99231.yaml | 5 ----- docs/changelog/99604.yaml | 5 ----- docs/changelog/99660.yaml | 5 ----- docs/changelog/99673.yaml | 5 ----- docs/changelog/99677.yaml | 5 ----- docs/changelog/99724.yaml | 5 ----- docs/changelog/99738.yaml | 6 ------ docs/changelog/99803.yaml | 5 ----- docs/changelog/99814.yaml | 6 ------ docs/changelog/99818.yaml | 6 ------ docs/changelog/99846.yaml | 5 ----- docs/changelog/99868.yaml | 6 ------ docs/changelog/99892.yaml | 6 ------ docs/changelog/99914.yaml | 5 ----- docs/changelog/99946.yaml | 5 ----- 20 files changed, 106 deletions(-) delete mode 100644 docs/changelog/100106.yaml delete mode 100644 docs/changelog/100134.yaml delete mode 100644 docs/changelog/100179.yaml delete mode 100644 docs/changelog/100207.yaml delete mode 100644 docs/changelog/100284.yaml delete mode 100644 docs/changelog/99231.yaml delete mode 100644 docs/changelog/99604.yaml delete mode 100644 docs/changelog/99660.yaml delete mode 100644 docs/changelog/99673.yaml delete mode 100644 docs/changelog/99677.yaml delete mode 100644 docs/changelog/99724.yaml delete mode 100644 docs/changelog/99738.yaml delete mode 100644 docs/changelog/99803.yaml delete mode 100644 docs/changelog/99814.yaml delete mode 100644 docs/changelog/99818.yaml delete mode 100644 docs/changelog/99846.yaml delete mode 100644 docs/changelog/99868.yaml delete mode 100644 docs/changelog/99892.yaml delete mode 100644 docs/changelog/99914.yaml delete mode 100644 docs/changelog/99946.yaml diff --git a/docs/changelog/100106.yaml b/docs/changelog/100106.yaml deleted file mode 100644 index c3e3d50d2597a..0000000000000 --- a/docs/changelog/100106.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 100106 -summary: Validate enrich index before completing policy execution -area: Ingest Node -type: bug -issues: [] diff --git a/docs/changelog/100134.yaml b/docs/changelog/100134.yaml deleted file mode 100644 index 3836ec2793050..0000000000000 --- a/docs/changelog/100134.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 100134 -summary: Implement matches() on `SourceConfirmedTextQuery` -area: Highlighting -type: enhancement -issues: [] diff --git a/docs/changelog/100179.yaml b/docs/changelog/100179.yaml deleted file mode 100644 index 2b7824c1575e6..0000000000000 --- a/docs/changelog/100179.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 100179 -summary: ILM introduce the `check-ts-end-time-passed` step -area: ILM+SLM -type: bug -issues: - - 99696 diff --git a/docs/changelog/100207.yaml b/docs/changelog/100207.yaml deleted file mode 100644 index 10e55992f0e45..0000000000000 --- a/docs/changelog/100207.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 100207 -summary: ILM the delete action waits for a TSDS index time/bounds to lapse -area: ILM+SLM -type: bug -issues: [] diff --git a/docs/changelog/100284.yaml b/docs/changelog/100284.yaml deleted file mode 100644 index 956fc472d6656..0000000000000 --- a/docs/changelog/100284.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 100284 -summary: Defend against negative datafeed start times -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/99231.yaml b/docs/changelog/99231.yaml deleted file mode 100644 index 9f5dfa1137587..0000000000000 --- a/docs/changelog/99231.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99231 -summary: Add manage permission for fleet managed threat intel indices -area: Authorization -type: enhancement -issues: [] diff --git a/docs/changelog/99604.yaml b/docs/changelog/99604.yaml deleted file mode 100644 index 0bace7aef1b26..0000000000000 --- a/docs/changelog/99604.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99604 -summary: Show a concrete error when the enrich index does not exist rather than a NullPointerException -area: Ingest Node -type: enhancement -issues: [] diff --git a/docs/changelog/99660.yaml b/docs/changelog/99660.yaml deleted file mode 100644 index ea19e24d51fff..0000000000000 --- a/docs/changelog/99660.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99660 -summary: Close expired search contexts on SEARCH thread -area: Search -type: bug -issues: [] diff --git a/docs/changelog/99673.yaml b/docs/changelog/99673.yaml deleted file mode 100644 index b48d620b21f49..0000000000000 --- a/docs/changelog/99673.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99673 -summary: Adding retry logic for start model deployment API -area: Machine Learning -type: bug -issues: [ ] diff --git a/docs/changelog/99677.yaml b/docs/changelog/99677.yaml deleted file mode 100644 index 04c1c28cf2e12..0000000000000 --- a/docs/changelog/99677.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99677 -summary: Using 1 MB chunks for elser model storage -area: Machine Learning -type: bug -issues: [ ] diff --git a/docs/changelog/99724.yaml b/docs/changelog/99724.yaml deleted file mode 100644 index 4fe78687bf72b..0000000000000 --- a/docs/changelog/99724.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99724 -summary: Upgrade bundled JDK to Java 21 -area: Packaging -type: upgrade -issues: [] diff --git a/docs/changelog/99738.yaml b/docs/changelog/99738.yaml deleted file mode 100644 index 1b65926aed741..0000000000000 --- a/docs/changelog/99738.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 99738 -summary: Ignore "index not found" error when `delete_dest_index` flag is set but the - dest index doesn't exist -area: Transform -type: bug -issues: [] diff --git a/docs/changelog/99803.yaml b/docs/changelog/99803.yaml deleted file mode 100644 index ce0929eb20e07..0000000000000 --- a/docs/changelog/99803.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99803 -summary: Do not use PIT in the presence of remote indices in source -area: Transform -type: bug -issues: [] diff --git a/docs/changelog/99814.yaml b/docs/changelog/99814.yaml deleted file mode 100644 index 1632be42b4e4c..0000000000000 --- a/docs/changelog/99814.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 99814 -summary: Fix cardinality agg for `const_keyword` -area: Aggregations -type: bug -issues: - - 99776 diff --git a/docs/changelog/99818.yaml b/docs/changelog/99818.yaml deleted file mode 100644 index 8835bcf28e050..0000000000000 --- a/docs/changelog/99818.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 99818 -summary: Add checks in term and terms queries that input terms are not too long -area: Search -type: enhancement -issues: - - 99802 diff --git a/docs/changelog/99846.yaml b/docs/changelog/99846.yaml deleted file mode 100644 index 198b0b6f939ac..0000000000000 --- a/docs/changelog/99846.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99846 -summary: Update version range in `jvm.options` for the Panama Vector API -area: Vector Search -type: bug -issues: [] diff --git a/docs/changelog/99868.yaml b/docs/changelog/99868.yaml deleted file mode 100644 index 33d582f9ebd0a..0000000000000 --- a/docs/changelog/99868.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 99868 -summary: Fix fields API for `geo_point` fields inside other arrays -area: Search -type: bug -issues: - - 99781 diff --git a/docs/changelog/99892.yaml b/docs/changelog/99892.yaml deleted file mode 100644 index 5090d1d888b65..0000000000000 --- a/docs/changelog/99892.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 99892 -summary: Support $ and / in restore rename replacements -area: Snapshot/Restore -type: bug -issues: - - 99078 diff --git a/docs/changelog/99914.yaml b/docs/changelog/99914.yaml deleted file mode 100644 index 8b0026a8ff9ca..0000000000000 --- a/docs/changelog/99914.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99914 -summary: Let `_stats` internally timeout if checkpoint information can not be retrieved -area: Transform -type: bug -issues: [] diff --git a/docs/changelog/99946.yaml b/docs/changelog/99946.yaml deleted file mode 100644 index 11dc4090baa0e..0000000000000 --- a/docs/changelog/99946.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99946 -summary: Skip settings validation during desired nodes updates -area: Distributed -type: bug -issues: [] From 7787cc6ab09b482df74d444c3de64b67837cbe51 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Tue, 10 Oct 2023 17:54:15 +0100 Subject: [PATCH 12/15] [ML] Unmute now fixed JobUpdateTest/testMergeWithJob (#100611) (#100623) Closes #98626 --- .../elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java index 543360fc24d89..09ff29f768dce 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java @@ -258,7 +258,6 @@ protected JobUpdate doParseInstance(XContentParser parser) { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98626") public void testMergeWithJob() { List detectorUpdates = new ArrayList<>(); List detectionRules1 = Collections.singletonList( From ae1b419d56fcd3d5e6248c1d9983c71a4280a0ed Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Tue, 10 Oct 2023 10:00:57 -0700 Subject: [PATCH 13/15] Use separate search execution context for each pipeline (#100563) (#100606) Synthetic source doesn't seem to work correctly with either inter-segment or intra-segment parallelism. Neither of these parallelisms were available when the synthetic source was developed. The new test fails with the doc or segment data_partitioning. While we are working on a proper fix, this PR introduces a workaround by creating a separate search execution context for each execution pipeline, restoring the sequential execution invariants. I believe that the overhead added by this workaround is minimal. --- x-pack/plugin/esql/build.gradle | 1 + .../compute/lucene/ValueSources.java | 16 ++++- .../lucene/ValuesSourceReaderOperator.java | 5 +- .../operator/OrdinalsGroupingOperator.java | 13 +++- .../ValuesSourceReaderOperatorTests.java | 2 +- .../xpack/esql/action/SyntheticSourceIT.java | 69 +++++++++++++++++++ .../planner/EsPhysicalOperationProviders.java | 20 +++--- 7 files changed, 111 insertions(+), 15 deletions(-) create mode 100644 x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/SyntheticSourceIT.java diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle index a21c3d0990333..9643e2b2d8e1e 100644 --- a/x-pack/plugin/esql/build.gradle +++ b/x-pack/plugin/esql/build.gradle @@ -34,6 +34,7 @@ dependencies { testImplementation('org.webjars.npm:fontsource__roboto-mono:4.5.7') internalClusterTestImplementation project(":client:rest-high-level") + internalClusterTestImplementation project(":modules:mapper-extras") } /* diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValueSources.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValueSources.java index b7eb47a7a52d3..e5ce5436990b7 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValueSources.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValueSources.java @@ -29,6 +29,7 @@ import org.elasticsearch.search.aggregations.support.FieldContext; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.internal.ShardSearchRequest; import java.io.IOException; import java.util.ArrayList; @@ -49,7 +50,20 @@ public static List sources( List sources = new ArrayList<>(searchContexts.size()); for (SearchContext searchContext : searchContexts) { - SearchExecutionContext ctx = searchContext.getSearchExecutionContext(); + // TODO: remove this workaround + // Create a separate SearchExecutionContext for each ValuesReader, as it seems that + // the synthetic source doesn't work properly with inter-segment or intra-segment parallelism. + ShardSearchRequest shardRequest = searchContext.request(); + SearchExecutionContext ctx = searchContext.readerContext() + .indexService() + .newSearchExecutionContext( + shardRequest.shardId().id(), + shardRequest.shardRequestIndex(), + searchContext.searcher(), + shardRequest::nowInMillis, + shardRequest.getClusterAlias(), + shardRequest.getRuntimeMappings() + ); var fieldType = ctx.getFieldType(fieldName); if (fieldType == null) { sources.add(new ValueSourceInfo(new NullValueSourceType(), new NullValueSource(), elementType, ctx.getIndexReader())); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java index b3ac80ee099b7..83fc902bd5077 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.Objects; import java.util.TreeMap; +import java.util.function.Supplier; /** * Operator that extracts doc_values from a Lucene index out of pages that have been produced by {@link LuceneSourceOperator} @@ -42,12 +43,12 @@ public class ValuesSourceReaderOperator extends AbstractPageMappingOperator { * @param docChannel the channel containing the shard, leaf/segment and doc id * @param field the lucene field being loaded */ - public record ValuesSourceReaderOperatorFactory(List sources, int docChannel, String field) + public record ValuesSourceReaderOperatorFactory(Supplier> sources, int docChannel, String field) implements OperatorFactory { @Override public Operator get(DriverContext driverContext) { - return new ValuesSourceReaderOperator(sources, docChannel, field); + return new ValuesSourceReaderOperator(sources.get(), docChannel, field); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java index 1c068815f1aae..4dab7faa2a074 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java @@ -42,6 +42,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.joining; @@ -51,7 +52,7 @@ */ public class OrdinalsGroupingOperator implements Operator { public record OrdinalsGroupingOperatorFactory( - List sources, + Supplier> sources, int docChannel, String groupingField, List aggregators, @@ -61,7 +62,15 @@ public record OrdinalsGroupingOperatorFactory( @Override public Operator get(DriverContext driverContext) { - return new OrdinalsGroupingOperator(sources, docChannel, groupingField, aggregators, maxPageSize, bigArrays, driverContext); + return new OrdinalsGroupingOperator( + sources.get(), + docChannel, + groupingField, + aggregators, + maxPageSize, + bigArrays, + driverContext + ); } @Override diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java index 3ce202c0e4608..ec1697e9aedd2 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java @@ -109,7 +109,7 @@ static Operator.OperatorFactory factory(IndexReader reader, ValuesSourceType vsT FieldContext fc = new FieldContext(ft.name(), fd, ft); ValuesSource vs = vsType.getField(fc, null); return new ValuesSourceReaderOperator.ValuesSourceReaderOperatorFactory( - List.of(new ValueSourceInfo(vsType, vs, elementType, reader)), + () -> List.of(new ValueSourceInfo(vsType, vs, elementType, reader)), 0, ft.name() ); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/SyntheticSourceIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/SyntheticSourceIT.java new file mode 100644 index 0000000000000..f0365ce78f44a --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/SyntheticSourceIT.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.action; + +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; + +public class SyntheticSourceIT extends AbstractEsqlIntegTestCase { + @Override + protected Collection> nodePlugins() { + var plugins = new ArrayList<>(super.nodePlugins()); + plugins.add(MapperExtrasPlugin.class); + return plugins; + } + + public void testMatchOnlyText() throws Exception { + XContentBuilder mapping = JsonXContent.contentBuilder(); + mapping.startObject(); + if (true || randomBoolean()) { + mapping.startObject("_source"); + mapping.field("mode", "synthetic"); + mapping.endObject(); + } + { + mapping.startObject("properties"); + mapping.startObject("uid"); + mapping.field("type", "keyword"); + mapping.endObject(); + mapping.startObject("name"); + mapping.field("type", "match_only_text"); + mapping.endObject(); + mapping.endObject(); + } + mapping.endObject(); + + assertAcked(client().admin().indices().prepareCreate("test").setMapping(mapping)); + + int numDocs = between(10, 1000); + for (int i = 0; i < numDocs; i++) { + IndexRequestBuilder indexRequest = client().prepareIndex("test").setSource("uid", "u" + i); + if (randomInt(100) < 5) { + indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + } + indexRequest.get(); + } + client().admin().indices().prepareRefresh("test").get(); + try (EsqlQueryResponse resp = run("from test | keep uid, name | sort uid asc | limit 1")) { + Iterator row = resp.values().next(); + assertThat(row.next(), equalTo("u0")); + assertNull(row.next()); + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index ce5e277deaad8..3131b8c8c1e20 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -15,6 +15,7 @@ import org.elasticsearch.compute.lucene.LuceneOperator; import org.elasticsearch.compute.lucene.LuceneSourceOperator; import org.elasticsearch.compute.lucene.LuceneTopNSourceOperator; +import org.elasticsearch.compute.lucene.ValueSourceInfo; import org.elasticsearch.compute.lucene.ValueSources; import org.elasticsearch.compute.lucene.ValuesSourceReaderOperator; import org.elasticsearch.compute.operator.Operator; @@ -39,10 +40,12 @@ import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.type.DataType; import java.util.ArrayList; import java.util.List; import java.util.function.Function; +import java.util.function.Supplier; import static org.elasticsearch.common.lucene.search.Queries.newNonNestedFilter; import static org.elasticsearch.compute.lucene.LuceneSourceOperator.NO_LIMIT; @@ -74,19 +77,18 @@ public final PhysicalOperation fieldExtractPhysicalOperation(FieldExtractExec fi layout.append(attr); Layout previousLayout = op.layout; - var sources = ValueSources.sources( + DataType dataType = attr.dataType(); + String fieldName = attr.name(); + Supplier> sources = () -> ValueSources.sources( searchContexts, - attr.name(), - EsqlDataTypes.isUnsupported(attr.dataType()), - LocalExecutionPlanner.toElementType(attr.dataType()) + fieldName, + EsqlDataTypes.isUnsupported(dataType), + LocalExecutionPlanner.toElementType(dataType) ); int docChannel = previousLayout.get(sourceAttr.id()).channel(); - op = op.with( - new ValuesSourceReaderOperator.ValuesSourceReaderOperatorFactory(sources, docChannel, attr.name()), - layout.build() - ); + op = op.with(new ValuesSourceReaderOperator.ValuesSourceReaderOperatorFactory(sources, docChannel, fieldName), layout.build()); } return op; } @@ -173,7 +175,7 @@ public final Operator.OperatorFactory ordinalGroupingOperatorFactory( // The grouping-by values are ready, let's group on them directly. // Costin: why are they ready and not already exposed in the layout? return new OrdinalsGroupingOperator.OrdinalsGroupingOperatorFactory( - ValueSources.sources( + () -> ValueSources.sources( searchContexts, attrSource.name(), EsqlDataTypes.isUnsupported(attrSource.dataType()), From 736420ebf8fee285eb02ab2f3f0d19dbf965a705 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 10 Oct 2023 13:12:39 -0500 Subject: [PATCH 14/15] Bump versions after 8.10.3 release --- .buildkite/pipelines/intake.yml | 2 +- .buildkite/pipelines/periodic-packaging.yml | 16 ++++++++++++++++ .buildkite/pipelines/periodic.yml | 10 ++++++++++ .ci/bwcVersions | 1 + .ci/snapshotBwcVersions | 2 +- .../src/main/java/org/elasticsearch/Version.java | 1 + 6 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index dacb30376d2ec..d723fd77c7f1d 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -40,7 +40,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["7.17.15", "8.10.3", "8.11.0"] + BWC_VERSION: ["7.17.15", "8.10.4", "8.11.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index ab8037a8d2d3b..b7aef77c76292 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -1664,6 +1664,22 @@ steps: env: BWC_VERSION: 8.10.3 + - label: "{{matrix.image}} / 8.10.4 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.10.4 + timeout_in_minutes: 300 + matrix: + setup: + image: + - rocky-8 + - ubuntu-2004 + agents: + provider: gcp + image: family/elasticsearch-{{matrix.image}} + machineType: custom-16-32768 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 8.10.4 + - label: "{{matrix.image}} / 8.11.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.0 timeout_in_minutes: 300 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index bbfa1c0cb9b45..38725d7d4bb24 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -1022,6 +1022,16 @@ steps: buildDirectory: /dev/shm/bk env: BWC_VERSION: 8.10.3 + - label: 8.10.4 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.4#bwcTest + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 8.10.4 - label: 8.11.0 / bwc command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest timeout_in_minutes: 300 diff --git a/.ci/bwcVersions b/.ci/bwcVersions index a2487240e55c9..1140870b5208a 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -101,4 +101,5 @@ BWC_VERSION: - "8.10.1" - "8.10.2" - "8.10.3" + - "8.10.4" - "8.11.0" diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index 69322c4bf1826..7e6397c559c9a 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,4 +1,4 @@ BWC_VERSION: - "7.17.15" - - "8.10.3" + - "8.10.4" - "8.11.0" diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 7f0e1b9f9f558..d39897d8dcce6 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -152,6 +152,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_8_10_1 = new Version(8_10_01_99); public static final Version V_8_10_2 = new Version(8_10_02_99); public static final Version V_8_10_3 = new Version(8_10_03_99); + public static final Version V_8_10_4 = new Version(8_10_04_99); public static final Version V_8_11_0 = new Version(8_11_00_99); public static final Version CURRENT = V_8_11_0; From 55b9617208fdd38a759a1491d6d7edc63c1c08b4 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Tue, 10 Oct 2023 20:28:03 +0200 Subject: [PATCH 15/15] Switch visibility to public in ESQL REST spec (#100622) (#100628) This update the visibility field in ESQL's REST spec to public. It also updates the types of quotes used for one the REST object parameter to backticks, for consistency. --- .../src/main/resources/rest-api-spec/api/esql.query.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/esql.query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.query.json index ffcd30fa6c717..c038ac4f3b749 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/esql.query.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.query.json @@ -5,7 +5,7 @@ "description":"Executes an ESQL request" }, "stability":"experimental", - "visibility":"private", + "visibility":"public", "headers":{ "accept": [ "application/json"], "content_type": ["application/json"] @@ -32,7 +32,7 @@ } }, "body":{ - "description":"Use the `query` element to start a query. Use `time_zone` to specify an execution time zone and 'columnar' to format the answer.", + "description":"Use the `query` element to start a query. Use `time_zone` to specify an execution time zone and `columnar` to format the answer.", "required":true } }