() {
@Override
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/ProfilingDiversifyingChildrenByteKnnVectorQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/ProfilingDiversifyingChildrenByteKnnVectorQuery.java
new file mode 100644
index 0000000000000..568516f494df4
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/vectors/ProfilingDiversifyingChildrenByteKnnVectorQuery.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.vectors;
+
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.join.BitSetProducer;
+import org.apache.lucene.search.join.DiversifyingChildrenByteKnnVectorQuery;
+import org.elasticsearch.search.profile.query.QueryProfiler;
+
+public class ProfilingDiversifyingChildrenByteKnnVectorQuery extends DiversifyingChildrenByteKnnVectorQuery implements ProfilingQuery {
+ private long vectorOpsCount;
+
+ public ProfilingDiversifyingChildrenByteKnnVectorQuery(
+ String field,
+ byte[] query,
+ Query childFilter,
+ int k,
+ BitSetProducer parentsFilter
+ ) {
+ super(field, query, childFilter, k, parentsFilter);
+ }
+
+ @Override
+ protected TopDocs mergeLeafResults(TopDocs[] perLeafResults) {
+ TopDocs topK = super.mergeLeafResults(perLeafResults);
+ vectorOpsCount = topK.totalHits.value;
+ return topK;
+ }
+
+ @Override
+ public void profile(QueryProfiler queryProfiler) {
+ queryProfiler.setVectorOpsCount(vectorOpsCount);
+ }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/ProfilingDiversifyingChildrenFloatKnnVectorQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/ProfilingDiversifyingChildrenFloatKnnVectorQuery.java
new file mode 100644
index 0000000000000..3d0d2c512dc46
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/vectors/ProfilingDiversifyingChildrenFloatKnnVectorQuery.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.vectors;
+
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.join.BitSetProducer;
+import org.apache.lucene.search.join.DiversifyingChildrenFloatKnnVectorQuery;
+import org.elasticsearch.search.profile.query.QueryProfiler;
+
+public class ProfilingDiversifyingChildrenFloatKnnVectorQuery extends DiversifyingChildrenFloatKnnVectorQuery implements ProfilingQuery {
+ private long vectorOpsCount;
+
+ public ProfilingDiversifyingChildrenFloatKnnVectorQuery(
+ String field,
+ float[] query,
+ Query childFilter,
+ int k,
+ BitSetProducer parentsFilter
+ ) {
+ super(field, query, childFilter, k, parentsFilter);
+ }
+
+ @Override
+ protected TopDocs mergeLeafResults(TopDocs[] perLeafResults) {
+ TopDocs topK = super.mergeLeafResults(perLeafResults);
+ vectorOpsCount = topK.totalHits.value;
+ return topK;
+ }
+
+ @Override
+ public void profile(QueryProfiler queryProfiler) {
+ queryProfiler.setVectorOpsCount(vectorOpsCount);
+ }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/ProfilingKnnByteVectorQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/ProfilingKnnByteVectorQuery.java
new file mode 100644
index 0000000000000..212cd51c3c22c
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/vectors/ProfilingKnnByteVectorQuery.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.vectors;
+
+import org.apache.lucene.search.KnnByteVectorQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TopDocs;
+import org.elasticsearch.search.profile.query.QueryProfiler;
+
+public class ProfilingKnnByteVectorQuery extends KnnByteVectorQuery implements ProfilingQuery {
+ private long vectorOpsCount;
+
+ public ProfilingKnnByteVectorQuery(String field, byte[] target, int k, Query filter) {
+ super(field, target, k, filter);
+ }
+
+ @Override
+ protected TopDocs mergeLeafResults(TopDocs[] perLeafResults) {
+ TopDocs topK = super.mergeLeafResults(perLeafResults);
+ vectorOpsCount = topK.totalHits.value;
+ return topK;
+ }
+
+ @Override
+ public void profile(QueryProfiler queryProfiler) {
+ queryProfiler.setVectorOpsCount(vectorOpsCount);
+ }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/ProfilingKnnFloatVectorQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/ProfilingKnnFloatVectorQuery.java
new file mode 100644
index 0000000000000..f0818d71ebeab
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/vectors/ProfilingKnnFloatVectorQuery.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.vectors;
+
+import org.apache.lucene.search.KnnFloatVectorQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TopDocs;
+import org.elasticsearch.search.profile.query.QueryProfiler;
+
+public class ProfilingKnnFloatVectorQuery extends KnnFloatVectorQuery implements ProfilingQuery {
+ private long vectorOpsCount;
+
+ public ProfilingKnnFloatVectorQuery(String field, float[] target, int k, Query filter) {
+ super(field, target, k, filter);
+ }
+
+ @Override
+ protected TopDocs mergeLeafResults(TopDocs[] perLeafResults) {
+ TopDocs topK = super.mergeLeafResults(perLeafResults);
+ vectorOpsCount = topK.totalHits.value;
+ return topK;
+ }
+
+ @Override
+ public void profile(QueryProfiler queryProfiler) {
+ queryProfiler.setVectorOpsCount(vectorOpsCount);
+ }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/ProfilingQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/ProfilingQuery.java
new file mode 100644
index 0000000000000..0c12bf38b1d14
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/vectors/ProfilingQuery.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.vectors;
+
+import org.apache.lucene.document.KnnFloatVectorField;
+import org.elasticsearch.search.profile.query.QueryProfiler;
+
+/**
+ *
+ * This interface includes the declaration of an abstract method, profile(). Classes implementing this interface
+ * must provide an implementation for profile() to store profiling information in the {@link QueryProfiler}.
+ */
+
+public interface ProfilingQuery {
+
+ /**
+ * Store the profiling information in the {@link QueryProfiler}
+ * @param queryProfiler an instance of {@link KnnFloatVectorField}.
+ */
+ void profile(QueryProfiler queryProfiler);
+}
diff --git a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index 2263bfe78f218..f7362c7001c36 100644
--- a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -39,7 +39,6 @@
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.NotSerializableExceptionWrapper;
import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.util.CancellableThreadsTests;
@@ -129,9 +128,8 @@
public class ExceptionSerializationTests extends ESTestCase {
- public void testExceptionRegistration() throws ClassNotFoundException, IOException, URISyntaxException {
+ public void testExceptionRegistration() throws IOException, URISyntaxException {
final Set> notRegistered = new HashSet<>();
- final Set> hasDedicatedWrite = new HashSet<>();
final Set> registered = new HashSet<>();
final String path = "/org/elasticsearch";
final Path startPath = PathUtils.get(ElasticsearchException.class.getProtectionDomain().getCodeSource().getLocation().toURI())
@@ -146,13 +144,13 @@ public void testExceptionRegistration() throws ClassNotFoundException, IOExcepti
private Path pkgPrefix = PathUtils.get(path).getParent();
@Override
- public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
pkgPrefix = pkgPrefix.resolve(dir.getFileName());
return FileVisitResult.CONTINUE;
}
@Override
- public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
checkFile(file.getFileName().toString());
return FileVisitResult.CONTINUE;
}
@@ -180,13 +178,6 @@ private void checkClass(Class> clazz) {
notRegistered.add(clazz);
} else if (ElasticsearchException.isRegistered(clazz.asSubclass(Throwable.class), TransportVersion.current())) {
registered.add(clazz);
- try {
- if (clazz.getMethod("writeTo", StreamOutput.class) != null) {
- hasDedicatedWrite.add(clazz);
- }
- } catch (Exception e) {
- // fair enough
- }
}
}
@@ -199,7 +190,7 @@ private Class> loadClass(String filename) throws ClassNotFoundException {
for (Path p : pkgPrefix) {
pkg.append(p.getFileName().toString()).append(".");
}
- pkg.append(filename.substring(0, filename.length() - 6));
+ pkg.append(filename, 0, filename.length() - 6);
return getClass().getClassLoader().loadClass(pkg.toString());
}
@@ -209,7 +200,7 @@ public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOExce
}
@Override
- public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+ public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
pkgPrefix = pkgPrefix.getParent();
return FileVisitResult.CONTINUE;
}
@@ -220,7 +211,7 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOEx
Files.walkFileTree(testStartPath, visitor);
assertTrue(notRegistered.remove(TestException.class));
assertTrue(notRegistered.remove(UnknownHeaderException.class));
- assertTrue("Classes subclassing ElasticsearchException must be registered \n" + notRegistered.toString(), notRegistered.isEmpty());
+ assertTrue("Classes subclassing ElasticsearchException must be registered \n" + notRegistered, notRegistered.isEmpty());
assertTrue(registered.removeAll(ElasticsearchException.getRegisteredKeys())); // check
assertEquals(registered.toString(), 0, registered.size());
}
@@ -344,7 +335,7 @@ public void testInvalidIndexTemplateException() throws IOException {
assertEquals(ex.name(), "foo");
ex = serialize(new InvalidIndexTemplateException(null, "bar"));
assertEquals(ex.getMessage(), "index_template [null] invalid, cause [bar]");
- assertEquals(ex.name(), null);
+ assertNull(ex.name());
}
public void testActionTransportException() throws IOException {
@@ -353,17 +344,12 @@ public void testActionTransportException() throws IOException {
assertEquals("[name?][" + transportAddress + "][ACTION BABY!] message?", ex.getMessage());
}
- @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/102868")
public void testSearchContextMissingException() throws IOException {
ShardSearchContextId contextId = new ShardSearchContextId(UUIDs.randomBase64UUID(), randomLong());
- TransportVersion version = TransportVersionUtils.randomVersion(random());
+ TransportVersion version = TransportVersionUtils.randomCompatibleVersion(random());
SearchContextMissingException ex = serialize(new SearchContextMissingException(contextId), version);
assertThat(ex.contextId().getId(), equalTo(contextId.getId()));
- if (version.onOrAfter(TransportVersions.V_7_7_0)) {
- assertThat(ex.contextId().getSessionId(), equalTo(contextId.getSessionId()));
- } else {
- assertThat(ex.contextId().getSessionId(), equalTo(""));
- }
+ assertThat(ex.contextId().getSessionId(), equalTo(contextId.getSessionId()));
}
public void testCircuitBreakingException() throws IOException {
@@ -422,7 +408,7 @@ public void testConnectTransportException() throws IOException {
}
public void testSearchPhaseExecutionException() throws IOException {
- ShardSearchFailure[] empty = new ShardSearchFailure[0];
+ ShardSearchFailure[] empty = ShardSearchFailure.EMPTY_ARRAY;
SearchPhaseExecutionException ex = serialize(new SearchPhaseExecutionException("boom", "baam", new NullPointerException(), empty));
assertEquals("boom", ex.getPhaseName());
assertEquals("baam", ex.getMessage());
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStatsTests.java
index 10419719a5ed1..cc4509500f9c1 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStatsTests.java
@@ -8,6 +8,7 @@
package org.elasticsearch.action.admin.cluster.stats;
+import org.apache.lucene.tests.util.LuceneTestCase;
import org.elasticsearch.TransportVersion;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.Writeable.Reader;
@@ -19,6 +20,7 @@
import java.util.List;
import java.util.Map;
+@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/102920") // failing test is final, mute whole suite
public class SearchUsageStatsTests extends AbstractWireSerializingTestCase {
private static final List QUERY_TYPES = List.of(
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java
index fec7a86bd3e59..6c79946cce15f 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java
@@ -12,9 +12,9 @@
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
-import org.apache.lucene.codecs.lucene95.Lucene95Codec;
-import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
@@ -54,7 +54,7 @@
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Weight;
-import org.apache.lucene.search.suggest.document.Completion90PostingsFormat;
+import org.apache.lucene.search.suggest.document.Completion99PostingsFormat;
import org.apache.lucene.search.suggest.document.CompletionPostingsFormat;
import org.apache.lucene.search.suggest.document.SuggestField;
import org.apache.lucene.store.Directory;
@@ -263,7 +263,7 @@ public void testKnnVectors() throws Exception {
logger.info("--> stats {}", stats);
long dataBytes = (long) numDocs * dimension * Float.BYTES; // size of flat vector data
- long indexBytesEstimate = (long) numDocs * (Lucene95HnswVectorsFormat.DEFAULT_MAX_CONN / 4); // rough size of HNSW graph
+ long indexBytesEstimate = (long) numDocs * (Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN / 4); // rough size of HNSW graph
assertThat("numDocs=" + numDocs + ";dimension=" + dimension, stats.total().getKnnVectorsBytes(), greaterThan(dataBytes));
long connectionOverhead = stats.total().getKnnVectorsBytes() - dataBytes;
assertThat("numDocs=" + numDocs, connectionOverhead, greaterThan(indexBytesEstimate));
@@ -326,11 +326,11 @@ public void testTriangle() throws Exception {
public void testCompletionField() throws Exception {
IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
.setUseCompoundFile(false)
- .setCodec(new Lucene95Codec(Lucene95Codec.Mode.BEST_SPEED) {
+ .setCodec(new Lucene99Codec(Lucene99Codec.Mode.BEST_SPEED) {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
if (field.startsWith("suggest_")) {
- return new Completion90PostingsFormat(randomFrom(CompletionPostingsFormat.FSTLoadMode.values()));
+ return new Completion99PostingsFormat(randomFrom(CompletionPostingsFormat.FSTLoadMode.values()));
} else {
return super.postingsFormat();
}
@@ -413,25 +413,25 @@ private static void addFieldsToDoc(Document doc, IndexableField[] fields) {
enum CodecMode {
BEST_SPEED {
@Override
- Lucene95Codec.Mode mode() {
- return Lucene95Codec.Mode.BEST_SPEED;
+ Lucene99Codec.Mode mode() {
+ return Lucene99Codec.Mode.BEST_SPEED;
}
},
BEST_COMPRESSION {
@Override
- Lucene95Codec.Mode mode() {
- return Lucene95Codec.Mode.BEST_COMPRESSION;
+ Lucene99Codec.Mode mode() {
+ return Lucene99Codec.Mode.BEST_COMPRESSION;
}
};
- abstract Lucene95Codec.Mode mode();
+ abstract Lucene99Codec.Mode mode();
}
static void indexRandomly(Directory directory, CodecMode codecMode, int numDocs, Consumer addFields) throws IOException {
IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
.setUseCompoundFile(randomBoolean())
- .setCodec(new Lucene95Codec(codecMode.mode()));
+ .setCodec(new Lucene99Codec(codecMode.mode()));
try (IndexWriter writer = new IndexWriter(directory, config)) {
for (int i = 0; i < numDocs; i++) {
final Document doc = new Document();
@@ -639,10 +639,10 @@ static void rewriteIndexWithPerFieldCodec(Directory source, CodecMode mode, Dire
try (DirectoryReader reader = DirectoryReader.open(source)) {
IndexWriterConfig config = new IndexWriterConfig().setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
.setUseCompoundFile(randomBoolean())
- .setCodec(new Lucene95Codec(mode.mode()) {
+ .setCodec(new Lucene99Codec(mode.mode()) {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
- return new Lucene90PostingsFormat();
+ return new Lucene99PostingsFormat();
}
@Override
@@ -652,7 +652,7 @@ public DocValuesFormat getDocValuesFormatForField(String field) {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
- return new Lucene95HnswVectorsFormat();
+ return new Lucene99HnswVectorsFormat();
}
@Override
@@ -709,7 +709,7 @@ static void collectPerFieldStats(SegmentReader reader, IndexDiskUsageStats stats
stats.addStoredField("_all_stored_fields", bytes);
case TVX, TVD -> stats.addTermVectors("_all_vectors_fields", bytes);
case NVD, NVM -> stats.addNorms("_all_norms_fields", bytes);
- case VEM, VEC, VEX -> stats.addKnnVectors(fieldLookup.getVectorsField(file), bytes);
+ case VEM, VEMF, VEC, VEX, VEQ, VEMQ -> stats.addKnnVectors(fieldLookup.getVectorsField(file), bytes);
}
}
} finally {
diff --git a/server/src/test/java/org/elasticsearch/action/search/BottomSortValuesCollectorTests.java b/server/src/test/java/org/elasticsearch/action/search/BottomSortValuesCollectorTests.java
index 31f3fe7066bed..4305d0af9a7c1 100644
--- a/server/src/test/java/org/elasticsearch/action/search/BottomSortValuesCollectorTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/BottomSortValuesCollectorTests.java
@@ -10,6 +10,7 @@
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.Pruning;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopFieldDocs;
import org.apache.lucene.search.TotalHits;
@@ -234,7 +235,7 @@ private Object[] newDateNanoArray(String... values) {
private TopFieldDocs createTopDocs(SortField sortField, int totalHits, Object[] values) {
FieldDoc[] fieldDocs = new FieldDoc[values.length];
@SuppressWarnings("unchecked")
- FieldComparator